extras/exercises/07_pytorch_experiment_tracking_exercise_template.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/mrdbourke/pytorch-deep-learning/blob/main/extras/exercises/07_pytorch_experiment_tracking_exercise_template.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zNqPNlYylluR"
   },
   "source": [
    "# 07. PyTorch Experiment Tracking Exercise Template\n",
    "\n",
    "Welcome to the 07. PyTorch Experiment Tracking exercise template notebook.\n",
    "\n",
    "> **Note:** There may be more than one solution to each of the exercises. This notebook only shows one possible example.\n",
    "\n",
    "## Resources\n",
    "\n",
    "1. These exercises/solutions are based on [section 07. PyTorch Transfer Learning](https://www.learnpytorch.io/07_pytorch_experiment_tracking/) of the Learn PyTorch for Deep Learning course by Zero to Mastery.\n",
    "2. See a live [walkthrough of the solutions (errors and all) on YouTube](https://youtu.be/cO_r2FYcAjU).\n",
    "3. See [other solutions on the course GitHub](https://github.com/mrdbourke/pytorch-deep-learning/tree/main/extras/solutions).\n",
    "\n",
    "> **Note:** The first section of this notebook is dedicated to getting various helper functions and datasets used for the exercises. The exercises start at the heading \"Exercise 1: ...\"."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "sf8ab9cyHTzU"
   },
   "source": [
    "### Get various imports and helper functions\n",
    "\n",
    "We'll need to make sure we have `torch` v.1.12+ and `torchvision` v0.13+."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5MOv1De4mxeL",
    "outputId": "a4f9fdd9-f225-4861-e204-e26186365bea"
   },
   "outputs": [],
   "source": [
    "# For this notebook to run with updated APIs, we need torch 1.12+ and torchvision 0.13+\n",
    "import torch\n",
    "import torchvision\n",
    "\n",
    "#try:\n",
    "#    import torch\n",
    "#    import torchvision\n",
    "#    assert int(torch.__version__.split(\".\")[1]) >= 12, \"torch version should be 1.12+\"\n",
    "#    assert int(torchvision.__version__.split(\".\")[1]) >= 13, \"torchvision version should be 0.13+\"\n",
    "#    print(f\"torch version: {torch.__version__}\")\n",
    "#    print(f\"torchvision version: {torchvision.__version__}\")\n",
    "#except:\n",
    "#    print(f\"[INFO] torch/torchvision versions not as required, installing nightly versions.\")\n",
    "#    !pip3 install -U --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cu113\n",
    "#    import torch\n",
    "#    import torchvision\n",
    "#    print(f\"torch version: {torch.__version__}\")\n",
    "#    print(f\"torchvision version: {torchvision.__version__}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "Nf-DsrZipCE9",
    "outputId": "f4cc7d1c-da78-4eb4-c753-4e135771650c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # Make sure we have a GPU\n",
    " device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    " device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "i_52puIeoab3"
   },
   "outputs": [],
   "source": [
    "# Get regular imports \n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torchvision\n",
    "\n",
    "from torch import nn\n",
    "from torchvision import transforms\n",
    "\n",
    "from torchinfo import summary\n",
    "\n",
    "from going_modular.sergio import data_setup, engine, helper_functions\n",
    "\n",
    "# Try to get torchinfo, install it if it doesn't work\n",
    "#try:\n",
    "#    from torchinfo import summary\n",
    "#except:\n",
    "#    print(\"[INFO] Couldn't find torchinfo... installing it.\")\n",
    "#    !pip install -q torchinfo\n",
    "#    from torchinfo import summary\n",
    "\n",
    "# Try to import the going_modular directory, download it from GitHub if it doesn't work\n",
    "#try:\n",
    "#    from going_modular.going_modular import data_setup, engine\n",
    "#except:\n",
    "#    # Get the going_modular scripts\n",
    "#    print(\"[INFO] Couldn't find going_modular scripts... downloading them from GitHub.\")\n",
    "#    !git clone https://github.com/mrdbourke/pytorch-deep-learning\n",
    "#    !mv pytorch-deep-learning/going_modular .\n",
    "#    !rm -rf pytorch-deep-learning\n",
    "#    from going_modular.going_modular import data_setup, engine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "DBj8I3P9pNK2"
   },
   "outputs": [],
   "source": [
    "# Set seeds\n",
    "def set_seeds(seed: int=42):\n",
    "    \"\"\"Sets random sets for torch operations.\n",
    "\n",
    "    Args:\n",
    "        seed (int, optional): Random seed to set. Defaults to 42.\n",
    "    \"\"\"\n",
    "    # Set the seed for general torch operations\n",
    "    torch.manual_seed(seed)\n",
    "    # Set the seed for CUDA torch operations (ones that happen on the GPU)\n",
    "    torch.cuda.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "m6R-CS53pTLS",
    "outputId": "3f7688b7-0b86-4cd8-bb11-4e71f8e7270f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] data\\pizza_steak_sushi directory exists, skipping download.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "WindowsPath('data/pizza_steak_sushi')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import zipfile\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "import requests\n",
    "\n",
    "def download_data(source: str, \n",
    "                  destination: str,\n",
    "                  remove_source: bool = True) -> Path:\n",
    "    \"\"\"Downloads a zipped dataset from source and unzips to destination.\n",
    "\n",
    "    Args:\n",
    "        source (str): A link to a zipped file containing data.\n",
    "        destination (str): A target directory to unzip data to.\n",
    "        remove_source (bool): Whether to remove the source after downloading and extracting.\n",
    "    \n",
    "    Returns:\n",
    "        pathlib.Path to downloaded data.\n",
    "    \n",
    "    Example usage:\n",
    "        download_data(source=\"https://github.com/mrdbourke/pytorch-deep-learning/raw/main/data/pizza_steak_sushi.zip\",\n",
    "                      destination=\"pizza_steak_sushi\")\n",
    "    \"\"\"\n",
    "    # Setup path to data folder\n",
    "    data_path = Path(\"data/\")\n",
    "    image_path = data_path / destination\n",
    "\n",
    "    # If the image folder doesn't exist, download it and prepare it... \n",
    "    if image_path.is_dir():\n",
    "        print(f\"[INFO] {image_path} directory exists, skipping download.\")\n",
    "    else:\n",
    "        print(f\"[INFO] Did not find {image_path} directory, creating one...\")\n",
    "        image_path.mkdir(parents=True, exist_ok=True)\n",
    "        \n",
    "        # Download pizza, steak, sushi data\n",
    "        target_file = Path(source).name\n",
    "        with open(data_path / target_file, \"wb\") as f:\n",
    "            request = requests.get(source)\n",
    "            print(f\"[INFO] Downloading {target_file} from {source}...\")\n",
    "            f.write(request.content)\n",
    "\n",
    "        # Unzip pizza, steak, sushi data\n",
    "        with zipfile.ZipFile(data_path / target_file, \"r\") as zip_ref:\n",
    "            print(f\"[INFO] Unzipping {target_file} data...\") \n",
    "            zip_ref.extractall(image_path)\n",
    "\n",
    "        # Remove .zip file\n",
    "        if remove_source:\n",
    "            os.remove(data_path / target_file)\n",
    "    \n",
    "    return image_path\n",
    "\n",
    "image_path = download_data(source=\"https://github.com/mrdbourke/pytorch-deep-learning/raw/main/data/pizza_steak_sushi.zip\",\n",
    "                           destination=\"pizza_steak_sushi\")\n",
    "image_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "BE60IEEkr89l"
   },
   "outputs": [],
   "source": [
    "from torch.utils.tensorboard import SummaryWriter\n",
    "def create_writer(experiment_name: str, \n",
    "                  model_name: str, \n",
    "                  extra: str=None):\n",
    "    \"\"\"Creates a torch.utils.tensorboard.writer.SummaryWriter() instance saving to a specific log_dir.\n",
    "\n",
    "    log_dir is a combination of runs/timestamp/experiment_name/model_name/extra.\n",
    "\n",
    "    Where timestamp is the current date in YYYY-MM-DD format.\n",
    "\n",
    "    Args:\n",
    "        experiment_name (str): Name of experiment.\n",
    "        model_name (str): Name of model.\n",
    "        extra (str, optional): Anything extra to add to the directory. Defaults to None.\n",
    "\n",
    "    Returns:\n",
    "        torch.utils.tensorboard.writer.SummaryWriter(): Instance of a writer saving to log_dir.\n",
    "\n",
    "    Example usage:\n",
    "        # Create a writer saving to \"runs/2022-06-04/data_10_percent/effnetb2/5_epochs/\"\n",
    "        writer = create_writer(experiment_name=\"data_10_percent\",\n",
    "                               model_name=\"effnetb2\",\n",
    "                               extra=\"5_epochs\")\n",
    "        # The above is the same as:\n",
    "        writer = SummaryWriter(log_dir=\"runs/2022-06-04/data_10_percent/effnetb2/5_epochs/\")\n",
    "    \"\"\"\n",
    "    from datetime import datetime\n",
    "    import os\n",
    "\n",
    "    # Get timestamp of current date (all experiments on certain day live in same folder)\n",
    "    timestamp = datetime.now().strftime(\"%Y-%m-%d\") # returns current date in YYYY-MM-DD format\n",
    "\n",
    "    if extra:\n",
    "        # Create log directory path\n",
    "        log_dir = os.path.join(\"runs\", timestamp, experiment_name, model_name, extra)\n",
    "    else:\n",
    "        log_dir = os.path.join(\"runs\", timestamp, experiment_name, model_name)\n",
    "        \n",
    "    print(f\"[INFO] Created SummaryWriter, saving to: {log_dir}...\")\n",
    "    return SummaryWriter(log_dir=log_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "S0BH4ONGsgNB",
    "outputId": "077f7b39-f4d0-44dd-b74f-8ead04fb7add"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Created SummaryWriter, saving to: runs\\2024-11-15\\test_experiment_name\\this_is_the_model_name\\add_a_little_extra_if_you_want...\n"
     ]
    }
   ],
   "source": [
    "# Create a test writer\n",
    "#from going_modular.sergio.engine import create_writer\n",
    "\n",
    "writer = engine.create_writer(experiment_name=\"test_experiment_name\",\n",
    "                       model_name=\"this_is_the_model_name\",\n",
    "                       extra=\"add_a_little_extra_if_you_want\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "VwO0Q1eFsusV"
   },
   "outputs": [],
   "source": [
    "from typing import Dict, List\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "#from going_modular.going_modular.engine import train_step, test_step\n",
    "\n",
    "# Add writer parameter to train()\n",
    "#def train(model: torch.nn.Module, \n",
    "#          train_dataloader: torch.utils.data.DataLoader, \n",
    "#          test_dataloader: torch.utils.data.DataLoader, \n",
    "#          optimizer: torch.optim.Optimizer,\n",
    "#          loss_fn: torch.nn.Module,\n",
    "#          epochs: int,\n",
    "#          device: torch.device, \n",
    "#          writer: torch.utils.tensorboard.writer.SummaryWriter # new parameter to take in a writer\n",
    "#          ) -> Dict[str, List]:\n",
    "#    \"\"\"Trains and tests a PyTorch model.\n",
    "#\n",
    "#    Passes a target PyTorch models through train_step() and test_step()\n",
    "#    functions for a number of epochs, training and testing the model\n",
    "#    in the same epoch loop.\n",
    "\n",
    "#    Calculates, prints and stores evaluation metrics throughout.\n",
    "\n",
    "#    Stores metrics to specified writer log_dir if present.\n",
    "\n",
    "#    Args:\n",
    "#      model: A PyTorch model to be trained and tested.\n",
    "#      train_dataloader: A DataLoader instance for the model to be trained on.\n",
    "#      test_dataloader: A DataLoader instance for the model to be tested on.\n",
    "#      optimizer: A PyTorch optimizer to help minimize the loss function.\n",
    "#      loss_fn: A PyTorch loss function to calculate loss on both datasets.\n",
    "#      epochs: An integer indicating how many epochs to train for.\n",
    "#      device: A target device to compute on (e.g. \"cuda\" or \"cpu\").\n",
    "#      writer: A SummaryWriter() instance to log model results to.\n",
    "\n",
    "#    Returns:\n",
    "#      A dictionary of training and testing loss as well as training and\n",
    "#      testing accuracy metrics. Each metric has a value in a list for \n",
    "#      each epoch.\n",
    "#      In the form: {train_loss: [...],\n",
    "#                train_acc: [...],\n",
    "#                test_loss: [...],\n",
    "#                test_acc: [...]} \n",
    "#      For example if training for epochs=2: \n",
    "#              {train_loss: [2.0616, 1.0537],\n",
    "#                train_acc: [0.3945, 0.3945],\n",
    "#                test_loss: [1.2641, 1.5706],\n",
    "#                test_acc: [0.3400, 0.2973]} \n",
    "#    \"\"\"\n",
    "    # Create empty results dictionary\n",
    "#    results = {\"train_loss\": [],\n",
    "#               \"train_acc\": [],\n",
    "#               \"test_loss\": [],\n",
    "#               \"test_acc\": []\n",
    "#    }\n",
    "\n",
    "    # Loop through training and testing steps for a number of epochs\n",
    "#    for epoch in tqdm(range(epochs)):\n",
    "#        train_loss, train_acc = train_step(model=model,\n",
    "#                                          dataloader=train_dataloader,\n",
    "#                                          loss_fn=loss_fn,\n",
    "#                                          optimizer=optimizer,\n",
    "#                                          device=device)\n",
    "#        test_loss, test_acc = test_step(model=model,\n",
    "#          dataloader=test_dataloader,\n",
    "#          loss_fn=loss_fn,\n",
    "#          device=device)\n",
    "\n",
    "        # Print out what's happening\n",
    "#        print(\n",
    "#          f\"Epoch: {epoch+1} | \"\n",
    "#          f\"train_loss: {train_loss:.4f} | \"\n",
    "#          f\"train_acc: {train_acc:.4f} | \"\n",
    "#          f\"test_loss: {test_loss:.4f} | \"\n",
    "#          f\"test_acc: {test_acc:.4f}\"\n",
    "#        )\n",
    "\n",
    "        # Update results dictionary\n",
    "#        results[\"train_loss\"].append(train_loss)\n",
    "#        results[\"train_acc\"].append(train_acc)\n",
    "#        results[\"test_loss\"].append(test_loss)\n",
    "#        results[\"test_acc\"].append(test_acc)\n",
    "\n",
    "\n",
    "        ### New: Use the writer parameter to track experiments ###\n",
    "        # See if there's a writer, if so, log to it\n",
    "#        if writer:\n",
    "#            # Add results to SummaryWriter\n",
    "#            writer.add_scalars(main_tag=\"Loss\", \n",
    "#                               tag_scalar_dict={\"train_loss\": train_loss,\n",
    "#                                                \"test_loss\": test_loss},\n",
    "#                               global_step=epoch)\n",
    "#            writer.add_scalars(main_tag=\"Accuracy\", \n",
    "#                               tag_scalar_dict={\"train_acc\": train_acc,\n",
    "#                                                \"test_acc\": test_acc}, \n",
    "#                               global_step=epoch)#\n",
    "\n",
    "            # Close the writer\n",
    "#            writer.close()\n",
    "#        else:\n",
    "#            pass\n",
    "    ### End new ###\n",
    "\n",
    "    # Return the filled results at the end of the epochs\n",
    "#    return results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nh8jKzHYHYL3"
   },
   "source": [
    "### Download data\n",
    "\n",
    "Using the same data from https://www.learnpytorch.io/07_pytorch_experiment_tracking/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "68QGCR_1tzif",
    "outputId": "a5073b19-1463-4d8a-ec08-5399945196a4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] data\\pizza_steak_sushi directory exists, skipping download.\n",
      "[INFO] data\\pizza_steak_sushi_20_percent directory exists, skipping download.\n"
     ]
    }
   ],
   "source": [
    "# Download 10 percent and 20 percent training data (if necessary)\n",
    "data_10_percent_path = helper_functions.download_data(source=\"https://github.com/mrdbourke/pytorch-deep-learning/raw/main/data/pizza_steak_sushi.zip\",\n",
    "                                     destination=\"pizza_steak_sushi\")\n",
    "\n",
    "data_20_percent_path = helper_functions.download_data(source=\"https://github.com/mrdbourke/pytorch-deep-learning/raw/main/data/pizza_steak_sushi_20_percent.zip\",\n",
    "                                     destination=\"pizza_steak_sushi_20_percent\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9L2rCRxvt1ED",
    "outputId": "30b8202e-96f3-443f-e89e-2a83e11b1c85"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training directory 10%: data\\pizza_steak_sushi\\train\n",
      "Training directory 20%: data\\pizza_steak_sushi_20_percent\\train\n",
      "Testing directory: data\\pizza_steak_sushi\\test\n"
     ]
    }
   ],
   "source": [
    "# Setup training directory paths\n",
    "train_dir_10_percent = data_10_percent_path / \"train\"\n",
    "train_dir_20_percent = data_20_percent_path / \"train\"\n",
    "\n",
    "# Setup testing directory paths (note: use the same test dataset for both to compare the results)\n",
    "test_dir = data_10_percent_path / \"test\"\n",
    "\n",
    "# Check the directories\n",
    "print(f\"Training directory 10%: {train_dir_10_percent}\")\n",
    "print(f\"Training directory 20%: {train_dir_20_percent}\")\n",
    "print(f\"Testing directory: {test_dir}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "K35q9wswt6NH"
   },
   "outputs": [],
   "source": [
    "from torchvision import transforms\n",
    "\n",
    "# Create a transform to normalize data distribution to be inline with ImageNet\n",
    "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], # values per colour channel [red, green, blue]\n",
    "                                 std=[0.229, 0.224, 0.225])\n",
    "\n",
    "# Create a transform pipeline\n",
    "simple_transform = transforms.Compose([\n",
    "                                       transforms.Resize((224, 224)),\n",
    "                                       transforms.ToTensor(), # get image values between 0 & 1\n",
    "                                       normalize\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision.transforms import v2\n",
    "\n",
    "normalize = v2.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "simple_transform = v2.Compose([\n",
    "    v2.Resize((224, 224)),\n",
    "    v2.ToImage(), v2.ToDtype(torch.float32, scale=True),\n",
    "    normalize\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SBuEla8pHea9"
   },
   "source": [
    "### Turn data into DataLoaders "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xlQU94HBuqOq",
    "outputId": "db4b9144-fd3b-4f31-e0c1-ebd60f906232"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of batches of size 32 in 10 percent training data: 8\n",
      "Number of batches of size 32 in 20 percent training data: 15\n",
      "Number of batches of size 32 in testing data: 8 (all experiments will use the same test set)\n",
      "Number of classes: 3, class names: ['pizza', 'steak', 'sushi']\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 32\n",
    "\n",
    "# Create 10% training and test DataLoaders\n",
    "train_dataloader_10_percent, test_dataloader, class_names = data_setup.create_dataloaders(train_dir=train_dir_10_percent,\n",
    "                                                                                          test_dir=test_dir,\n",
    "                                                                                          transform=simple_transform,\n",
    "                                                                                          batch_size=BATCH_SIZE)\n",
    "\n",
    "# Create 20% training and test DataLoaders\n",
    "train_dataloader_20_percent, test_dataloader, class_names = data_setup.create_dataloaders(train_dir=train_dir_20_percent,\n",
    "                                                                                          test_dir=test_dir,\n",
    "                                                                                          transform=simple_transform,\n",
    "                                                                                          batch_size=BATCH_SIZE)\n",
    "\n",
    "# Find the number of samples/batches per dataloader (using the same test_dataloader for both experiments)\n",
    "print(f\"Number of batches of size {BATCH_SIZE} in 10 percent training data: {len(train_dataloader_10_percent)}\")\n",
    "print(f\"Number of batches of size {BATCH_SIZE} in 20 percent training data: {len(train_dataloader_20_percent)}\")\n",
    "print(f\"Number of batches of size {BATCH_SIZE} in testing data: {len(train_dataloader_10_percent)} (all experiments will use the same test set)\")\n",
    "print(f\"Number of classes: {len(class_names)}, class names: {class_names}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nwmoMhW8IqSu"
   },
   "source": [
    "## Exercise 1: Pick a larger model from [`torchvision.models`](https://pytorch.org/vision/main/models.html) to add to the list of experiments (for example, EffNetB3 or higher)\n",
    "\n",
    "* How does it perform compared to our existing models?\n",
    "* **Hint:** You'll need to set up an exerpiment similar to [07. PyTorch Experiment Tracking section 7.6](https://www.learnpytorch.io/07_pytorch_experiment_tracking/#76-create-experiments-and-set-up-training-code)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EfficientNet(\n",
       "  (features): Sequential(\n",
       "    (0): Conv2dNormActivation(\n",
       "      (0): Conv2d(3, 40, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): SiLU(inplace=True)\n",
       "    )\n",
       "    (1): Sequential(\n",
       "      (0): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(40, 40, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=40, bias=False)\n",
       "            (1): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(40, 10, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(10, 40, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (2): Conv2dNormActivation(\n",
       "            (0): Conv2d(40, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.0, mode=row)\n",
       "      )\n",
       "      (1): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(24, 24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=24, bias=False)\n",
       "            (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(24, 6, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(6, 24, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (2): Conv2dNormActivation(\n",
       "            (0): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.007692307692307693, mode=row)\n",
       "      )\n",
       "    )\n",
       "    (2): Sequential(\n",
       "      (0): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(144, 144, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=144, bias=False)\n",
       "            (1): BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(144, 6, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(6, 144, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(144, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.015384615384615385, mode=row)\n",
       "      )\n",
       "      (1): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192, bias=False)\n",
       "            (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(192, 8, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(8, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.02307692307692308, mode=row)\n",
       "      )\n",
       "      (2): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192, bias=False)\n",
       "            (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(192, 8, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(8, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.03076923076923077, mode=row)\n",
       "      )\n",
       "    )\n",
       "    (3): Sequential(\n",
       "      (0): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(192, 192, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=192, bias=False)\n",
       "            (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(192, 8, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(8, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(192, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.038461538461538464, mode=row)\n",
       "      )\n",
       "      (1): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(48, 288, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(288, 288, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=288, bias=False)\n",
       "            (1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(288, 12, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(12, 288, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(288, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.04615384615384616, mode=row)\n",
       "      )\n",
       "      (2): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(48, 288, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(288, 288, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=288, bias=False)\n",
       "            (1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(288, 12, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(12, 288, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(288, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.05384615384615385, mode=row)\n",
       "      )\n",
       "    )\n",
       "    (4): Sequential(\n",
       "      (0): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(48, 288, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(288, 288, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=288, bias=False)\n",
       "            (1): BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(288, 12, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(12, 288, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(288, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.06153846153846154, mode=row)\n",
       "      )\n",
       "      (1): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(96, 576, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(576, 576, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=576, bias=False)\n",
       "            (1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(576, 24, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(24, 576, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(576, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.06923076923076923, mode=row)\n",
       "      )\n",
       "      (2): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(96, 576, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(576, 576, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=576, bias=False)\n",
       "            (1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(576, 24, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(24, 576, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(576, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.07692307692307693, mode=row)\n",
       "      )\n",
       "      (3): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(96, 576, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(576, 576, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=576, bias=False)\n",
       "            (1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(576, 24, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(24, 576, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(576, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.08461538461538462, mode=row)\n",
       "      )\n",
       "      (4): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(96, 576, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(576, 576, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=576, bias=False)\n",
       "            (1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(576, 24, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(24, 576, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(576, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.09230769230769233, mode=row)\n",
       "      )\n",
       "    )\n",
       "    (5): Sequential(\n",
       "      (0): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(96, 576, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(576, 576, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=576, bias=False)\n",
       "            (1): BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(576, 24, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(24, 576, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(576, 136, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(136, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.1, mode=row)\n",
       "      )\n",
       "      (1): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(136, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(816, 816, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=816, bias=False)\n",
       "            (1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(816, 34, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(34, 816, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(816, 136, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(136, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.1076923076923077, mode=row)\n",
       "      )\n",
       "      (2): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(136, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(816, 816, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=816, bias=False)\n",
       "            (1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(816, 34, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(34, 816, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(816, 136, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(136, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.11538461538461539, mode=row)\n",
       "      )\n",
       "      (3): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(136, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(816, 816, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=816, bias=False)\n",
       "            (1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(816, 34, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(34, 816, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(816, 136, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(136, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.12307692307692308, mode=row)\n",
       "      )\n",
       "      (4): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(136, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(816, 816, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=816, bias=False)\n",
       "            (1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(816, 34, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(34, 816, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(816, 136, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(136, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.13076923076923078, mode=row)\n",
       "      )\n",
       "    )\n",
       "    (6): Sequential(\n",
       "      (0): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(136, 816, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(816, 816, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=816, bias=False)\n",
       "            (1): BatchNorm2d(816, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(816, 34, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(34, 816, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(816, 232, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(232, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.13846153846153847, mode=row)\n",
       "      )\n",
       "      (1): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(232, 1392, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 1392, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1392, bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(1392, 58, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(58, 1392, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 232, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(232, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.14615384615384616, mode=row)\n",
       "      )\n",
       "      (2): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(232, 1392, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 1392, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1392, bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(1392, 58, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(58, 1392, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 232, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(232, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.15384615384615385, mode=row)\n",
       "      )\n",
       "      (3): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(232, 1392, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 1392, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1392, bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(1392, 58, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(58, 1392, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 232, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(232, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.16153846153846155, mode=row)\n",
       "      )\n",
       "      (4): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(232, 1392, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 1392, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1392, bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(1392, 58, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(58, 1392, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 232, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(232, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.16923076923076924, mode=row)\n",
       "      )\n",
       "      (5): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(232, 1392, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 1392, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1392, bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(1392, 58, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(58, 1392, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 232, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(232, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.17692307692307693, mode=row)\n",
       "      )\n",
       "    )\n",
       "    (7): Sequential(\n",
       "      (0): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(232, 1392, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 1392, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1392, bias=False)\n",
       "            (1): BatchNorm2d(1392, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(1392, 58, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(58, 1392, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(1392, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.18461538461538465, mode=row)\n",
       "      )\n",
       "      (1): MBConv(\n",
       "        (block): Sequential(\n",
       "          (0): Conv2dNormActivation(\n",
       "            (0): Conv2d(384, 2304, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(2304, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (1): Conv2dNormActivation(\n",
       "            (0): Conv2d(2304, 2304, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2304, bias=False)\n",
       "            (1): BatchNorm2d(2304, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): SiLU(inplace=True)\n",
       "          )\n",
       "          (2): SqueezeExcitation(\n",
       "            (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "            (fc1): Conv2d(2304, 96, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (fc2): Conv2d(96, 2304, kernel_size=(1, 1), stride=(1, 1))\n",
       "            (activation): SiLU(inplace=True)\n",
       "            (scale_activation): Sigmoid()\n",
       "          )\n",
       "          (3): Conv2dNormActivation(\n",
       "            (0): Conv2d(2304, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "          )\n",
       "        )\n",
       "        (stochastic_depth): StochasticDepth(p=0.19230769230769232, mode=row)\n",
       "      )\n",
       "    )\n",
       "    (8): Conv2dNormActivation(\n",
       "      (0): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (1): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (2): SiLU(inplace=True)\n",
       "    )\n",
       "  )\n",
       "  (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
       "  (classifier): Sequential(\n",
       "    (0): Dropout(p=0.3, inplace=True)\n",
       "    (1): Linear(in_features=1536, out_features=1000, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights = torchvision.models.EfficientNet_B3_Weights.DEFAULT\n",
    "model = torchvision.models.efficientnet_b3(weights=weights).to(device)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "F-35y0uxJ8tg"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Created new effnetb3 model.\n"
     ]
    }
   ],
   "source": [
    "import torchvision\n",
    "from torch import nn\n",
    "\n",
    "# Get num out features (one for each class pizza, steak, sushi)\n",
    "OUT_FEATURES = len(class_names)\n",
    "\n",
    "# Create an EffNetB0 feature extractor\n",
    "def create_effnetb0():\n",
    "    # 1. Get the base mdoel with pretrained weights and send to target device\n",
    "    weights = torchvision.models.EfficientNet_B0_Weights.DEFAULT\n",
    "    model = torchvision.models.efficientnet_b0(weights=weights).to(device)\n",
    "\n",
    "    # 2. Freeze the base model layers\n",
    "    for param in model.features.parameters():\n",
    "        param.requires_grad = False\n",
    "\n",
    "    # 3. Set the seeds\n",
    "    set_seeds()\n",
    "\n",
    "    # 4. Change the classifier head\n",
    "    model.classifier = nn.Sequential(\n",
    "        nn.Dropout(p=0.2),\n",
    "        nn.Linear(in_features=1280, out_features=OUT_FEATURES)\n",
    "    ).to(device)\n",
    "\n",
    "    # 5. Give the model a name\n",
    "    model.name = \"effnetb0\"\n",
    "    print(f\"[INFO] Created new {model.name} model.\")\n",
    "    return model\n",
    "\n",
    "# Create an EffNetB2 feature extractor\n",
    "def create_effnetb2():\n",
    "    # 1. Get the base model with pretrained weights and send to target device\n",
    "    weights = torchvision.models.EfficientNet_B2_Weights.DEFAULT\n",
    "    model = torchvision.models.efficientnet_b2(weights=weights).to(device)\n",
    "\n",
    "    # 2. Freeze the base model layers\n",
    "    for param in model.features.parameters():\n",
    "        param.requires_grad = False\n",
    "\n",
    "    # 3. Set the seeds\n",
    "    set_seeds()\n",
    "\n",
    "    # 4. Change the classifier head\n",
    "    model.classifier = nn.Sequential(\n",
    "        nn.Dropout(p=0.3),\n",
    "        nn.Linear(in_features=1408, out_features=OUT_FEATURES)\n",
    "    ).to(device)\n",
    "\n",
    "    # 5. Give the model a name\n",
    "    model.name = \"effnetb2\"\n",
    "    print(f\"[INFO] Created new {model.name} model.\")\n",
    "    return model\n",
    "\n",
    "\n",
    "# Create an EffNetB3 feature extractor\n",
    "def create_effnetb3():\n",
    "    # 1. Get the base model with pretrained weights and send to target device\n",
    "    weights = torchvision.models.EfficientNet_B3_Weights.DEFAULT\n",
    "    model = torchvision.models.efficientnet_b3(weights=weights).to(device)\n",
    "\n",
    "    # 2. Freeze the base model layers\n",
    "    for param in model.features.parameters():\n",
    "        param.requires_grad = False\n",
    "\n",
    "    # 3. Set the seeds\n",
    "    set_seeds()\n",
    "\n",
    "    # 4. Change the classifier head\n",
    "    model.classifier = nn.Sequential(\n",
    "        nn.Dropout(p=0.3),\n",
    "        nn.Linear(in_features=1536, out_features=OUT_FEATURES)\n",
    "    ).to(device)\n",
    "\n",
    "    # 5. Give the model a name\n",
    "    model.name = \"effnetb3\"\n",
    "    print(f\"[INFO] Created new {model.name} model.\")\n",
    "    return model\n",
    "\n",
    "effnetb3 = create_effnetb3() \n",
    "\n",
    "# Get an output summary of the layers in our EffNetB0 feature extractor model (uncomment to view full output)\n",
    "#summary(model=effnetb3, \n",
    "#        input_size=(32, 3, 224, 224), # make sure this is \"input_size\", not \"input_shape\"\n",
    "#        # col_names=[\"input_size\"], # uncomment for smaller output\n",
    "#        col_names=[\"input_size\", \"output_size\", \"num_params\", \"trainable\"],\n",
    "#        col_width=20,\n",
    "#        row_settings=[\"var_names\"]\n",
    "#)\n",
    "\n",
    "def create_effnetv2s():\n",
    "    # 1. Get the base model with pretrained weights and send to target device\n",
    "    weights = torchvision.models.EfficientNet_V2_S_Weights.DEFAULT\n",
    "    model = torchvision.models.efficientnet_v2_s(weights=weights).to(device)\n",
    "\n",
    "    # 2. Freeze the base model layers\n",
    "    for param in model.features.parameters():\n",
    "        param.requires_grad = False\n",
    "\n",
    "    # 3. Set the seeds\n",
    "    set_seeds()\n",
    "\n",
    "    # 4. Change the classifier head\n",
    "    model.classifier = nn.Sequential(\n",
    "        nn.Dropout(p=0.2),\n",
    "        nn.Linear(in_features=1280, out_features=OUT_FEATURES)\n",
    "    ).to(device)\n",
    "\n",
    "    # 5. Give the model a name\n",
    "    model.name = \"effnet_v2_s\"\n",
    "    print(f\"[INFO] Created new {model.name} model.\")\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Create epochs list\n",
    "num_epochs = [10]\n",
    "\n",
    "# 2. Create models list (need to create a new model for each experiment)\n",
    "models = [\"effnetb0\", \"effnetb2\", \"effnetb3\"]\n",
    "\n",
    "# 3. Create dataloaders dictionary for various dataloaders\n",
    "train_dataloaders = {\"data_20_percent\": train_dataloader_20_percent}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Experiment number: 1\n",
      "[INFO] Model: effnetb0\n",
      "[INFO] DataLoader: data_20_percent\n",
      "[INFO] Number of epochs: 10\n",
      "[INFO] Created new effnetb0 model.\n",
      "[INFO] Created SummaryWriter, saving to: runs\\2024-11-15\\data_20_percent\\effnetb0\\10_epochs...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad6d13607b9e457c8d5119f10a014e73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 | train_loss: 0.9587 | train_acc: 0.6125 | test_loss: 0.6551 | test_acc: 0.8655\n",
      "Epoch: 2 | train_loss: 0.6894 | train_acc: 0.8521 | test_loss: 0.5800 | test_acc: 0.8873\n",
      "Epoch: 3 | train_loss: 0.5805 | train_acc: 0.8604 | test_loss: 0.4576 | test_acc: 0.9176\n",
      "Epoch: 4 | train_loss: 0.4937 | train_acc: 0.8646 | test_loss: 0.4454 | test_acc: 0.9176\n",
      "Epoch: 5 | train_loss: 0.4886 | train_acc: 0.8500 | test_loss: 0.3914 | test_acc: 0.9176\n",
      "Epoch: 6 | train_loss: 0.3708 | train_acc: 0.8833 | test_loss: 0.3565 | test_acc: 0.9072\n",
      "Epoch: 7 | train_loss: 0.3558 | train_acc: 0.9208 | test_loss: 0.3182 | test_acc: 0.9072\n",
      "Epoch: 8 | train_loss: 0.3739 | train_acc: 0.8938 | test_loss: 0.3346 | test_acc: 0.8977\n",
      "Epoch: 9 | train_loss: 0.2976 | train_acc: 0.9375 | test_loss: 0.3087 | test_acc: 0.9280\n",
      "Epoch: 10 | train_loss: 0.3625 | train_acc: 0.8479 | test_loss: 0.2771 | test_acc: 0.9072\n",
      "[INFO] Saving model to: models\\07_effnetb0_data_20_percent_10_epochs.pth\n",
      "--------------------------------------------------\n",
      "\n",
      "[INFO] Experiment number: 2\n",
      "[INFO] Model: effnetb2\n",
      "[INFO] DataLoader: data_20_percent\n",
      "[INFO] Number of epochs: 10\n",
      "[INFO] Created new effnetb2 model.\n",
      "[INFO] Created SummaryWriter, saving to: runs\\2024-11-15\\data_20_percent\\effnetb2\\10_epochs...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1865707d1a9843e88d9cc494d876d41c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 | train_loss: 0.9819 | train_acc: 0.5604 | test_loss: 0.7775 | test_acc: 0.8049\n",
      "Epoch: 2 | train_loss: 0.7297 | train_acc: 0.8021 | test_loss: 0.6664 | test_acc: 0.8873\n",
      "Epoch: 3 | train_loss: 0.6011 | train_acc: 0.8479 | test_loss: 0.5633 | test_acc: 0.9280\n",
      "Epoch: 4 | train_loss: 0.5429 | train_acc: 0.8354 | test_loss: 0.5685 | test_acc: 0.8977\n",
      "Epoch: 5 | train_loss: 0.4410 | train_acc: 0.8708 | test_loss: 0.4469 | test_acc: 0.9280\n",
      "Epoch: 6 | train_loss: 0.3876 | train_acc: 0.9146 | test_loss: 0.4564 | test_acc: 0.8977\n",
      "Epoch: 7 | train_loss: 0.3480 | train_acc: 0.9292 | test_loss: 0.4226 | test_acc: 0.9384\n",
      "Epoch: 8 | train_loss: 0.3861 | train_acc: 0.8792 | test_loss: 0.4350 | test_acc: 0.9280\n",
      "Epoch: 9 | train_loss: 0.3315 | train_acc: 0.8979 | test_loss: 0.4253 | test_acc: 0.9081\n",
      "Epoch: 10 | train_loss: 0.3382 | train_acc: 0.8979 | test_loss: 0.3908 | test_acc: 0.9384\n",
      "[INFO] Saving model to: models\\07_effnetb2_data_20_percent_10_epochs.pth\n",
      "--------------------------------------------------\n",
      "\n",
      "[INFO] Experiment number: 3\n",
      "[INFO] Model: effnetv2s\n",
      "[INFO] DataLoader: data_20_percent\n",
      "[INFO] Number of epochs: 10\n",
      "[INFO] Created new effnetb3 model.\n",
      "[INFO] Created SummaryWriter, saving to: runs\\2024-11-15\\data_20_percent\\effnetv2s\\10_epochs...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21e29f7dcb984ae785729ef65547b534",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 | train_loss: 0.9828 | train_acc: 0.5729 | test_loss: 0.8723 | test_acc: 0.8144\n",
      "Epoch: 2 | train_loss: 0.7654 | train_acc: 0.7792 | test_loss: 0.6766 | test_acc: 0.8248\n",
      "Epoch: 3 | train_loss: 0.5752 | train_acc: 0.8875 | test_loss: 0.5858 | test_acc: 0.8759\n",
      "Epoch: 4 | train_loss: 0.5342 | train_acc: 0.8417 | test_loss: 0.5413 | test_acc: 0.8551\n",
      "Epoch: 5 | train_loss: 0.4505 | train_acc: 0.8854 | test_loss: 0.4895 | test_acc: 0.8248\n",
      "Epoch: 6 | train_loss: 0.4345 | train_acc: 0.8875 | test_loss: 0.4969 | test_acc: 0.8049\n",
      "Epoch: 7 | train_loss: 0.3849 | train_acc: 0.9042 | test_loss: 0.4520 | test_acc: 0.8248\n",
      "Epoch: 8 | train_loss: 0.3965 | train_acc: 0.8854 | test_loss: 0.4641 | test_acc: 0.7945\n",
      "Epoch: 9 | train_loss: 0.3518 | train_acc: 0.9021 | test_loss: 0.4131 | test_acc: 0.8759\n",
      "Epoch: 10 | train_loss: 0.3376 | train_acc: 0.9229 | test_loss: 0.4390 | test_acc: 0.8049\n",
      "[INFO] Saving model to: models\\07_effnetv2s_data_20_percent_10_epochs.pth\n",
      "--------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#from going_modular.going_modular.utils import save_model\n",
    "from going_modular.sergio.helper_functions import save_model\n",
    "\n",
    "# 1. Set the random seeds\n",
    "set_seeds(seed=42)\n",
    "\n",
    "# 2. Keep track of experiment numbers\n",
    "experiment_number = 0\n",
    "\n",
    "# 3. Loop through each DataLoader\n",
    "for dataloader_name, train_dataloader in train_dataloaders.items():\n",
    "\n",
    "    # 4. Loop through each number of epochs\n",
    "    for epochs in num_epochs: \n",
    "\n",
    "        # 5. Loop through each model name and create a new model based on the name\n",
    "        for model_name in models:\n",
    "\n",
    "            # 6. Create information print outs\n",
    "            experiment_number += 1\n",
    "            print(f\"[INFO] Experiment number: {experiment_number}\")\n",
    "            print(f\"[INFO] Model: {model_name}\")\n",
    "            print(f\"[INFO] DataLoader: {dataloader_name}\")\n",
    "            print(f\"[INFO] Number of epochs: {epochs}\")  \n",
    "\n",
    "            # 7. Select the model\n",
    "            if model_name == \"effnetb0\":\n",
    "                model = create_effnetb0() # creates a new model each time (important because we want each experiment to start from scratch)\n",
    "            else:\n",
    "                if model_name == \"effnetb2\":\n",
    "                    model = create_effnetb2() # creates a new model each time (important because we want each experiment to start from scratch)\n",
    "                else:\n",
    "                    model = create_effnetb3() # creates a new model each time (important because we want each experiment to start from scratch\n",
    "            \n",
    "            # 8. Create a new loss and optimizer for every model\n",
    "            loss_fn = nn.CrossEntropyLoss()\n",
    "            optimizer = torch.optim.Adam(params=model.parameters(), lr=0.001)\n",
    "\n",
    "            # 9. Train target model with target dataloaders and track experiments\n",
    "            engine.train(model=model,\n",
    "                        train_dataloader=train_dataloader,\n",
    "                        test_dataloader=test_dataloader, \n",
    "                        optimizer=optimizer,\n",
    "                        loss_fn=loss_fn,\n",
    "                        epochs=epochs,\n",
    "                        device=device,\n",
    "                        writer=create_writer(experiment_name=dataloader_name,\n",
    "                                            model_name=model_name,\n",
    "                                            extra=f\"{epochs}_epochs\"))\n",
    "            \n",
    "            # 10. Save the model to file so we can get back the best model\n",
    "            save_filepath = f\"07_{model_name}_{dataloader_name}_{epochs}_epochs.pth\"\n",
    "            save_model(model=model,\n",
    "                       target_dir=\"models\",\n",
    "                       model_name=save_filepath)\n",
    "            print(\"-\"*50 + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Create epochs list\n",
    "num_epochs = [10]\n",
    "\n",
    "# 2. Create models list (need to create a new model for each experiment)\n",
    "models = [\"effnetv2s\"]\n",
    "\n",
    "# 3. Create dataloaders dictionary for various dataloaders\n",
    "train_dataloaders = {\"data_20_percent\": train_dataloader_20_percent}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Experiment number: 1\n",
      "[INFO] Model: effnetv2s\n",
      "[INFO] DataLoader: data_20_percent\n",
      "[INFO] Number of epochs: 10\n",
      "[INFO] Created new effnet_v2_s model.\n",
      "[INFO] Created SummaryWriter, saving to: runs\\2024-11-15\\data_20_percent\\effnetv2s\\10_epochs...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bb4ebc40dde49708304e3f3ab8cc0f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 | train_loss: 0.9843 | train_acc: 0.5750 | test_loss: 0.7440 | test_acc: 0.8665\n",
      "Epoch: 2 | train_loss: 0.7295 | train_acc: 0.7792 | test_loss: 0.6413 | test_acc: 0.8258\n",
      "Epoch: 3 | train_loss: 0.6193 | train_acc: 0.8167 | test_loss: 0.5199 | test_acc: 0.8759\n",
      "Epoch: 4 | train_loss: 0.5584 | train_acc: 0.8229 | test_loss: 0.4902 | test_acc: 0.8561\n",
      "Epoch: 5 | train_loss: 0.5464 | train_acc: 0.8000 | test_loss: 0.4352 | test_acc: 0.8769\n",
      "Epoch: 6 | train_loss: 0.4449 | train_acc: 0.8542 | test_loss: 0.4155 | test_acc: 0.9072\n",
      "Epoch: 7 | train_loss: 0.4324 | train_acc: 0.8458 | test_loss: 0.3856 | test_acc: 0.8769\n",
      "Epoch: 8 | train_loss: 0.4077 | train_acc: 0.8667 | test_loss: 0.4068 | test_acc: 0.8769\n",
      "Epoch: 9 | train_loss: 0.3640 | train_acc: 0.9000 | test_loss: 0.4022 | test_acc: 0.8873\n",
      "Epoch: 10 | train_loss: 0.3907 | train_acc: 0.8667 | test_loss: 0.3456 | test_acc: 0.9176\n",
      "[INFO] Saving model to: models\\07_effnetv2s_data_20_percent_10_epochs.pth\n",
      "--------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 1. Set the random seeds\n",
    "set_seeds(seed=42)\n",
    "\n",
    "# 2. Keep track of experiment numbers\n",
    "experiment_number = 0\n",
    "\n",
    "# 3. Loop through each DataLoader\n",
    "for dataloader_name, train_dataloader in train_dataloaders.items():\n",
    "\n",
    "    # 4. Loop through each number of epochs\n",
    "    for epochs in num_epochs: \n",
    "\n",
    "        # 5. Loop through each model name and create a new model based on the name\n",
    "        for model_name in models:\n",
    "\n",
    "            # 6. Create information print outs\n",
    "            experiment_number += 1\n",
    "            print(f\"[INFO] Experiment number: {experiment_number}\")\n",
    "            print(f\"[INFO] Model: {model_name}\")\n",
    "            print(f\"[INFO] DataLoader: {dataloader_name}\")\n",
    "            print(f\"[INFO] Number of epochs: {epochs}\")  \n",
    "\n",
    "            # 7. Select the model\n",
    "            if model_name == \"effnetb0\":\n",
    "                model = create_effnetb0() # creates a new model each time (important because we want each experiment to start from scratch)\n",
    "            else:\n",
    "                if model_name == \"effnetb2\":\n",
    "                    model = create_effnetb2() # creates a new model each time (important because we want each experiment to start from scratch)\n",
    "                else:\n",
    "                    if model_name == \"effnetb3\":\n",
    "                        model = create_effnetb3() # creates a new model each time (important because we want each experiment to start from scratch\n",
    "                    else:\n",
    "                        model = create_effnetv2s() # creates a new model each time (important because we want each experiment to start from scratch\n",
    "            \n",
    "            \n",
    "            # 8. Create a new loss and optimizer for every model\n",
    "            loss_fn = nn.CrossEntropyLoss()\n",
    "            optimizer = torch.optim.Adam(params=model.parameters(), lr=0.001)\n",
    "\n",
    "            # 9. Train target model with target dataloaders and track experiments\n",
    "            engine.train(model=model,\n",
    "                        train_dataloader=train_dataloader,\n",
    "                        test_dataloader=test_dataloader, \n",
    "                        optimizer=optimizer,\n",
    "                        loss_fn=loss_fn,\n",
    "                        epochs=epochs,\n",
    "                        device=device,\n",
    "                        writer=create_writer(experiment_name=dataloader_name,\n",
    "                                            model_name=model_name,\n",
    "                                            extra=f\"{epochs}_epochs\"))\n",
    "            \n",
    "            # 10. Save the model to file so we can get back the best model\n",
    "            save_filepath = f\"07_{model_name}_{dataloader_name}_{epochs}_epochs.pth\"\n",
    "            save_model(model=model,\n",
    "                       target_dir=\"models\",\n",
    "                       model_name=save_filepath)\n",
    "            print(\"-\"*50 + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext tensorboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems that EffnetB3 overfits: better results on training but worse on test compared to B0 and B2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YqlStPo-gbrF"
   },
   "source": [
    "## Exercise 2. Introduce data augmentation to the list of experiments using the 20% pizza, steak, sushi training and test datasets, does this change anything?\n",
    "    \n",
    "* For example, you could have one training DataLoader that uses data augmentation (e.g. `train_dataloader_20_percent_aug` and `train_dataloader_20_percent_no_aug`) and then compare the results of two of the same model types training on these two DataLoaders.\n",
    "* **Note:** You may need to alter the `create_dataloaders()` function to be able to take a transform for the training data and the testing data (because you don't need to perform data augmentation on the test data). See [04. PyTorch Custom Datasets section 6](https://www.learnpytorch.io/04_pytorch_custom_datasets/#6-other-forms-of-transforms-data-augmentation) for examples of using data augmentation or the script below for an example:\n",
    "\n",
    "```python\n",
    "# Note: Data augmentation transform like this should only be performed on training data\n",
    "train_transform_data_aug = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.TrivialAugmentWide(),\n",
    "    transforms.ToTensor(),\n",
    "    normalize\n",
    "])\n",
    "\n",
    "# Create a helper function to visualize different augmented (and not augmented) images\n",
    "def view_dataloader_images(dataloader, n=10):\n",
    "    if n > 10:\n",
    "        print(f\"Having n higher than 10 will create messy plots, lowering to 10.\")\n",
    "        n = 10\n",
    "    imgs, labels = next(iter(dataloader))\n",
    "    plt.figure(figsize=(16, 8))\n",
    "    for i in range(n):\n",
    "        # Min max scale the image for display purposes\n",
    "        targ_image = imgs[i]\n",
    "        sample_min, sample_max = targ_image.min(), targ_image.max()\n",
    "        sample_scaled = (targ_image - sample_min)/(sample_max - sample_min)\n",
    "\n",
    "        # Plot images with appropriate axes information\n",
    "        plt.subplot(1, 10, i+1)\n",
    "        plt.imshow(sample_scaled.permute(1, 2, 0)) # resize for Matplotlib requirements\n",
    "        plt.title(class_names[labels[i]])\n",
    "        plt.axis(False)\n",
    "\n",
    "# Have to update `create_dataloaders()` to handle different augmentations\n",
    "import os\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "\n",
    "NUM_WORKERS = os.cpu_count() # use maximum number of CPUs for workers to load data \n",
    "\n",
    "# Note: this is an update version of data_setup.create_dataloaders to handle\n",
    "# differnt train and test transforms.\n",
    "def create_dataloaders(\n",
    "    train_dir, \n",
    "    test_dir, \n",
    "    train_transform, # add parameter for train transform (transforms on train dataset)\n",
    "    test_transform,  # add parameter for test transform (transforms on test dataset)\n",
    "    batch_size=32, num_workers=NUM_WORKERS\n",
    "):\n",
    "    # Use ImageFolder to create dataset(s)\n",
    "    train_data = datasets.ImageFolder(train_dir, transform=train_transform)\n",
    "    test_data = datasets.ImageFolder(test_dir, transform=test_transform)\n",
    "\n",
    "    # Get class names\n",
    "    class_names = train_data.classes\n",
    "\n",
    "    # Turn images into data loaders\n",
    "    train_dataloader = DataLoader(\n",
    "        train_data,\n",
    "        batch_size=batch_size,\n",
    "        shuffle=True,\n",
    "        num_workers=num_workers,\n",
    "        pin_memory=True,\n",
    "    )\n",
    "    test_dataloader = DataLoader(\n",
    "        test_data,\n",
    "        batch_size=batch_size,\n",
    "        shuffle=True,\n",
    "        num_workers=num_workers,\n",
    "        pin_memory=True,\n",
    "    )\n",
    "\n",
    "    return train_dataloader, test_dataloader, class_names\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "E1N3yyDOoH2t"
   },
   "outputs": [],
   "source": [
    "# Data augmentation\n",
    "normalize = v2.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "transform_data_aug = v2.Compose([\n",
    "    v2.Resize((224, 224)),\n",
    "    v2.TrivialAugmentWide(),\n",
    "    v2.ToImage(), v2.ToDtype(torch.float32, scale=True),\n",
    "    normalize\n",
    "])\n",
    "\n",
    "# Create a helper function to visualize different augmented (and not augmented) images\n",
    "def view_dataloader_images(dataloader, n=10):\n",
    "    if n > 10:\n",
    "        print(f\"Having n higher than 10 will create messy plots, lowering to 10.\")\n",
    "        n = 10\n",
    "    imgs, labels = next(iter(dataloader))\n",
    "    plt.figure(figsize=(16, 8))\n",
    "    for i in range(n):\n",
    "        # Min max scale the image for display purposes\n",
    "        targ_image = imgs[i]\n",
    "        sample_min, sample_max = targ_image.min(), targ_image.max()\n",
    "        sample_scaled = (targ_image - sample_min)/(sample_max - sample_min)\n",
    "\n",
    "        # Plot images with appropriate axes information\n",
    "        plt.subplot(1, 10, i+1)\n",
    "        plt.imshow(sample_scaled.permute(1, 2, 0)) # resize for Matplotlib requirements\n",
    "        plt.title(class_names[labels[i]])\n",
    "        plt.axis(False)\n",
    "\n",
    "# Have to update `create_dataloaders()` to handle different augmentations\n",
    "import os\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "\n",
    "NUM_WORKERS = os.cpu_count() # use maximum number of CPUs for workers to load data \n",
    "\n",
    "# Note: this is an update version of data_setup.create_dataloaders to handle\n",
    "# differnt train and test transforms.\n",
    "def create_dataloaders(\n",
    "    train_dir, \n",
    "    test_dir, \n",
    "    train_transform, # add parameter for train transform (transforms on train dataset)\n",
    "    test_transform,  # add parameter for test transform (transforms on test dataset)\n",
    "    batch_size=32, num_workers=NUM_WORKERS\n",
    "):\n",
    "    # Use ImageFolder to create dataset(s)\n",
    "    train_data = datasets.ImageFolder(train_dir, transform=train_transform)\n",
    "    test_data = datasets.ImageFolder(test_dir, transform=test_transform)\n",
    "\n",
    "    # Get class names\n",
    "    class_names = train_data.classes\n",
    "\n",
    "    # Turn images into data loaders\n",
    "    train_dataloader = DataLoader(\n",
    "        train_data,\n",
    "        batch_size=batch_size,\n",
    "        shuffle=True,\n",
    "        num_workers=num_workers,\n",
    "        pin_memory=True,\n",
    "    )\n",
    "    test_dataloader = DataLoader(\n",
    "        test_data,\n",
    "        batch_size=batch_size,\n",
    "        shuffle=True,\n",
    "        num_workers=num_workers,\n",
    "        pin_memory=True,\n",
    "    )\n",
    "\n",
    "    return train_dataloader, test_dataloader, class_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of batches of size 32 in 20 percent training data aug: 15\n",
      "Number of batches of size 32 in testing data: 15 (all experiments will use the same test set)\n",
      "Number of classes: 3, class names: ['pizza', 'steak', 'sushi']\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 32\n",
    "\n",
    "# Create 20% training and test DataLoaders with augmentation\n",
    "train_dataloader_20_percent_aug, test_dataloader, class_names = create_dataloaders(train_dir=train_dir_20_percent,\n",
    "                                                                                    test_dir=test_dir,\n",
    "                                                                                    train_transform=transform_data_aug,\n",
    "                                                                                    test_transform=simple_transform,\n",
    "                                                                                    batch_size=BATCH_SIZE)\n",
    "\n",
    "# Find the number of samples/batches per dataloader (using the same test_dataloader for both experiments)\n",
    "print(f\"Number of batches of size {BATCH_SIZE} in 20 percent training data aug: {len(train_dataloader_20_percent_aug)}\")\n",
    "print(f\"Number of batches of size {BATCH_SIZE} in testing data: {len(train_dataloader_20_percent)} (all experiments will use the same test set)\")\n",
    "print(f\"Number of classes: {len(class_names)}, class names: {class_names}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Create epochs list\n",
    "num_epochs = [10]\n",
    "\n",
    "# 2. Create models list (need to create a new model for each experiment)\n",
    "models = [\"effnetb2\"]\n",
    "\n",
    "# 3. Create dataloaders dictionary for various dataloaders\n",
    "train_dataloaders = {\"data_20_percent_aug\": train_dataloader_20_percent_aug}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Experiment number: 1\n",
      "[INFO] Model: effnetb2\n",
      "[INFO] DataLoader: data_20_percent_aug\n",
      "[INFO] Number of epochs: 10\n",
      "[INFO] Created new effnetb2 model.\n",
      "[INFO] Created SummaryWriter, saving to: runs\\2024-11-15\\data_20_percent_aug\\effnetb2\\10_epochs...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61349426d1674c6ab53d5861d536887e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 1. Set the random seeds\n",
    "set_seeds(seed=42)\n",
    "\n",
    "# 2. Keep track of experiment numbers\n",
    "experiment_number = 0\n",
    "\n",
    "# 3. Loop through each DataLoader\n",
    "for dataloader_name, train_dataloader in train_dataloaders.items():\n",
    "\n",
    "    # 4. Loop through each number of epochs\n",
    "    for epochs in num_epochs: \n",
    "\n",
    "        # 5. Loop through each model name and create a new model based on the name\n",
    "        for model_name in models:\n",
    "\n",
    "            # 6. Create information print outs\n",
    "            experiment_number += 1\n",
    "            print(f\"[INFO] Experiment number: {experiment_number}\")\n",
    "            print(f\"[INFO] Model: {model_name}\")\n",
    "            print(f\"[INFO] DataLoader: {dataloader_name}\")\n",
    "            print(f\"[INFO] Number of epochs: {epochs}\")  \n",
    "\n",
    "            # 7. Select the model\n",
    "            if model_name == \"effnetb0\":\n",
    "                model = create_effnetb0() # creates a new model each time (important because we want each experiment to start from scratch)\n",
    "            else:\n",
    "                if model_name == \"effnetb2\":\n",
    "                    model = create_effnetb2() # creates a new model each time (important because we want each experiment to start from scratch)\n",
    "                else:\n",
    "                    model = create_effnetb3() # creates a new model each time (important because we want each experiment to start from scratch\n",
    "            \n",
    "            # 8. Create a new loss and optimizer for every model\n",
    "            loss_fn = nn.CrossEntropyLoss()\n",
    "            optimizer = torch.optim.Adam(params=model.parameters(), lr=0.001)\n",
    "\n",
    "            # 9. Train target model with target dataloaders and track experiments\n",
    "            engine.train(model=model,\n",
    "                        train_dataloader=train_dataloader,\n",
    "                        test_dataloader=test_dataloader, \n",
    "                        optimizer=optimizer,\n",
    "                        loss_fn=loss_fn,\n",
    "                        epochs=epochs,\n",
    "                        device=device,\n",
    "                        writer=create_writer(experiment_name=dataloader_name,\n",
    "                                            model_name=model_name,\n",
    "                                            extra=f\"{epochs}_epochs\"))\n",
    "            \n",
    "            # 10. Save the model to file so we can get back the best model\n",
    "            save_filepath = f\"07_{model_name}_{dataloader_name}_{epochs}_epochs.pth\"\n",
    "            save_model(model=model,\n",
    "                       target_dir=\"models\",\n",
    "                       model_name=save_filepath)\n",
    "            print(\"-\"*50 + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1IvuTskxgjaw"
   },
   "source": [
    "## Exercise 3. Scale up the dataset to turn FoodVision Mini into FoodVision Big using the entire [Food101 dataset from `torchvision.models`](https://pytorch.org/vision/stable/generated/torchvision.datasets.Food101.html#torchvision.datasets.Food101)\n",
    "    \n",
    "* You could take the best performing model from your various experiments or even the EffNetB2 feature extractor we created in this notebook and see how it goes fitting for 5 epochs on all of Food101.\n",
    "* If you try more than one model, it would be good to have the model's results tracked.\n",
    "* If you load the Food101 dataset from `torchvision.models`, you'll have to create PyTorch DataLoaders to use it in training.\n",
    "* **Note:** Due to the larger amount of data in Food101 compared to our pizza, steak, sushi dataset, this model will take longer to train."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "YehliYnYoP1x"
   },
   "outputs": [],
   "source": [
    "# Get Food101 Dataset\n",
    "\n",
    "# Create a transform to normalize data distribution to be inline with ImageNet\n",
    "normalize = v2.Normalize(mean=[0.485, 0.456, 0.406], # values per colour channel [red, green, blue]\n",
    "                         std=[0.229, 0.224, 0.225])\n",
    "\n",
    "# Create a transform pipeline\n",
    "simple_transform = v2.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    v2.ToImage(), v2.ToDtype(torch.float32, scale=True),\n",
    "    normalize\n",
    "])\n",
    "\n",
    "# Download and transform Food101 data (note: this may take ~5 minutes in Google Colab)\n",
    "train_data = torchvision.datasets.Food101(root=\"data\",\n",
    "                                          split=\"train\",\n",
    "                                          transform=simple_transform,\n",
    "                                          download=True)\n",
    "\n",
    "test_data = torchvision.datasets.Food101(root=\"data\",\n",
    "                                        split=\"test\",\n",
    "                                        transform=simple_transform,\n",
    "                                        download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "BATCH_SIZE = 32 # use a big batch size to get through all the images (100,000+ in Food101)\n",
    "\n",
    "train_dataloader_big = torch.utils.data.DataLoader(train_data,\n",
    "                                                   shuffle=True,\n",
    "                                                   batch_size=BATCH_SIZE,\n",
    "                                                   num_workers=os.cpu_count(),\n",
    "                                                   pin_memory=True) # avoid copies of the data into and out of memory, where possible (for speed ups)\n",
    "\n",
    "test_dataloader_big = torch.utils.data.DataLoader(test_data,\n",
    "                                                  shuffle=False,\n",
    "                                                  batch_size=BATCH_SIZE,\n",
    "                                                  num_workers=os.cpu_count(),\n",
    "                                                  pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75750, 25250)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check sample numbers\n",
    "len(train_data), len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Create epochs list\n",
    "num_epochs = [10]\n",
    "\n",
    "# 2. Create models list (need to create a new model for each experiment)\n",
    "models = [\"effnetv2s\"]\n",
    "\n",
    "# 3. Create dataloaders dictionary for various dataloaders\n",
    "train_dataloaders = {\"data_20_percent_big\": train_dataloader_big}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Experiment number: 1\n",
      "[INFO] Model: effnetv2s\n",
      "[INFO] DataLoader: data_20_percent_big\n",
      "[INFO] Number of epochs: 10\n",
      "[INFO] Created SummaryWriter, saving to: runs\\2024-11-15\\data_20_percent_big\\effnetv2s\\10_epochs...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0736e36e786a419c92c581dffbfd3b21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 | train_loss: 4.5378 | train_acc: 0.0343 | test_loss: 4.6126 | test_acc: 0.0104\n",
      "Epoch: 2 | train_loss: 4.3578 | train_acc: 0.1144 | test_loss: 4.5911 | test_acc: 0.0000\n",
      "Epoch: 3 | train_loss: 4.2007 | train_acc: 0.1916 | test_loss: 4.5712 | test_acc: 0.0000\n",
      "Epoch: 4 | train_loss: 4.0563 | train_acc: 0.2377 | test_loss: 4.5547 | test_acc: 0.0104\n",
      "Epoch: 5 | train_loss: 3.9232 | train_acc: 0.2660 | test_loss: 4.5318 | test_acc: 0.0208\n",
      "Epoch: 6 | train_loss: 3.8024 | train_acc: 0.2856 | test_loss: 4.5725 | test_acc: 0.0208\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[17], line 45\u001b[0m\n\u001b[0;32m     42\u001b[0m optimizer \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39moptim\u001b[38;5;241m.\u001b[39mAdam(params\u001b[38;5;241m=\u001b[39mmodel\u001b[38;5;241m.\u001b[39mparameters(), lr\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.00001\u001b[39m)\n\u001b[0;32m     44\u001b[0m \u001b[38;5;66;03m# 9. Train target model with target dataloaders and track experiments\u001b[39;00m\n\u001b[1;32m---> 45\u001b[0m engine\u001b[38;5;241m.\u001b[39mtrain(model\u001b[38;5;241m=\u001b[39mmodel,\n\u001b[0;32m     46\u001b[0m             train_dataloader\u001b[38;5;241m=\u001b[39mtrain_dataloader,\n\u001b[0;32m     47\u001b[0m             test_dataloader\u001b[38;5;241m=\u001b[39mtest_dataloader, \n\u001b[0;32m     48\u001b[0m             optimizer\u001b[38;5;241m=\u001b[39moptimizer,\n\u001b[0;32m     49\u001b[0m             loss_fn\u001b[38;5;241m=\u001b[39mloss_fn,\n\u001b[0;32m     50\u001b[0m             epochs\u001b[38;5;241m=\u001b[39mepochs,\n\u001b[0;32m     51\u001b[0m             device\u001b[38;5;241m=\u001b[39mdevice,\n\u001b[0;32m     52\u001b[0m             writer\u001b[38;5;241m=\u001b[39mcreate_writer(experiment_name\u001b[38;5;241m=\u001b[39mdataloader_name,\n\u001b[0;32m     53\u001b[0m                                 model_name\u001b[38;5;241m=\u001b[39mmodel_name,\n\u001b[0;32m     54\u001b[0m                                 extra\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mepochs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m_epochs\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n\u001b[0;32m     56\u001b[0m \u001b[38;5;66;03m# 10. Save the model to file so we can get back the best model\u001b[39;00m\n\u001b[0;32m     57\u001b[0m save_filepath \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m07_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdataloader_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mepochs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m_epochs_big.pth\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\Projects\\pytorch-deep-learning\\extras\\exercises\\going_modular\\sergio\\engine.py:191\u001b[0m, in \u001b[0;36mtrain\u001b[1;34m(model, train_dataloader, test_dataloader, optimizer, loss_fn, epochs, device, writer)\u001b[0m\n\u001b[0;32m    189\u001b[0m \u001b[38;5;66;03m# Loop through training and testing steps for a number of epochs\u001b[39;00m\n\u001b[0;32m    190\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28mrange\u001b[39m(epochs)):\n\u001b[1;32m--> 191\u001b[0m     train_loss, train_acc \u001b[38;5;241m=\u001b[39m train_step(model\u001b[38;5;241m=\u001b[39mmodel,\n\u001b[0;32m    192\u001b[0m                                       dataloader\u001b[38;5;241m=\u001b[39mtrain_dataloader,\n\u001b[0;32m    193\u001b[0m                                       loss_fn\u001b[38;5;241m=\u001b[39mloss_fn,\n\u001b[0;32m    194\u001b[0m                                       optimizer\u001b[38;5;241m=\u001b[39moptimizer,\n\u001b[0;32m    195\u001b[0m                                       device\u001b[38;5;241m=\u001b[39mdevice)\n\u001b[0;32m    196\u001b[0m     test_loss, test_acc \u001b[38;5;241m=\u001b[39m test_step(model\u001b[38;5;241m=\u001b[39mmodel,\n\u001b[0;32m    197\u001b[0m       dataloader\u001b[38;5;241m=\u001b[39mtest_dataloader,\n\u001b[0;32m    198\u001b[0m       loss_fn\u001b[38;5;241m=\u001b[39mloss_fn,\n\u001b[0;32m    199\u001b[0m       device\u001b[38;5;241m=\u001b[39mdevice)\n\u001b[0;32m    201\u001b[0m     \u001b[38;5;66;03m# Print out what's happening\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\Projects\\pytorch-deep-learning\\extras\\exercises\\going_modular\\sergio\\engine.py:64\u001b[0m, in \u001b[0;36mtrain_step\u001b[1;34m(model, dataloader, loss_fn, optimizer, device)\u001b[0m\n\u001b[0;32m     61\u001b[0m X, y \u001b[38;5;241m=\u001b[39m X\u001b[38;5;241m.\u001b[39mto(device), y\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m     63\u001b[0m \u001b[38;5;66;03m# 1. Forward pass\u001b[39;00m\n\u001b[1;32m---> 64\u001b[0m y_pred \u001b[38;5;241m=\u001b[39m model(X)\n\u001b[0;32m     66\u001b[0m \u001b[38;5;66;03m# 2. Calculate  and accumulate loss\u001b[39;00m\n\u001b[0;32m     67\u001b[0m loss \u001b[38;5;241m=\u001b[39m loss_fn(y_pred, y)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torchvision\\models\\efficientnet.py:343\u001b[0m, in \u001b[0;36mEfficientNet.forward\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m    342\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 343\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_impl(x)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torchvision\\models\\efficientnet.py:333\u001b[0m, in \u001b[0;36mEfficientNet._forward_impl\u001b[1;34m(self, x)\u001b[0m\n\u001b[0;32m    332\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_forward_impl\u001b[39m(\u001b[38;5;28mself\u001b[39m, x: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 333\u001b[0m     x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfeatures(x)\n\u001b[0;32m    335\u001b[0m     x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mavgpool(x)\n\u001b[0;32m    336\u001b[0m     x \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mflatten(x, \u001b[38;5;241m1\u001b[39m)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\container.py:250\u001b[0m, in \u001b[0;36mSequential.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    248\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m):\n\u001b[0;32m    249\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[1;32m--> 250\u001b[0m         \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m module(\u001b[38;5;28minput\u001b[39m)\n\u001b[0;32m    251\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\container.py:250\u001b[0m, in \u001b[0;36mSequential.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    248\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m):\n\u001b[0;32m    249\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[1;32m--> 250\u001b[0m         \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m module(\u001b[38;5;28minput\u001b[39m)\n\u001b[0;32m    251\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torchvision\\models\\efficientnet.py:164\u001b[0m, in \u001b[0;36mMBConv.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    163\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 164\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mblock(\u001b[38;5;28minput\u001b[39m)\n\u001b[0;32m    165\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_res_connect:\n\u001b[0;32m    166\u001b[0m         result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstochastic_depth(result)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\container.py:250\u001b[0m, in \u001b[0;36mSequential.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    248\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m):\n\u001b[0;32m    249\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[1;32m--> 250\u001b[0m         \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m module(\u001b[38;5;28minput\u001b[39m)\n\u001b[0;32m    251\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torchvision\\ops\\misc.py:259\u001b[0m, in \u001b[0;36mSqueezeExcitation.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    258\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 259\u001b[0m     scale \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_scale(\u001b[38;5;28minput\u001b[39m)\n\u001b[0;32m    260\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m scale \u001b[38;5;241m*\u001b[39m \u001b[38;5;28minput\u001b[39m\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torchvision\\ops\\misc.py:255\u001b[0m, in \u001b[0;36mSqueezeExcitation._scale\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    253\u001b[0m scale \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc1(scale)\n\u001b[0;32m    254\u001b[0m scale \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mactivation(scale)\n\u001b[1;32m--> 255\u001b[0m scale \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc2(scale)\n\u001b[0;32m    256\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscale_activation(scale)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\conv.py:554\u001b[0m, in \u001b[0;36mConv2d.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    553\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 554\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_conv_forward(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mweight, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbias)\n",
      "File \u001b[1;32mc:\\Users\\ssre_\\.conda\\envs\\.venv_pytorch_gpu\\Lib\\site-packages\\torch\\nn\\modules\\conv.py:549\u001b[0m, in \u001b[0;36mConv2d._conv_forward\u001b[1;34m(self, input, weight, bias)\u001b[0m\n\u001b[0;32m    537\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mzeros\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    538\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mconv2d(\n\u001b[0;32m    539\u001b[0m         F\u001b[38;5;241m.\u001b[39mpad(\n\u001b[0;32m    540\u001b[0m             \u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reversed_padding_repeated_twice, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding_mode\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    547\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups,\n\u001b[0;32m    548\u001b[0m     )\n\u001b[1;32m--> 549\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mconv2d(\n\u001b[0;32m    550\u001b[0m     \u001b[38;5;28minput\u001b[39m, weight, bias, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstride, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdilation, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgroups\n\u001b[0;32m    551\u001b[0m )\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# 1. Set the random seeds\n",
    "set_seeds(seed=42)\n",
    "\n",
    "# 2. Keep track of experiment numbers\n",
    "experiment_number = 0\n",
    "\n",
    "# 3. Loop through each DataLoader\n",
    "for dataloader_name, train_dataloader in train_dataloaders.items():\n",
    "\n",
    "    # 4. Loop through each number of epochs\n",
    "    for epochs in num_epochs: \n",
    "\n",
    "        # 5. Loop through each model name and create a new model based on the name\n",
    "        for model_name in models:\n",
    "\n",
    "            # 6. Create information print outs\n",
    "            experiment_number += 1\n",
    "            print(f\"[INFO] Experiment number: {experiment_number}\")\n",
    "            print(f\"[INFO] Model: {model_name}\")\n",
    "            print(f\"[INFO] DataLoader: {dataloader_name}\")\n",
    "            print(f\"[INFO] Number of epochs: {epochs}\")  \n",
    "\n",
    "            # 7. Create the model      \n",
    "            #weights = torchvision.models.EfficientNet_B2_Weights.DEFAULT\n",
    "            #model = torchvision.models.efficientnet_b2(weights=weights).to(device)            \n",
    "            weights = torchvision.models.EfficientNet_V2_S_Weights.DEFAULT\n",
    "            model = torchvision.models.efficientnet_v2_s(weights=weights).to(device)            \n",
    "            for param in model.features.parameters():\n",
    "                param.requires_grad = False\n",
    "            set_seeds()          \n",
    "            #model.classifier = nn.Sequential(\n",
    "            #    nn.Dropout(p=0.3),\n",
    "            #    nn.Linear(in_features=1408, out_features=101)\n",
    "            #).to(device)\n",
    "            model.classifier = nn.Sequential(\n",
    "                nn.Dropout(p=0.2),\n",
    "                nn.Linear(in_features=1280, out_features=101)\n",
    "            ).to(device)\n",
    "                        \n",
    "            # 8. Create a new loss and optimizer for every model\n",
    "            loss_fn = nn.CrossEntropyLoss()\n",
    "            optimizer = torch.optim.Adam(params=model.parameters(), lr=0.00001)\n",
    "\n",
    "            # 9. Train target model with target dataloaders and track experiments\n",
    "            engine.train(model=model,\n",
    "                        train_dataloader=train_dataloader,\n",
    "                        test_dataloader=test_dataloader, \n",
    "                        optimizer=optimizer,\n",
    "                        loss_fn=loss_fn,\n",
    "                        epochs=epochs,\n",
    "                        device=device,\n",
    "                        writer=create_writer(experiment_name=dataloader_name,\n",
    "                                            model_name=model_name,\n",
    "                                            extra=f\"{epochs}_epochs\"))\n",
    "            \n",
    "            # 10. Save the model to file so we can get back the best model\n",
    "            save_filepath = f\"07_{model_name}_{dataloader_name}_{epochs}_epochs_big.pth\"\n",
    "            save_model(model=model,\n",
    "                       target_dir=\"models\",\n",
    "                       model_name=save_filepath)\n",
    "            print(\"-\"*50 + \"\\n\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyP4+Nwb43yrG43qNz11d5C4",
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "07_pytorch_experiment_tracking_exercise_template.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": ".venv_pytorch_gpu",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}