Deep Learning Models: A collection of various deep learning architectures, models, & their implementations (PyTorch or TensorFlow) in Jupyter Notebooks.
- Author: Chizoba Obasi
- Github Repo: https://github.com/chizkidd/deep-learning-models
InĀ [1]:
!pip install watermark
Collecting watermark Downloading watermark-2.5.1-py3-none-any.whl.metadata (1.5 kB) Requirement already satisfied: ipython>=6.0 in /usr/local/lib/python3.11/dist-packages (from watermark) (7.34.0) Requirement already satisfied: importlib-metadata>=1.4 in /usr/local/lib/python3.11/dist-packages (from watermark) (8.7.0) Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from watermark) (75.2.0) Requirement already satisfied: zipp>=3.20 in /usr/local/lib/python3.11/dist-packages (from importlib-metadata>=1.4->watermark) (3.23.0) Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.11/dist-packages (from ipython>=6.0->watermark) (0.19.2) Requirement already satisfied: decorator in /usr/local/lib/python3.11/dist-packages (from ipython>=6.0->watermark) (4.4.2) Requirement already satisfied: pickleshare in /usr/local/lib/python3.11/dist-packages (from ipython>=6.0->watermark) (0.7.5) Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.11/dist-packages (from ipython>=6.0->watermark) (5.7.1) Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from ipython>=6.0->watermark) (3.0.51) Requirement already satisfied: pygments in /usr/local/lib/python3.11/dist-packages (from ipython>=6.0->watermark) (2.19.2) Requirement already satisfied: backcall in /usr/local/lib/python3.11/dist-packages (from ipython>=6.0->watermark) (0.2.0) Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.11/dist-packages (from ipython>=6.0->watermark) (0.1.7) Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.11/dist-packages (from ipython>=6.0->watermark) (4.9.0) Requirement already satisfied: parso<0.9.0,>=0.8.4 in /usr/local/lib/python3.11/dist-packages (from jedi>=0.16->ipython>=6.0->watermark) (0.8.4) Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.11/dist-packages (from pexpect>4.3->ipython>=6.0->watermark) (0.7.0) Requirement already satisfied: wcwidth in /usr/local/lib/python3.11/dist-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=6.0->watermark) (0.2.13) Downloading watermark-2.5.1-py3-none-any.whl (7.8 kB) Installing collected packages: watermark Successfully installed watermark-2.5.1
InĀ [2]:
!pip install torchinfo
Requirement already satisfied: torchinfo in /usr/local/lib/python3.11/dist-packages (1.8.0)
InĀ [3]:
%load_ext watermark
# %reload_ext watermark
%watermark -a 'Chizoba Obasi' -v -p torch
Author: Chizoba Obasi Python implementation: CPython Python version : 3.11.13 IPython version : 7.34.0 torch: 2.6.0+cu124
GoogLeNet Classifier on CIFAR-10Ā¶
Network Architecture:
REFERENCES:
- Szegedy, Christian, et al. "Going Deeper with Convolutions." Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2015. arXiv:1409.4842.
- Johnson, J. (2022, February 2). Lecture 8: CNN Architectures [PowerPoint Slides]. University of Michigan EECS 498/598: Deep Learning for Computer Vision. pp 67-76. https://web.eecs.umich.edu/~justincj/slides/eecs498/WI2022/598_WI2022_lecture08.pdf
- Kim, S. (n.d.). GoogLeNet-Inception [Source code]. GitHub. https://github.com/conan7882/GoogLeNet-Inception
InĀ [4]:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
from sklearn.model_selection import train_test_split
from torchinfo import summary
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
%matplotlib inline
import pandas as pd
import numpy as np
import math
import time
import os
import random
if torch.cuda.is_available():
torch.backends.cudnn.deterministic = True
InĀ [5]:
def set_all_seeds(seed):
os.environ["PL_GLOBAL_SEED"] = str(seed)
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
InĀ [6]:
##########################
### SETTINGS
##########################
# Hyperparameters
RANDOM_SEED = 123
BATCH_SIZE = 128
LEARNING_RATE = 0.01
NUM_EPOCHS = 20
INPUT_SIZE = 32 # CIFAR-10 is 32x32
NUM_CLASSES = 10
# Set device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(device)
cuda
InĀ [7]:
# Mean and Std Dev for CIFAR-10 (3 channels)
CIFAR_MEAN = (0.4914, 0.4822, 0.4465)
CIFAR_STD = (0.2471, 0.2435, 0.2616)
# CIFAR-10 preprocessing
train_transforms = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(CIFAR_MEAN, CIFAR_STD)
])
test_transforms = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(CIFAR_MEAN, CIFAR_STD)
])
# Load Datasets
train_dataset = datasets.CIFAR10(root='./data', train=True, download=True, transform=train_transforms)
test_dataset = datasets.CIFAR10(root='./data', train=False, download=True, transform=test_transforms)
# Split training set into training and validation sets
train_size = int(0.9 * len(train_dataset))
val_size = len(train_dataset) - train_size
train_subset, val_subset = torch.utils.data.random_split(train_dataset, [train_size, val_size])
# Create DataLoaders
train_loader = DataLoader(train_subset, batch_size=BATCH_SIZE, shuffle=True)
val_loader = DataLoader(val_subset, batch_size=BATCH_SIZE, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)
100%|āāāāāāāāāā| 170M/170M [00:01<00:00, 95.5MB/s]
InĀ [8]:
class Inception(nn.Module):
def __init__(self, in_planes, n1x1, n3x3red, n3x3, n5x5red, n5x5, pool_planes):
super(Inception, self).__init__()
# 1x1 conv branch
self.b1 = nn.Sequential(
nn.Conv2d(in_planes, n1x1, kernel_size=1),
nn.BatchNorm2d(n1x1),
nn.ReLU(True),
)
# 1x1 conv -> 3x3 conv branch
self.b2 = nn.Sequential(
nn.Conv2d(in_planes, n3x3red, kernel_size=1),
nn.BatchNorm2d(n3x3red),
nn.ReLU(True),
nn.Conv2d(n3x3red, n3x3, kernel_size=3, padding=1),
nn.BatchNorm2d(n3x3),
nn.ReLU(True),
)
# 1x1 conv -> 5x5 conv branch
self.b3 = nn.Sequential(
nn.Conv2d(in_planes, n5x5red, kernel_size=1),
nn.BatchNorm2d(n5x5red),
nn.ReLU(True),
nn.Conv2d(n5x5red, n5x5, kernel_size=5, padding=2), # Note: padding=2 keeps size same
nn.BatchNorm2d(n5x5),
nn.ReLU(True),
)
# self.b3 = nn.Sequential(
# nn.Conv2d(in_planes, n5x5red, kernel_size=1),
# nn.BatchNorm2d(n5x5red),
# nn.ReLU(True),
# nn.Conv2d(n5x5red, n5x5, kernel_size=3, padding=1), # Paper uses 5x5, but two 3x3 is common
# nn.BatchNorm2d(n5x5),
# nn.ReLU(True),
# nn.Conv2d(n5x5, n5x5, kernel_size=3, padding=1),
# nn.BatchNorm2d(n5x5),
# nn.ReLU(True),
# )
# 3x3 pool -> 1x1 conv branch
self.b4 = nn.Sequential(
nn.MaxPool2d(3, stride=1, padding=1),
nn.Conv2d(in_planes, pool_planes, kernel_size=1),
nn.BatchNorm2d(pool_planes),
nn.ReLU(True),
)
def forward(self, x):
y1 = self.b1(x)
y2 = self.b2(x)
y3 = self.b3(x)
y4 = self.b4(x)
return torch.cat([y1, y2, y3, y4], 1)
class GoogLeNet(nn.Module):
def __init__(self, num_classes=10):
super(GoogLeNet, self).__init__()
self.pre_layers = nn.Sequential(
nn.Conv2d(3, 192, kernel_size=3, padding=1),
nn.BatchNorm2d(192),
nn.ReLU(True),
)
self.a3 = Inception(192, 64, 96, 128, 16, 32, 32)
self.b3 = Inception(256, 128, 128, 192, 32, 96, 64)
self.maxpool = nn.MaxPool2d(3, stride=2, padding=1)
self.a4 = Inception(480, 192, 96, 208, 16, 48, 64)
self.b4 = Inception(512, 160, 112, 224, 24, 64, 64)
self.c4 = Inception(512, 128, 128, 256, 24, 64, 64)
self.d4 = Inception(512, 112, 144, 288, 32, 64, 64)
self.e4 = Inception(528, 256, 160, 320, 32, 128, 128)
self.a5 = Inception(832, 256, 160, 320, 32, 128, 128)
self.b5 = Inception(832, 384, 192, 384, 48, 128, 128)
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
self.dropout = nn.Dropout(0.4)
self.linear = nn.Linear(1024, num_classes)
def forward(self, x):
out = self.pre_layers(x)
out = self.a3(out)
out = self.b3(out)
out = self.maxpool(out)
out = self.a4(out)
out = self.b4(out)
out = self.c4(out)
out = self.d4(out)
out = self.e4(out)
out = self.maxpool(out)
out = self.a5(out)
out = self.b5(out)
out = self.avgpool(out)
out = out.view(out.size(0), -1)
out = self.dropout(out)
out = self.linear(out)
return out
InĀ [9]:
set_all_seeds(RANDOM_SEED)
model = GoogLeNet(num_classes=NUM_CLASSES).to(device)
# Model parameter details
summary(model,
input_size=(BATCH_SIZE, 3, INPUT_SIZE, INPUT_SIZE), # verbose=1,
col_names=["input_size", "output_size", "num_params", "kernel_size", "mult_adds"]
)
Out[9]:
===================================================================================================================================================================== Layer (type:depth-idx) Input Shape Output Shape Param # Kernel Shape Mult-Adds ===================================================================================================================================================================== GoogLeNet [128, 3, 32, 32] [128, 10] -- -- -- āāSequential: 1-1 [128, 3, 32, 32] [128, 192, 32, 32] -- -- -- ā āāConv2d: 2-1 [128, 3, 32, 32] [128, 192, 32, 32] 5,376 [3, 3] 704,643,072 ā āāBatchNorm2d: 2-2 [128, 192, 32, 32] [128, 192, 32, 32] 384 -- 49,152 ā āāReLU: 2-3 [128, 192, 32, 32] [128, 192, 32, 32] -- -- -- āāInception: 1-2 [128, 192, 32, 32] [128, 256, 32, 32] -- -- -- ā āāSequential: 2-4 [128, 192, 32, 32] [128, 64, 32, 32] -- -- -- ā ā āāConv2d: 3-1 [128, 192, 32, 32] [128, 64, 32, 32] 12,352 [1, 1] 1,619,001,344 ā ā āāBatchNorm2d: 3-2 [128, 64, 32, 32] [128, 64, 32, 32] 128 -- 16,384 ā ā āāReLU: 3-3 [128, 64, 32, 32] [128, 64, 32, 32] -- -- -- ā āāSequential: 2-5 [128, 192, 32, 32] [128, 128, 32, 32] -- -- -- ā ā āāConv2d: 3-4 [128, 192, 32, 32] [128, 96, 32, 32] 18,528 [1, 1] 2,428,502,016 ā ā āāBatchNorm2d: 3-5 [128, 96, 32, 32] [128, 96, 32, 32] 192 -- 24,576 ā ā āāReLU: 3-6 [128, 96, 32, 32] [128, 96, 32, 32] -- -- -- ā ā āāConv2d: 3-7 [128, 96, 32, 32] [128, 128, 32, 32] 110,720 [3, 3] 14,512,291,840 ā ā āāBatchNorm2d: 3-8 [128, 128, 32, 32] [128, 128, 32, 32] 256 -- 32,768 ā ā āāReLU: 3-9 [128, 128, 32, 32] [128, 128, 32, 32] -- -- -- ā āāSequential: 2-6 [128, 192, 32, 32] [128, 32, 32, 32] -- -- -- ā ā āāConv2d: 3-10 [128, 192, 32, 32] [128, 16, 32, 32] 3,088 [1, 1] 404,750,336 ā ā āāBatchNorm2d: 3-11 [128, 16, 32, 32] [128, 16, 32, 32] 32 -- 4,096 ā ā āāReLU: 3-12 [128, 16, 32, 32] [128, 16, 32, 32] -- -- -- ā ā āāConv2d: 3-13 [128, 16, 32, 32] [128, 32, 32, 32] 12,832 [5, 5] 1,681,915,904 ā ā āāBatchNorm2d: 3-14 [128, 32, 32, 32] [128, 32, 32, 32] 64 -- 8,192 ā ā āāReLU: 3-15 [128, 32, 32, 32] [128, 32, 32, 32] -- -- -- ā āāSequential: 2-7 [128, 192, 32, 32] [128, 32, 32, 32] -- -- -- ā ā āāMaxPool2d: 3-16 [128, 192, 32, 32] [128, 192, 32, 32] -- 3 -- ā ā āāConv2d: 3-17 [128, 192, 32, 32] [128, 32, 32, 32] 6,176 [1, 1] 809,500,672 ā ā āāBatchNorm2d: 3-18 [128, 32, 32, 32] [128, 32, 32, 32] 64 -- 8,192 ā ā āāReLU: 3-19 [128, 32, 32, 32] [128, 32, 32, 32] -- -- -- āāInception: 1-3 [128, 256, 32, 32] [128, 480, 32, 32] -- -- -- ā āāSequential: 2-8 [128, 256, 32, 32] [128, 128, 32, 32] -- -- -- ā ā āāConv2d: 3-20 [128, 256, 32, 32] [128, 128, 32, 32] 32,896 [1, 1] 4,311,744,512 ā ā āāBatchNorm2d: 3-21 [128, 128, 32, 32] [128, 128, 32, 32] 256 -- 32,768 ā ā āāReLU: 3-22 [128, 128, 32, 32] [128, 128, 32, 32] -- -- -- ā āāSequential: 2-9 [128, 256, 32, 32] [128, 192, 32, 32] -- -- -- ā ā āāConv2d: 3-23 [128, 256, 32, 32] [128, 128, 32, 32] 32,896 [1, 1] 4,311,744,512 ā ā āāBatchNorm2d: 3-24 [128, 128, 32, 32] [128, 128, 32, 32] 256 -- 32,768 ā ā āāReLU: 3-25 [128, 128, 32, 32] [128, 128, 32, 32] -- -- -- ā ā āāConv2d: 3-26 [128, 128, 32, 32] [128, 192, 32, 32] 221,376 [3, 3] 29,016,195,072 ā ā āāBatchNorm2d: 3-27 [128, 192, 32, 32] [128, 192, 32, 32] 384 -- 49,152 ā ā āāReLU: 3-28 [128, 192, 32, 32] [128, 192, 32, 32] -- -- -- ā āāSequential: 2-10 [128, 256, 32, 32] [128, 96, 32, 32] -- -- -- ā ā āāConv2d: 3-29 [128, 256, 32, 32] [128, 32, 32, 32] 8,224 [1, 1] 1,077,936,128 ā ā āāBatchNorm2d: 3-30 [128, 32, 32, 32] [128, 32, 32, 32] 64 -- 8,192 ā ā āāReLU: 3-31 [128, 32, 32, 32] [128, 32, 32, 32] -- -- -- ā ā āāConv2d: 3-32 [128, 32, 32, 32] [128, 96, 32, 32] 76,896 [5, 5] 10,078,912,512 ā ā āāBatchNorm2d: 3-33 [128, 96, 32, 32] [128, 96, 32, 32] 192 -- 24,576 ā ā āāReLU: 3-34 [128, 96, 32, 32] [128, 96, 32, 32] -- -- -- ā āāSequential: 2-11 [128, 256, 32, 32] [128, 64, 32, 32] -- -- -- ā ā āāMaxPool2d: 3-35 [128, 256, 32, 32] [128, 256, 32, 32] -- 3 -- ā ā āāConv2d: 3-36 [128, 256, 32, 32] [128, 64, 32, 32] 16,448 [1, 1] 2,155,872,256 ā ā āāBatchNorm2d: 3-37 [128, 64, 32, 32] [128, 64, 32, 32] 128 -- 16,384 ā ā āāReLU: 3-38 [128, 64, 32, 32] [128, 64, 32, 32] -- -- -- āāMaxPool2d: 1-4 [128, 480, 32, 32] [128, 480, 16, 16] -- 3 -- āāInception: 1-5 [128, 480, 16, 16] [128, 512, 16, 16] -- -- -- ā āāSequential: 2-12 [128, 480, 16, 16] [128, 192, 16, 16] -- -- -- ā ā āāConv2d: 3-39 [128, 480, 16, 16] [128, 192, 16, 16] 92,352 [1, 1] 3,026,190,336 ā ā āāBatchNorm2d: 3-40 [128, 192, 16, 16] [128, 192, 16, 16] 384 -- 49,152 ā ā āāReLU: 3-41 [128, 192, 16, 16] [128, 192, 16, 16] -- -- -- ā āāSequential: 2-13 [128, 480, 16, 16] [128, 208, 16, 16] -- -- -- ā ā āāConv2d: 3-42 [128, 480, 16, 16] [128, 96, 16, 16] 46,176 [1, 1] 1,513,095,168 ā ā āāBatchNorm2d: 3-43 [128, 96, 16, 16] [128, 96, 16, 16] 192 -- 24,576 ā ā āāReLU: 3-44 [128, 96, 16, 16] [128, 96, 16, 16] -- -- -- ā ā āāConv2d: 3-45 [128, 96, 16, 16] [128, 208, 16, 16] 179,920 [3, 3] 5,895,618,560 ā ā āāBatchNorm2d: 3-46 [128, 208, 16, 16] [128, 208, 16, 16] 416 -- 53,248 ā ā āāReLU: 3-47 [128, 208, 16, 16] [128, 208, 16, 16] -- -- -- ā āāSequential: 2-14 [128, 480, 16, 16] [128, 48, 16, 16] -- -- -- ā ā āāConv2d: 3-48 [128, 480, 16, 16] [128, 16, 16, 16] 7,696 [1, 1] 252,182,528 ā ā āāBatchNorm2d: 3-49 [128, 16, 16, 16] [128, 16, 16, 16] 32 -- 4,096 ā ā āāReLU: 3-50 [128, 16, 16, 16] [128, 16, 16, 16] -- -- -- ā ā āāConv2d: 3-51 [128, 16, 16, 16] [128, 48, 16, 16] 19,248 [5, 5] 630,718,464 ā ā āāBatchNorm2d: 3-52 [128, 48, 16, 16] [128, 48, 16, 16] 96 -- 12,288 ā ā āāReLU: 3-53 [128, 48, 16, 16] [128, 48, 16, 16] -- -- -- ā āāSequential: 2-15 [128, 480, 16, 16] [128, 64, 16, 16] -- -- -- ā ā āāMaxPool2d: 3-54 [128, 480, 16, 16] [128, 480, 16, 16] -- 3 -- ā ā āāConv2d: 3-55 [128, 480, 16, 16] [128, 64, 16, 16] 30,784 [1, 1] 1,008,730,112 ā ā āāBatchNorm2d: 3-56 [128, 64, 16, 16] [128, 64, 16, 16] 128 -- 16,384 ā ā āāReLU: 3-57 [128, 64, 16, 16] [128, 64, 16, 16] -- -- -- āāInception: 1-6 [128, 512, 16, 16] [128, 512, 16, 16] -- -- -- ā āāSequential: 2-16 [128, 512, 16, 16] [128, 160, 16, 16] -- -- -- ā ā āāConv2d: 3-58 [128, 512, 16, 16] [128, 160, 16, 16] 82,080 [1, 1] 2,689,597,440 ā ā āāBatchNorm2d: 3-59 [128, 160, 16, 16] [128, 160, 16, 16] 320 -- 40,960 ā ā āāReLU: 3-60 [128, 160, 16, 16] [128, 160, 16, 16] -- -- -- ā āāSequential: 2-17 [128, 512, 16, 16] [128, 224, 16, 16] -- -- -- ā ā āāConv2d: 3-61 [128, 512, 16, 16] [128, 112, 16, 16] 57,456 [1, 1] 1,882,718,208 ā ā āāBatchNorm2d: 3-62 [128, 112, 16, 16] [128, 112, 16, 16] 224 -- 28,672 ā ā āāReLU: 3-63 [128, 112, 16, 16] [128, 112, 16, 16] -- -- -- ā ā āāConv2d: 3-64 [128, 112, 16, 16] [128, 224, 16, 16] 226,016 [3, 3] 7,406,092,288 ā ā āāBatchNorm2d: 3-65 [128, 224, 16, 16] [128, 224, 16, 16] 448 -- 57,344 ā ā āāReLU: 3-66 [128, 224, 16, 16] [128, 224, 16, 16] -- -- -- ā āāSequential: 2-18 [128, 512, 16, 16] [128, 64, 16, 16] -- -- -- ā ā āāConv2d: 3-67 [128, 512, 16, 16] [128, 24, 16, 16] 12,312 [1, 1] 403,439,616 ā ā āāBatchNorm2d: 3-68 [128, 24, 16, 16] [128, 24, 16, 16] 48 -- 6,144 ā ā āāReLU: 3-69 [128, 24, 16, 16] [128, 24, 16, 16] -- -- -- ā ā āāConv2d: 3-70 [128, 24, 16, 16] [128, 64, 16, 16] 38,464 [5, 5] 1,260,388,352 ā ā āāBatchNorm2d: 3-71 [128, 64, 16, 16] [128, 64, 16, 16] 128 -- 16,384 ā ā āāReLU: 3-72 [128, 64, 16, 16] [128, 64, 16, 16] -- -- -- ā āāSequential: 2-19 [128, 512, 16, 16] [128, 64, 16, 16] -- -- -- ā ā āāMaxPool2d: 3-73 [128, 512, 16, 16] [128, 512, 16, 16] -- 3 -- ā ā āāConv2d: 3-74 [128, 512, 16, 16] [128, 64, 16, 16] 32,832 [1, 1] 1,075,838,976 ā ā āāBatchNorm2d: 3-75 [128, 64, 16, 16] [128, 64, 16, 16] 128 -- 16,384 ā ā āāReLU: 3-76 [128, 64, 16, 16] [128, 64, 16, 16] -- -- -- āāInception: 1-7 [128, 512, 16, 16] [128, 512, 16, 16] -- -- -- ā āāSequential: 2-20 [128, 512, 16, 16] [128, 128, 16, 16] -- -- -- ā ā āāConv2d: 3-77 [128, 512, 16, 16] [128, 128, 16, 16] 65,664 [1, 1] 2,151,677,952 ā ā āāBatchNorm2d: 3-78 [128, 128, 16, 16] [128, 128, 16, 16] 256 -- 32,768 ā ā āāReLU: 3-79 [128, 128, 16, 16] [128, 128, 16, 16] -- -- -- ā āāSequential: 2-21 [128, 512, 16, 16] [128, 256, 16, 16] -- -- -- ā ā āāConv2d: 3-80 [128, 512, 16, 16] [128, 128, 16, 16] 65,664 [1, 1] 2,151,677,952 ā ā āāBatchNorm2d: 3-81 [128, 128, 16, 16] [128, 128, 16, 16] 256 -- 32,768 ā ā āāReLU: 3-82 [128, 128, 16, 16] [128, 128, 16, 16] -- -- -- ā ā āāConv2d: 3-83 [128, 128, 16, 16] [128, 256, 16, 16] 295,168 [3, 3] 9,672,065,024 ā ā āāBatchNorm2d: 3-84 [128, 256, 16, 16] [128, 256, 16, 16] 512 -- 65,536 ā ā āāReLU: 3-85 [128, 256, 16, 16] [128, 256, 16, 16] -- -- -- ā āāSequential: 2-22 [128, 512, 16, 16] [128, 64, 16, 16] -- -- -- ā ā āāConv2d: 3-86 [128, 512, 16, 16] [128, 24, 16, 16] 12,312 [1, 1] 403,439,616 ā ā āāBatchNorm2d: 3-87 [128, 24, 16, 16] [128, 24, 16, 16] 48 -- 6,144 ā ā āāReLU: 3-88 [128, 24, 16, 16] [128, 24, 16, 16] -- -- -- ā ā āāConv2d: 3-89 [128, 24, 16, 16] [128, 64, 16, 16] 38,464 [5, 5] 1,260,388,352 ā ā āāBatchNorm2d: 3-90 [128, 64, 16, 16] [128, 64, 16, 16] 128 -- 16,384 ā ā āāReLU: 3-91 [128, 64, 16, 16] [128, 64, 16, 16] -- -- -- ā āāSequential: 2-23 [128, 512, 16, 16] [128, 64, 16, 16] -- -- -- ā ā āāMaxPool2d: 3-92 [128, 512, 16, 16] [128, 512, 16, 16] -- 3 -- ā ā āāConv2d: 3-93 [128, 512, 16, 16] [128, 64, 16, 16] 32,832 [1, 1] 1,075,838,976 ā ā āāBatchNorm2d: 3-94 [128, 64, 16, 16] [128, 64, 16, 16] 128 -- 16,384 ā ā āāReLU: 3-95 [128, 64, 16, 16] [128, 64, 16, 16] -- -- -- āāInception: 1-8 [128, 512, 16, 16] [128, 528, 16, 16] -- -- -- ā āāSequential: 2-24 [128, 512, 16, 16] [128, 112, 16, 16] -- -- -- ā ā āāConv2d: 3-96 [128, 512, 16, 16] [128, 112, 16, 16] 57,456 [1, 1] 1,882,718,208 ā ā āāBatchNorm2d: 3-97 [128, 112, 16, 16] [128, 112, 16, 16] 224 -- 28,672 ā ā āāReLU: 3-98 [128, 112, 16, 16] [128, 112, 16, 16] -- -- -- ā āāSequential: 2-25 [128, 512, 16, 16] [128, 288, 16, 16] -- -- -- ā ā āāConv2d: 3-99 [128, 512, 16, 16] [128, 144, 16, 16] 73,872 [1, 1] 2,420,637,696 ā ā āāBatchNorm2d: 3-100 [128, 144, 16, 16] [128, 144, 16, 16] 288 -- 36,864 ā ā āāReLU: 3-101 [128, 144, 16, 16] [128, 144, 16, 16] -- -- -- ā ā āāConv2d: 3-102 [128, 144, 16, 16] [128, 288, 16, 16] 373,536 [3, 3] 12,240,027,648 ā ā āāBatchNorm2d: 3-103 [128, 288, 16, 16] [128, 288, 16, 16] 576 -- 73,728 ā ā āāReLU: 3-104 [128, 288, 16, 16] [128, 288, 16, 16] -- -- -- ā āāSequential: 2-26 [128, 512, 16, 16] [128, 64, 16, 16] -- -- -- ā ā āāConv2d: 3-105 [128, 512, 16, 16] [128, 32, 16, 16] 16,416 [1, 1] 537,919,488 ā ā āāBatchNorm2d: 3-106 [128, 32, 16, 16] [128, 32, 16, 16] 64 -- 8,192 ā ā āāReLU: 3-107 [128, 32, 16, 16] [128, 32, 16, 16] -- -- -- ā ā āāConv2d: 3-108 [128, 32, 16, 16] [128, 64, 16, 16] 51,264 [5, 5] 1,679,818,752 ā ā āāBatchNorm2d: 3-109 [128, 64, 16, 16] [128, 64, 16, 16] 128 -- 16,384 ā ā āāReLU: 3-110 [128, 64, 16, 16] [128, 64, 16, 16] -- -- -- ā āāSequential: 2-27 [128, 512, 16, 16] [128, 64, 16, 16] -- -- -- ā ā āāMaxPool2d: 3-111 [128, 512, 16, 16] [128, 512, 16, 16] -- 3 -- ā ā āāConv2d: 3-112 [128, 512, 16, 16] [128, 64, 16, 16] 32,832 [1, 1] 1,075,838,976 ā ā āāBatchNorm2d: 3-113 [128, 64, 16, 16] [128, 64, 16, 16] 128 -- 16,384 ā ā āāReLU: 3-114 [128, 64, 16, 16] [128, 64, 16, 16] -- -- -- āāInception: 1-9 [128, 528, 16, 16] [128, 832, 16, 16] -- -- -- ā āāSequential: 2-28 [128, 528, 16, 16] [128, 256, 16, 16] -- -- -- ā ā āāConv2d: 3-115 [128, 528, 16, 16] [128, 256, 16, 16] 135,424 [1, 1] 4,437,573,632 ā ā āāBatchNorm2d: 3-116 [128, 256, 16, 16] [128, 256, 16, 16] 512 -- 65,536 ā ā āāReLU: 3-117 [128, 256, 16, 16] [128, 256, 16, 16] -- -- -- ā āāSequential: 2-29 [128, 528, 16, 16] [128, 320, 16, 16] -- -- -- ā ā āāConv2d: 3-118 [128, 528, 16, 16] [128, 160, 16, 16] 84,640 [1, 1] 2,773,483,520 ā ā āāBatchNorm2d: 3-119 [128, 160, 16, 16] [128, 160, 16, 16] 320 -- 40,960 ā ā āāReLU: 3-120 [128, 160, 16, 16] [128, 160, 16, 16] -- -- -- ā ā āāConv2d: 3-121 [128, 160, 16, 16] [128, 320, 16, 16] 461,120 [3, 3] 15,109,980,160 ā ā āāBatchNorm2d: 3-122 [128, 320, 16, 16] [128, 320, 16, 16] 640 -- 81,920 ā ā āāReLU: 3-123 [128, 320, 16, 16] [128, 320, 16, 16] -- -- -- ā āāSequential: 2-30 [128, 528, 16, 16] [128, 128, 16, 16] -- -- -- ā ā āāConv2d: 3-124 [128, 528, 16, 16] [128, 32, 16, 16] 16,928 [1, 1] 554,696,704 ā ā āāBatchNorm2d: 3-125 [128, 32, 16, 16] [128, 32, 16, 16] 64 -- 8,192 ā ā āāReLU: 3-126 [128, 32, 16, 16] [128, 32, 16, 16] -- -- -- ā ā āāConv2d: 3-127 [128, 32, 16, 16] [128, 128, 16, 16] 102,528 [5, 5] 3,359,637,504 ā ā āāBatchNorm2d: 3-128 [128, 128, 16, 16] [128, 128, 16, 16] 256 -- 32,768 ā ā āāReLU: 3-129 [128, 128, 16, 16] [128, 128, 16, 16] -- -- -- ā āāSequential: 2-31 [128, 528, 16, 16] [128, 128, 16, 16] -- -- -- ā ā āāMaxPool2d: 3-130 [128, 528, 16, 16] [128, 528, 16, 16] -- 3 -- ā ā āāConv2d: 3-131 [128, 528, 16, 16] [128, 128, 16, 16] 67,712 [1, 1] 2,218,786,816 ā ā āāBatchNorm2d: 3-132 [128, 128, 16, 16] [128, 128, 16, 16] 256 -- 32,768 ā ā āāReLU: 3-133 [128, 128, 16, 16] [128, 128, 16, 16] -- -- -- āāMaxPool2d: 1-10 [128, 832, 16, 16] [128, 832, 8, 8] -- 3 -- āāInception: 1-11 [128, 832, 8, 8] [128, 832, 8, 8] -- -- -- ā āāSequential: 2-32 [128, 832, 8, 8] [128, 256, 8, 8] -- -- -- ā ā āāConv2d: 3-134 [128, 832, 8, 8] [128, 256, 8, 8] 213,248 [1, 1] 1,746,927,616 ā ā āāBatchNorm2d: 3-135 [128, 256, 8, 8] [128, 256, 8, 8] 512 -- 65,536 ā ā āāReLU: 3-136 [128, 256, 8, 8] [128, 256, 8, 8] -- -- -- ā āāSequential: 2-33 [128, 832, 8, 8] [128, 320, 8, 8] -- -- -- ā ā āāConv2d: 3-137 [128, 832, 8, 8] [128, 160, 8, 8] 133,280 [1, 1] 1,091,829,760 ā ā āāBatchNorm2d: 3-138 [128, 160, 8, 8] [128, 160, 8, 8] 320 -- 40,960 ā ā āāReLU: 3-139 [128, 160, 8, 8] [128, 160, 8, 8] -- -- -- ā ā āāConv2d: 3-140 [128, 160, 8, 8] [128, 320, 8, 8] 461,120 [3, 3] 3,777,495,040 ā ā āāBatchNorm2d: 3-141 [128, 320, 8, 8] [128, 320, 8, 8] 640 -- 81,920 ā ā āāReLU: 3-142 [128, 320, 8, 8] [128, 320, 8, 8] -- -- -- ā āāSequential: 2-34 [128, 832, 8, 8] [128, 128, 8, 8] -- -- -- ā ā āāConv2d: 3-143 [128, 832, 8, 8] [128, 32, 8, 8] 26,656 [1, 1] 218,365,952 ā ā āāBatchNorm2d: 3-144 [128, 32, 8, 8] [128, 32, 8, 8] 64 -- 8,192 ā ā āāReLU: 3-145 [128, 32, 8, 8] [128, 32, 8, 8] -- -- -- ā ā āāConv2d: 3-146 [128, 32, 8, 8] [128, 128, 8, 8] 102,528 [5, 5] 839,909,376 ā ā āāBatchNorm2d: 3-147 [128, 128, 8, 8] [128, 128, 8, 8] 256 -- 32,768 ā ā āāReLU: 3-148 [128, 128, 8, 8] [128, 128, 8, 8] -- -- -- ā āāSequential: 2-35 [128, 832, 8, 8] [128, 128, 8, 8] -- -- -- ā ā āāMaxPool2d: 3-149 [128, 832, 8, 8] [128, 832, 8, 8] -- 3 -- ā ā āāConv2d: 3-150 [128, 832, 8, 8] [128, 128, 8, 8] 106,624 [1, 1] 873,463,808 ā ā āāBatchNorm2d: 3-151 [128, 128, 8, 8] [128, 128, 8, 8] 256 -- 32,768 ā ā āāReLU: 3-152 [128, 128, 8, 8] [128, 128, 8, 8] -- -- -- āāInception: 1-12 [128, 832, 8, 8] [128, 1024, 8, 8] -- -- -- ā āāSequential: 2-36 [128, 832, 8, 8] [128, 384, 8, 8] -- -- -- ā ā āāConv2d: 3-153 [128, 832, 8, 8] [128, 384, 8, 8] 319,872 [1, 1] 2,620,391,424 ā ā āāBatchNorm2d: 3-154 [128, 384, 8, 8] [128, 384, 8, 8] 768 -- 98,304 ā ā āāReLU: 3-155 [128, 384, 8, 8] [128, 384, 8, 8] -- -- -- ā āāSequential: 2-37 [128, 832, 8, 8] [128, 384, 8, 8] -- -- -- ā ā āāConv2d: 3-156 [128, 832, 8, 8] [128, 192, 8, 8] 159,936 [1, 1] 1,310,195,712 ā ā āāBatchNorm2d: 3-157 [128, 192, 8, 8] [128, 192, 8, 8] 384 -- 49,152 ā ā āāReLU: 3-158 [128, 192, 8, 8] [128, 192, 8, 8] -- -- -- ā ā āāConv2d: 3-159 [128, 192, 8, 8] [128, 384, 8, 8] 663,936 [3, 3] 5,438,963,712 ā ā āāBatchNorm2d: 3-160 [128, 384, 8, 8] [128, 384, 8, 8] 768 -- 98,304 ā ā āāReLU: 3-161 [128, 384, 8, 8] [128, 384, 8, 8] -- -- -- ā āāSequential: 2-38 [128, 832, 8, 8] [128, 128, 8, 8] -- -- -- ā ā āāConv2d: 3-162 [128, 832, 8, 8] [128, 48, 8, 8] 39,984 [1, 1] 327,548,928 ā ā āāBatchNorm2d: 3-163 [128, 48, 8, 8] [128, 48, 8, 8] 96 -- 12,288 ā ā āāReLU: 3-164 [128, 48, 8, 8] [128, 48, 8, 8] -- -- -- ā ā āāConv2d: 3-165 [128, 48, 8, 8] [128, 128, 8, 8] 153,728 [5, 5] 1,259,339,776 ā ā āāBatchNorm2d: 3-166 [128, 128, 8, 8] [128, 128, 8, 8] 256 -- 32,768 ā ā āāReLU: 3-167 [128, 128, 8, 8] [128, 128, 8, 8] -- -- -- ā āāSequential: 2-39 [128, 832, 8, 8] [128, 128, 8, 8] -- -- -- ā ā āāMaxPool2d: 3-168 [128, 832, 8, 8] [128, 832, 8, 8] -- 3 -- ā ā āāConv2d: 3-169 [128, 832, 8, 8] [128, 128, 8, 8] 106,624 [1, 1] 873,463,808 ā ā āāBatchNorm2d: 3-170 [128, 128, 8, 8] [128, 128, 8, 8] 256 -- 32,768 ā ā āāReLU: 3-171 [128, 128, 8, 8] [128, 128, 8, 8] -- -- -- āāAdaptiveAvgPool2d: 1-13 [128, 1024, 8, 8] [128, 1024, 1, 1] -- -- -- āāDropout: 1-14 [128, 1024] [128, 1024] -- -- -- āāLinear: 1-15 [128, 1024] [128, 10] 10,250 -- 1,312,000 ===================================================================================================================================================================== Total params: 5,879,066 Trainable params: 5,879,066 Non-trainable params: 0 Total mult-adds (Units.GIGABYTES): 185.54 ===================================================================================================================================================================== Input size (MB): 1.57 Forward/backward pass size (MB): 4737.48 Params size (MB): 23.52 Estimated Total Size (MB): 4762.57 =====================================================================================================================================================================
InĀ [12]:
criterion = nn.CrossEntropyLoss()
# optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=5e-4)
optimizer = optim.SGD(model.parameters(), lr=LEARNING_RATE, momentum=0.9, weight_decay=5e-4)
# scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=5)
def calculate_accuracy(loader):
model.eval()
correct = 0
total = 0
with torch.no_grad():
for inputs, labels in loader:
inputs, labels = inputs.to(device), labels.to(device)
outputs = model(inputs)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
return 100 * correct / total
# Training Loop
start_time = time.time()
history = {'train_loss': [], 'val_loss': [], 'train_acc': [], 'val_acc': [], 'lrs': []}
print(f"Starting training on {device}...")
for epoch in range(NUM_EPOCHS):
model.train()
running_loss = 0.0
for i, (inputs, labels) in enumerate(train_loader):
inputs, labels = inputs.to(device), labels.to(device)
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
train_loss = running_loss / len(train_subset)
train_acc = calculate_accuracy(train_loader)
# Calculate validation loss and accuracy
model.eval()
val_loss = 0.0
with torch.no_grad():
for inputs, labels in val_loader:
inputs, labels = inputs.to(device), labels.to(device)
outputs = model(inputs)
loss = criterion(outputs, labels)
val_loss += loss.item() * inputs.size(0)
val_loss /= len(val_subset)
val_acc = calculate_accuracy(val_loader)
# scheduler.step(val_loss)
current_lr = optimizer.param_groups[0]['lr']
# Log metrics
history['train_loss'].append(train_loss)
history['val_loss'].append(val_loss)
history['train_acc'].append(train_acc)
history['val_acc'].append(val_acc)
# history['lrs'].append(current_lr)
print(f'Epoch {epoch + 1}/{NUM_EPOCHS} | '
f'Train Loss: {train_loss:.5f}, '
f'Train Acc.: {train_acc:.2f}% | '
f'Val Loss: {val_loss:.5f}, '
f'Val Acc.: {val_acc:.2f}% | '
f'LR: {current_lr:.8f}'
)
print('\nTotal Training Time: %.2f min' % ((time.time() - start_time)/60))
# --- Final Evaluation ---
test_accuracy = calculate_accuracy(test_loader)
print(f'\nTest Accuracy: {test_accuracy:.2f}%')
Starting training on cuda... Epoch 1/20 | Train Loss: 1.28648, Train Acc.: 66.08% | Val Loss: 1.02909, Val Acc.: 65.24% | LR: 0.01000000 Epoch 2/20 | Train Loss: 0.78526, Train Acc.: 75.45% | Val Loss: 0.78191, Val Acc.: 72.12% | LR: 0.01000000 Epoch 3/20 | Train Loss: 0.59302, Train Acc.: 79.38% | Val Loss: 0.66688, Val Acc.: 77.94% | LR: 0.01000000 Epoch 4/20 | Train Loss: 0.48664, Train Acc.: 81.00% | Val Loss: 0.64376, Val Acc.: 79.00% | LR: 0.01000000 Epoch 5/20 | Train Loss: 0.41288, Train Acc.: 86.28% | Val Loss: 0.50406, Val Acc.: 83.12% | LR: 0.01000000 Epoch 6/20 | Train Loss: 0.35927, Train Acc.: 82.10% | Val Loss: 0.63354, Val Acc.: 78.94% | LR: 0.01000000 Epoch 7/20 | Train Loss: 0.32183, Train Acc.: 88.21% | Val Loss: 0.51100, Val Acc.: 83.24% | LR: 0.01000000 Epoch 8/20 | Train Loss: 0.29152, Train Acc.: 87.65% | Val Loss: 0.53622, Val Acc.: 82.36% | LR: 0.01000000 Epoch 9/20 | Train Loss: 0.25946, Train Acc.: 90.48% | Val Loss: 0.45209, Val Acc.: 85.66% | LR: 0.01000000 Epoch 10/20 | Train Loss: 0.23363, Train Acc.: 92.46% | Val Loss: 0.40076, Val Acc.: 87.16% | LR: 0.01000000 Epoch 11/20 | Train Loss: 0.21171, Train Acc.: 92.09% | Val Loss: 0.43253, Val Acc.: 85.98% | LR: 0.01000000 Epoch 12/20 | Train Loss: 0.19625, Train Acc.: 91.58% | Val Loss: 0.44711, Val Acc.: 85.94% | LR: 0.01000000 Epoch 13/20 | Train Loss: 0.17764, Train Acc.: 92.14% | Val Loss: 0.44353, Val Acc.: 86.56% | LR: 0.01000000 Epoch 14/20 | Train Loss: 0.16321, Train Acc.: 86.31% | Val Loss: 0.69792, Val Acc.: 80.80% | LR: 0.01000000 Epoch 15/20 | Train Loss: 0.14844, Train Acc.: 91.49% | Val Loss: 0.49269, Val Acc.: 85.58% | LR: 0.01000000 Epoch 16/20 | Train Loss: 0.14438, Train Acc.: 92.24% | Val Loss: 0.48355, Val Acc.: 85.56% | LR: 0.01000000 Epoch 17/20 | Train Loss: 0.13062, Train Acc.: 94.55% | Val Loss: 0.41590, Val Acc.: 88.12% | LR: 0.01000000 Epoch 18/20 | Train Loss: 0.12394, Train Acc.: 91.76% | Val Loss: 0.53496, Val Acc.: 84.78% | LR: 0.01000000 Epoch 19/20 | Train Loss: 0.11172, Train Acc.: 93.68% | Val Loss: 0.44400, Val Acc.: 86.44% | LR: 0.01000000 Epoch 20/20 | Train Loss: 0.10753, Train Acc.: 93.83% | Val Loss: 0.48871, Val Acc.: 86.34% | LR: 0.01000000 Total Training Time: 79.54 min Test Accuracy: 86.56%
InĀ [13]:
def plot_history(history):
"""
Plots the training and validation loss, accuracy, and learning rate
from a PyTorch history dictionary.
Args:
history (dict): Dictionary containing training metrics.
Expected keys: 'train_loss', 'val_loss', 'train_acc', 'val_acc', 'lrs'.
"""
epochs = range(len(history['train_loss']))
plt.figure(figsize=(18, 5))
# Plot 1: Loss
plt.subplot(1, 3, 1)
plt.plot(epochs, history['train_loss'], label='Training Loss')
plt.plot(epochs, history['val_loss'], label='Validation Loss')
plt.title('Loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
# plt.grid(True)
# Plot 2: Accuracy
plt.subplot(1, 3, 2)
plt.plot(epochs, history['train_acc'], label='Training Accuracy')
plt.plot(epochs, history['val_acc'], label='Validation Accuracy')
plt.title('Accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
# plt.grid(True)
plt.tight_layout()
plt.show()
# plt.savefig()
plot_history(history)
InĀ [14]:
%watermark -iv
matplotlib : 3.7.2 numpy : 1.26.4 pandas : 2.2.3 sklearn : 1.2.2 torch : 2.6.0+cu124 torchinfo : 1.8.0 torchvision: 0.21.0+cu124