In [ ]:
import pandas as pd
import torch
import numpy as np
import random
In [ ]:
df = pd.read_csv('sample_data/california_housing_train.csv')
In [ ]:
df.head()
Out[ ]:
| longitude | latitude | housing_median_age | total_rooms | total_bedrooms | population | households | median_income | median_house_value | |
|---|---|---|---|---|---|---|---|---|---|
| 0 | -114.31 | 34.19 | 15.0 | 5612.0 | 1283.0 | 1015.0 | 472.0 | 1.4936 | 66900.0 |
| 1 | -114.47 | 34.40 | 19.0 | 7650.0 | 1901.0 | 1129.0 | 463.0 | 1.8200 | 80100.0 |
| 2 | -114.56 | 33.69 | 17.0 | 720.0 | 174.0 | 333.0 | 117.0 | 1.6509 | 85700.0 |
| 3 | -114.57 | 33.64 | 14.0 | 1501.0 | 337.0 | 515.0 | 226.0 | 3.1917 | 73400.0 |
| 4 | -114.57 | 33.57 | 20.0 | 1454.0 | 326.0 | 624.0 | 262.0 | 1.9250 | 65500.0 |
In [ ]:
# """
# network.py
# ~~~~~~~~~~
# A module to implement the stochastic gradient descent learning
# algorithm for a feedforward neural network. Gradients are calculated
# using backpropagation. Note that I have focused on making the code
# simple, easily readable, and easily modifiable. It is not optimized,
# and omits many desirable features.
# """
# class Network(object):
# def __init__(self, sizes):
# """The list ``sizes`` contains the number of neurons in the
# respective layers of the network. For example, if the list
# was [2, 3, 1] then it would be a three-layer network, with the
# first layer containing 2 neurons, the second layer 3 neurons,
# and the third layer 1 neuron. The biases and weights for the
# network are initialized randomly, using a Gaussian
# distribution with mean 0, and variance 1. Note that the first
# layer is assumed to be an input layer, and by convention we
# won't set any biases for those neurons, since biases are only
# ever used in computing the outputs from later layers."""
# self.num_layers = len(sizes)
# self.sizes = sizes
# self.biases = [np.random.randn(y, 1) for y in sizes[1:]]
# self.weights = [np.random.randn(y, x)
# for x, y in zip(sizes[:-1], sizes[1:])]
# def feedforward(self, a):
# """Return the output of the network if ``a`` is input."""
# for b, w in zip(self.biases, self.weights):
# a = sigmoid(np.dot(w, a)+b)
# return a
# def SGD(self, training_data, epochs, mini_batch_size, eta,
# test_data=None):
# """Train the neural network using mini-batch stochastic
# gradient descent. The ``training_data`` is a list of tuples
# ``(x, y)`` representing the training inputs and the desired
# outputs. The other non-optional parameters are
# self-explanatory. If ``test_data`` is provided then the
# network will be evaluated against the test data after each
# epoch, and partial progress printed out. This is useful for
# tracking progress, but slows things down substantially."""
# if test_data: n_test = len(list(test_data))
# n = len(list(training_data))
# for j in range(epochs):
# random.shuffle(training_data)
# mini_batches = [
# training_data[k:k+mini_batch_size]
# for k in range(0, n, mini_batch_size)]
# for mini_batch in mini_batches:
# self.update_mini_batch(mini_batch, eta)
# if test_data:
# print("Epoch {0}: {1} / {2}".format(
# j, self.evaluate(test_data), n_test))
# else:
# print("Epoch {0} complete".format(j))
# def update_mini_batch(self, mini_batch, eta):
# """Update the network's weights and biases by applying
# gradient descent using backpropagation to a single mini batch.
# The ``mini_batch`` is a list of tuples ``(x, y)``, and ``eta``
# is the learning rate."""
# nabla_b = [np.zeros(b.shape) for b in self.biases]
# nabla_w = [np.zeros(w.shape) for w in self.weights]
# for x, y in mini_batch:
# delta_nabla_b, delta_nabla_w = self.backprop(x, y)
# nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
# nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
# self.weights = [w-(eta/len(mini_batch))*nw
# for w, nw in zip(self.weights, nabla_w)]
# self.biases = [b-(eta/len(mini_batch))*nb
# for b, nb in zip(self.biases, nabla_b)]
# def backprop(self, x, y):
# """Return a tuple ``(nabla_b, nabla_w)`` representing the
# gradient for the cost function C_x. ``nabla_b`` and
# ``nabla_w`` are layer-by-layer lists of numpy arrays, similar
# to ``self.biases`` and ``self.weights``."""
# nabla_b = [np.zeros(b.shape) for b in self.biases]
# nabla_w = [np.zeros(w.shape) for w in self.weights]
# # feedforward
# activation = x
# activations = [x] # list to store all the activations, layer by layer
# zs = [] # list to store all the z vectors, layer by layer
# for b, w in zip(self.biases, self.weights):
# z = np.dot(w, activation)+b
# zs.append(z)
# activation = sigmoid(z)
# activations.append(activation)
# # backward pass
# delta = self.cost_derivative(activations[-1], y) * \
# sigmoid_prime(zs[-1])
# nabla_b[-1] = delta
# nabla_w[-1] = np.dot(delta, activations[-2].transpose())
# # Note that the variable l in the loop below is used a little
# # differently to the notation in Chapter 2 of the book. Here,
# # l = 1 means the last layer of neurons, l = 2 is the
# # second-last layer, and so on. It's a renumbering of the
# # scheme in the book, used here to take advantage of the fact
# # that Python can use negative indices in lists.
# for l in range(2, self.num_layers):
# z = zs[-l]
# sp = sigmoid_prime(z)
# delta = np.dot(self.weights[-l+1].transpose(), delta) * sp
# nabla_b[-l] = delta
# nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())
# return (nabla_b, nabla_w)
# def evaluate(self, test_data):
# """Return the number of test inputs for which the neural
# network outputs the correct result. Note that the neural
# network's output is assumed to be the index of whichever
# neuron in the final layer has the highest activation."""
# test_results = [(np.argmax(self.feedforward(x)), y)
# for (x, y) in test_data]
# return sum(int(x == y) for (x, y) in test_results)
# def cost_derivative(self, output_activations, y):
# """Return the vector of partial derivatives \partial C_x /
# \partial a for the output activations."""
# return (output_activations-y)
# #### Miscellaneous functions
# def sigmoid(z):
# """The sigmoid function."""
# return 1.0/(1.0+np.exp(-z))
# def sigmoid_prime(z):
# """Derivative of the sigmoid function."""
# return sigmoid(z)*(1-sigmoid(z))
Pseudo-code for Matrix-based approach to Backpropagation¶
function SGD_Matrix(training_data, epochs, mini_batch_size, eta):
for epoch in range(epochs):
shuffle training_data
mini_batches = split training_data into mini-batches of size mini_batch_size
for mini_batch in mini_batches:
update_mini_batch_matrix(mini_batch, eta)
function update_mini_batch_matrix(mini_batch, eta):
X, Y = extract_input_output_matrices(mini_batch)
# Initialize gradient matrices to zero
nabla_b = [zeros_like(bias_matrix) for bias_matrix in biases]
nabla_w = [zeros_like(weight_matrix) for weight_matrix in weights]
# Perform forward and backward propagation
delta_nabla_b, delta_nabla_w = backprop_matrix(X, Y)
# Update weights and biases
weights = [w - (eta/len(mini_batch)) * nw for w, nw in zip(weights, delta_nabla_w)]
biases = [b - (eta/len(mini_batch)) * nb for b, nb in zip(biases, delta_nabla_b)]
function backprop_matrix(X, Y):
# X is the input matrix, Y is the output matrix for the mini-batch
# Initialize lists to store activations and z-values for each layer
activations = [X]
zs = []
# Feedforward pass
activation = X
for b, w in zip(biases, weights):
z = dot(w, activation) + b # Compute z for the current layer
zs.append(z)
activation = sigmoid(z) # Compute the activation for the current layer
activations.append(activation)
# Backpropagation pass
delta = cost_derivative(activations[-1], Y) * sigmoid_prime(zs[-1])
nabla_b[-1] = sum(delta along the batch dimension)
nabla_w[-1] = dot(delta, activations[-2].transpose())
for l in range(2, num_layers):
z = zs[-l]
sp = sigmoid_prime(z)
delta = dot(weights[-l+1].transpose(), delta) * sp
nabla_b[-l] = sum(delta along the batch dimension)
nabla_w[-l] = dot(delta, activations[-l-1].transpose())
return (nabla_b, nabla_w)
function cost_derivative(output_activations, Y):
return output_activations - Y # Derivative of the cost function with respect to activations
function sigmoid(z):
return 1.0 / (1.0 + exp(-z))
function sigmoid_prime(z):
return sigmoid(z) * (1 - sigmoid(z))
In [ ]:
class Network(object):
def __init__(self, sizes):
self.num_layers = len(sizes)
self.sizes = sizes
self.biases = [np.random.randn(y, 1) for y in sizes[1:]]
self.weights = [np.random.randn(y, x) for x, y in zip(sizes[:-1], sizes[1:])]
def feedforward(self, a):
for b, w in zip(self.biases, self.weights):
a = sigmoid(np.dot(w, a) + b)
return a
def SGD(self, training_data, epochs, mini_batch_size, eta, test_data=None):
if test_data: n_test = len(list(test_data))
n = len(list(training_data))
for j in range(epochs):
random.shuffle(training_data)
mini_batches = [training_data[k:k+mini_batch_size] for k in range(0, n, mini_batch_size)]
for mini_batch in mini_batches:
self.update_mini_batch_matrix(mini_batch, eta)
if test_data:
print("Epoch {0}: {1} / {2}".format(j, self.evaluate(test_data), n_test))
else:
print("Epoch {0} complete".format(j))
def update_mini_batch_matrix(self, mini_batch, eta):
X = np.column_stack([x for x, y in mini_batch]) # Input matrix
Y = np.column_stack([y for x, y in mini_batch]) # Output matrix
nabla_b = [np.zeros(b.shape) for b in self.biases]
nabla_w = [np.zeros(w.shape) for w in self.weights]
delta_nabla_b, delta_nabla_w = self.backprop_matrix(X, Y)
self.weights = [w - (eta / len(mini_batch)) * nw for w, nw in zip(self.weights, delta_nabla_w)]
self.biases = [b - (eta / len(mini_batch)) * nb for b, nb in zip(self.biases, delta_nabla_b)]
def backprop_matrix(self, X, Y):
nabla_b = [np.zeros(b.shape) for b in self.biases]
nabla_w = [np.zeros(w.shape) for w in self.weights]
# Feedforward
activation = X
activations = [X]
zs = []
for b, w in zip(self.biases, self.weights):
z = np.dot(w, activation) + b
zs.append(z)
activation = sigmoid(z)
activations.append(activation)
# Backward pass
delta = self.cost_derivative(activations[-1], Y) * sigmoid_prime(zs[-1])
nabla_b[-1] = np.sum(delta, axis=1, keepdims=True)
nabla_w[-1] = np.dot(delta, activations[-2].transpose())
for l in range(2, self.num_layers):
z = zs[-l]
sp = sigmoid_prime(z)
delta = np.dot(self.weights[-l + 1].transpose(), delta) * sp
nabla_b[-l] = np.sum(delta, axis=1, keepdims=True)
nabla_w[-l] = np.dot(delta, activations[-l - 1].transpose())
return (nabla_b, nabla_w)
def evaluate(self, test_data):
test_results = [(np.argmax(self.feedforward(x)), y) for (x, y) in test_data]
return sum(int(x == y) for (x, y) in test_results)
def cost_derivative(self, output_activations, y):
return output_activations - y
def sigmoid(z):
return 1.0 / (1.0 + np.exp(-z))
def sigmoid_prime(z):
return sigmoid(z) * (1 - sigmoid(z))
In [ ]:
!pip install mnist
Requirement already satisfied: mnist in /usr/local/lib/python3.10/dist-packages (0.2.2) Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from mnist) (1.26.4)
In [ ]:
# import torch
# from torchvision import datasets, transforms
# from torch.utils.data import DataLoader, TensorDataset
# def load_data_wrapper():
# """Load and preprocess the MNIST dataset."""
# transform = transforms.Compose([
# transforms.ToTensor(),
# transforms.Normalize((0.5,), (0.5,))
# ])
# # Download and load the training data
# train_data = datasets.MNIST(root='./data', train=True, download=True, transform=transform)
# test_data = datasets.MNIST(root='./data', train=False, download=True, transform=transform)
# # Wrap the data in a TensorDataset
# train_loader = DataLoader(train_data, batch_size=len(train_data), shuffle=False)
# test_loader = DataLoader(test_data, batch_size=len(test_data), shuffle=False)
# # Extract data and labels from the loaders
# def loader_to_list(loader):
# inputs, labels = next(iter(loader))
# inputs = inputs.view(inputs.size(0), -1) # Flatten the images
# return list(zip(inputs, labels))
# training_data = loader_to_list(train_loader)
# test_data = loader_to_list(test_loader)
# # No validation data provided, but you can split training data if needed
# validation_data = []
# return training_data, validation_data, test_data
# # Ensure that the output of the load_data_wrapper function is in the expected format
# training_data, validation_data, test_data = load_data_wrapper()
In [ ]:
import mnist_loader
# Load data
training_data, validation_data, test_data = mnist_loader.load_data_wrapper()
# Convert data to lists
training_data = list(training_data)
test_data = list(test_data)
# Determine device
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print(f"Using {device} device")
# Initialize and train network
net = Network([784, 30, 10])
net.SGD(training_data, 30, 10, 3.0, test_data=test_data)
Using cpu device Epoch 0: 9144 / 10000 Epoch 1: 9239 / 10000 Epoch 2: 9341 / 10000 Epoch 3: 9357 / 10000 Epoch 4: 9379 / 10000 Epoch 5: 9402 / 10000 Epoch 6: 9432 / 10000 Epoch 7: 9418 / 10000 Epoch 8: 9463 / 10000 Epoch 9: 9468 / 10000 Epoch 10: 9449 / 10000 Epoch 11: 9488 / 10000 Epoch 12: 9501 / 10000 Epoch 13: 9486 / 10000 Epoch 14: 9497 / 10000 Epoch 15: 9495 / 10000 Epoch 16: 9476 / 10000 Epoch 17: 9514 / 10000 Epoch 18: 9495 / 10000 Epoch 19: 9497 / 10000 Epoch 20: 9508 / 10000 Epoch 21: 9508 / 10000 Epoch 22: 9512 / 10000 Epoch 23: 9521 / 10000 Epoch 24: 9491 / 10000 Epoch 25: 9503 / 10000 Epoch 26: 9502 / 10000 Epoch 27: 9513 / 10000 Epoch 28: 9517 / 10000 Epoch 29: 9540 / 10000
In [ ]:
net2 = Network([784, 100, 10])
net2.SGD(training_data, 30, 10, 3.0, test_data=test_data)
Epoch 0: 7454 / 10000 Epoch 1: 7548 / 10000 Epoch 2: 7527 / 10000 Epoch 3: 7674 / 10000 Epoch 4: 7678 / 10000 Epoch 5: 7704 / 10000 Epoch 6: 7713 / 10000 Epoch 7: 7738 / 10000 Epoch 8: 7752 / 10000 Epoch 9: 7877 / 10000 Epoch 10: 8685 / 10000 Epoch 11: 8701 / 10000 Epoch 12: 8699 / 10000 Epoch 13: 8701 / 10000 Epoch 14: 8704 / 10000 Epoch 15: 8709 / 10000 Epoch 16: 8732 / 10000 Epoch 17: 8716 / 10000 Epoch 18: 8742 / 10000 Epoch 19: 8721 / 10000 Epoch 20: 8721 / 10000 Epoch 21: 8735 / 10000 Epoch 22: 8748 / 10000 Epoch 23: 8740 / 10000 Epoch 24: 8754 / 10000 Epoch 25: 8750 / 10000 Epoch 26: 8750 / 10000 Epoch 27: 8742 / 10000 Epoch 28: 8754 / 10000 Epoch 29: 8749 / 10000
In [ ]:
net3 = Network([784, 100, 10])
net3.SGD(training_data, 30, 10, 100.0, test_data=test_data)
Epoch 0: 1010 / 10000 Epoch 1: 1009 / 10000 Epoch 2: 1010 / 10000 Epoch 3: 1010 / 10000 Epoch 4: 1010 / 10000 Epoch 5: 1010 / 10000 Epoch 6: 1010 / 10000 Epoch 7: 1010 / 10000 Epoch 8: 1010 / 10000 Epoch 9: 1010 / 10000 Epoch 10: 1010 / 10000 Epoch 11: 1010 / 10000 Epoch 12: 1010 / 10000 Epoch 13: 1010 / 10000 Epoch 14: 1010 / 10000 Epoch 15: 1010 / 10000 Epoch 16: 1010 / 10000 Epoch 17: 1010 / 10000 Epoch 18: 1010 / 10000 Epoch 19: 1010 / 10000 Epoch 20: 1010 / 10000 Epoch 21: 1010 / 10000 Epoch 22: 1010 / 10000 Epoch 23: 1010 / 10000 Epoch 24: 1010 / 10000 Epoch 25: 1010 / 10000 Epoch 26: 1010 / 10000 Epoch 27: 1010 / 10000 Epoch 28: 1010 / 10000 Epoch 29: 1010 / 10000
In [ ]:
net4 = Network([784, 100, 10])
net4.SGD(training_data, 30, 10, 0.001, test_data=test_data)
Epoch 0: 756 / 10000 Epoch 1: 716 / 10000 Epoch 2: 749 / 10000 Epoch 3: 772 / 10000 Epoch 4: 806 / 10000 Epoch 5: 819 / 10000 Epoch 6: 847 / 10000 Epoch 7: 854 / 10000 Epoch 8: 867 / 10000 Epoch 9: 882 / 10000 Epoch 10: 895 / 10000 Epoch 11: 899 / 10000 Epoch 12: 905 / 10000 Epoch 13: 919 / 10000 Epoch 14: 940 / 10000 Epoch 15: 958 / 10000 Epoch 16: 974 / 10000 Epoch 17: 991 / 10000 Epoch 18: 1005 / 10000 Epoch 19: 1013 / 10000 Epoch 20: 1027 / 10000 Epoch 21: 1045 / 10000 Epoch 22: 1052 / 10000 Epoch 23: 1068 / 10000 Epoch 24: 1084 / 10000 Epoch 25: 1113 / 10000 Epoch 26: 1130 / 10000 Epoch 27: 1144 / 10000 Epoch 28: 1156 / 10000 Epoch 29: 1170 / 10000
In [ ]:
net5 = Network([784, 10])
net5.SGD(training_data, 30, 10, 3.0, test_data=test_data)
Epoch 0: 6304 / 10000 Epoch 1: 7465 / 10000 Epoch 2: 7527 / 10000 Epoch 3: 7539 / 10000 Epoch 4: 7555 / 10000 Epoch 5: 7578 / 10000 Epoch 6: 7556 / 10000 Epoch 7: 7566 / 10000 Epoch 8: 7555 / 10000 Epoch 9: 7566 / 10000 Epoch 10: 7569 / 10000 Epoch 11: 7566 / 10000 Epoch 12: 7582 / 10000 Epoch 13: 7573 / 10000 Epoch 14: 7570 / 10000 Epoch 15: 7569 / 10000 Epoch 16: 7575 / 10000 Epoch 17: 7573 / 10000 Epoch 18: 7586 / 10000 Epoch 19: 7584 / 10000 Epoch 20: 7589 / 10000 Epoch 21: 7589 / 10000 Epoch 22: 7578 / 10000 Epoch 23: 7585 / 10000 Epoch 24: 7602 / 10000 Epoch 25: 7604 / 10000 Epoch 26: 8344 / 10000 Epoch 27: 8372 / 10000 Epoch 28: 8386 / 10000 Epoch 29: 8399 / 10000
In [ ]: