import math, random
import numpy as np
import pandas as pd
import seaborn as sns
import torch
import torch.nn.functional as F
import nltk
from collections import defaultdict
from tqdm import tqdm


import matplotlib.pyplot as plt
from matplotlib.colors import Normalize
%matplotlib inline

#!wget https://raw.githubusercontent.com/karpathy/makemore/master/names.txt # use if you don't have the file

words = open('../data/names.txt', 'r').read().splitlines()
words[:10]

['emma',
 'olivia',
 'ava',
 'isabella',
 'sophia',
 'charlotte',
 'mia',
 'amelia',
 'harper',
 'evelyn']

len(words), min(len(w) for w in words), max(len(w) for w in words)

(32033, 2, 15)

words[:10]

['emma',
 'olivia',
 'ava',
 'isabella',
 'sophia',
 'charlotte',
 'mia',
 'amelia',
 'harper',
 'evelyn']

t = {}
for w in words:
    #chs = ['<S>'] + list(w) + ['<E>']
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        #print(ch1, ch2, ch3)
        trigram = (ch1, ch2, ch3)
        #print(trigram)
        
        # add up all bigrams & count how often they occur
        # if bigram is not in the dictionary, b.get(bigram,0) returns 0
        t[trigram] = t.get(trigram, 0) + 1

{key:t[key] for idx, key in enumerate(t) if idx<10}

{('.', 'e', 'm'): 288,
 ('e', 'm', 'm'): 100,
 ('m', 'm', 'a'): 72,
 ('m', 'a', '.'): 174,
 ('.', 'o', 'l'): 104,
 ('o', 'l', 'i'): 69,
 ('l', 'i', 'v'): 54,
 ('i', 'v', 'i'): 78,
 ('v', 'i', 'a'): 147,
 ('i', 'a', '.'): 903}

len(sorted(t.items(), key = lambda kv: -kv[1]))

6037

# Sorts the trigram dict. items based on their counts in desc. order 

# Sorts trigram dict items based on the counts of the bigrams (kv[1]) in desc. order, 
# resulting in a list of tuples sorted by frequency.
# lambda: extracts the 2nd element (kv[1]) of each key-value pair and 
        # negates it to sort in descending order.
        # kv: represents each tuple in the list (('a', 'b', 'c'), 198)
        # -kv[1]: returns negated value of second elem in tuple
        # key = lambda kv: -kv[1] --> sorting in descending order of frequency

sorted(t.items(), key = lambda kv: -kv[1])[:10] # 10 most common trigrams

[(('a', 'h', '.'), 1714),
 (('n', 'a', '.'), 1673),
 (('a', 'n', '.'), 1509),
 (('o', 'n', '.'), 1503),
 (('.', 'm', 'a'), 1453),
 (('.', 'j', 'a'), 1255),
 (('.', 'k', 'a'), 1254),
 (('e', 'n', '.'), 1217),
 (('l', 'y', 'n'), 976),
 (('y', 'n', '.'), 953)]

N = torch.zeros((27, 27, 27), dtype=torch.int32) # We add one dimension compared to the bigram model

chars = sorted(list(set(''.join(words))))
stoi = {s:i+1 for i,s in enumerate(chars)}
stoi['.'] = 0
itos = {i:s for s,i in stoi.items()}
print(f"string to integer dict: {stoi}\n")
print(f"integer to string dict: {itos}")

string to integer dict: {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26, '.': 0}

integer to string dict: {1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}

for w in words:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        N[ix1, ix2, ix3] +=1

df = pd.DataFrame(list(t.items()), columns=['Trigram', 'Count'])
#df['Trigram'] = df['Trigram'].apply(lambda x: (''.join(x[:-1]), x[-1]))
df[['Input', 'Target']] = df['Trigram'].apply(lambda x: pd.Series([''.join(x[:2]), x[2]]))
#df['Input'] = df['Input'].apply(lambda row: str.join(row))
df.drop(columns=['Trigram'], inplace=True)
df = df[['Input', 'Target', 'Count']]
df.head()

#df.groupby('Input').mean().sort_values('Count', ascending=False)
df.groupby('Input').sum().sort_values('Count', ascending=False)

N.sum(2, keepdim=True) .shape

torch.Size([27, 27, 1])

N.sum(2).shape

torch.Size([27, 27])

plt.imshow(N.sum(axis=2))
plt.show()

plt.figure(figsize=(16, 16))
plt.imshow(N.sum(axis=2), cmap='Blues')
for i in range(27):
    for j in range(27):
        chstr = itos[i] + itos[j]
        plt.text(j, i, chstr, ha="center", va="bottom", color='gray')
        plt.text(j, i, N.sum(axis=2)[i, j].item(), ha="center", va="top", color='gray')
plt.axis('off');

# Create subplots
fig, axes = plt.subplots(1, 3, figsize=(18, 6))

# Iterate over alpha values
for alpha, ax in enumerate(axes):
    im = ax.imshow(N.sum(axis=alpha), cmap='Blues')
    ax.set_xticks(range(27))
    ax.set_yticks(range(27))
    ax.set_xticklabels([itos[i] for i in range(27)], rotation=45)
    #ax.set_yticklabels([itos[i] for i in range(27)])
    ax.set_yticklabels([itos[j] + itos[k] for j in range(27) for k in range(27)])
    ax.set_xlabel('First Two Characters')
    ax.set_ylabel('Third Character')
    ax.set_title(f'Trigram Counts (Alpha={alpha})')
    
    # Add colorbar
    cbar = plt.colorbar(im, ax=ax, orientation='vertical', shrink=0.6, pad=0.05)
    #cbar.set_label('Trigram Count')

# Adjust layout
plt.tight_layout()
plt.show()

fig, axes = plt.subplots(1, 3, figsize=(18, 6), sharex=True, sharey=True)

# Loop through each axis and plot the trigram counts
for i, ax in enumerate(axes.flatten()):
    ax.imshow(N.sum(axis=i), cmap='Blues')
    ax.set_title(f'Trigram Counts (Axis {i})')
    ax.set_xlabel('Second Character')
    ax.set_ylabel('First Two Characters')
    ax.set_xticks(range(27))
    ax.set_yticks(range(27))
    ax.set_xticklabels([itos[i] for i in range(27)])
    ax.set_yticklabels([itos[i] for i in range(27)])
    ax.tick_params(axis='both', labelsize=8)

plt.colorbar(im, ax=axes.ravel().tolist()[0], label='Trigram Count')
plt.tight_layout()
plt.show()

# Plot the trigram counts
fig, axes = plt.subplots(1, 3, figsize=(18, 6), sharex=True, sharey=True)

# Loop through each axis and plot the trigram counts
for i, ax in enumerate(axes.flatten()):
    ax.imshow(N.sum(axis=i), cmap='Blues')
    ax.set_title(f'Trigram Counts (Axis {i})')
    ax.set_xlabel('Second Character')
    ax.set_ylabel('First Two Characters')
    ax.set_xticks(range(27))
    ax.set_yticks(range(27))
    ax.set_xticklabels([itos[i] for i in range(27)])
    ax.set_yticklabels([itos[j] + itos[k] for j in range(27) for k in range(27)])
    ax.tick_params(axis='both', labelsize=8)

plt.colorbar(im, ax=axes.ravel().tolist()[0], label='Trigram Count')
plt.tight_layout()
plt.show()

[itos[j] + itos[k] for j in range(27) for k in range(27)][:10]

['..', '.a', '.b', '.c', '.d', '.e', '.f', '.g', '.h', '.i']

slice_index = N.shape[0] // 2
slice_data = N[slice_index, :, :]

# Plot the slice
plt.imshow(slice_data, cmap='viridis')
plt.colorbar()
plt.title('Slice of 3D Tensor')
plt.show()

# Create grid coordinates
x, y, z = np.meshgrid(np.arange(27), np.arange(27), np.arange(27))

# Plot 3D tensor
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(x, y, z, c=N.flatten(), cmap='viridis')
ax.set_title('3D Tensor Visualization')
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
plt.show()

__words = '.'+'.'.join(words[:5])+'.'
__words

'.emma.olivia.ava.isabella.sophia.'

__bigrams = [__words[i:i+2] for i in range(0, len(__words)-1)]
__bigrams = [(s[0], s[1]) for s in __bigrams]
__bigrams[:1]

[('.', 'e')]

len(nltk.FreqDist(__bigrams)), nltk.FreqDist(__bigrams)

(27,
 FreqDist({('a', '.'): 5, ('i', 'a'): 2, ('.', 'e'): 1, ('e', 'm'): 1, ('m', 'm'): 1, ('m', 'a'): 1, ('.', 'o'): 1, ('o', 'l'): 1, ('l', 'i'): 1, ('i', 'v'): 1, ...}))

a = nltk.word_tokenize("I am going to the store")
a

['I', 'am', 'going', 'to', 'the', 'store']

a2 = nltk.ngrams(a,2)
list(a2)

[('I', 'am'),
 ('am', 'going'),
 ('going', 'to'),
 ('to', 'the'),
 ('the', 'store')]

nltk.FreqDist(__bigrams).most_common(4)

[(('a', '.'), 5), (('i', 'a'), 2), (('.', 'e'), 1), (('e', 'm'), 1)]

__words = '.'+'.'.join(words)+'.'
__bigrams = [__words[i:i+2] for i in range(0, len(__words)-1)]
__bigrams = [(s[0], s[1]) for s in __bigrams]
nltk.FreqDist(__bigrams).most_common(5)

[(('n', '.'), 6763),
 (('a', '.'), 6640),
 (('a', 'n'), 5438),
 (('.', 'a'), 4410),
 (('e', '.'), 3983)]

names_with_periods_list = ['.' + name + '.' for name in words]
names_with_periods_list[:5]

['.emma.', '.olivia.', '.ava.', '.isabella.', '.sophia.']

x = []
for i in names_with_periods_list[:5]:
    #print(i)
    for j in range(len(i)-2):
        #print(i[j:j+2])
        x.append(i[j:j+3])
x[:5]

['.em', 'emm', 'mma', 'ma.', '.ol']

def extract_ngrams(names:list, n:int):
    names = ['.' + name + '.' for name in names] # to keep track of start and end of names
    ngrams_list = []
    for name in names:
        for i in range(len(name)-n+1):
            ngrams_list.append(name[i:i+n])
    return ngrams_list

def find_ngrams(names, n):
    #names = ['.' + name + '.' for name in names] 
    names = '.'.join(names)+'.'
    return list(zip(*[names[i:] for i in range(n)]))


def make_char_ngram(names:list, N:int):
    data = defaultdict(int) # for speed
    text = '.'+'.'.join(names)+'.'
    for i in range(len(text)-N+1):
        x = text[i:i+N] # actual N-gram
        data[x] += 1    # add 1 to this N-gram key
    return data

len(find_ngrams(words, 3)),\
len(nltk.FreqDist(find_ngrams(words, 3))),\
nltk.FreqDist(find_ngrams(words, 3)).most_common(5)

(228144,
 6631,
 [(('a', 'h', '.'), 1714),
  (('n', 'a', '.'), 1673),
  (('a', 'n', '.'), 1509),
  (('o', 'n', '.'), 1503),
  (('.', 'm', 'a'), 1453)])

len(make_char_ngram(words,3))

6631

#make_char_ngram(words, 3).items()
sorted(make_char_ngram(words, 3).items(), key = lambda kv: -kv[1])[:10]

[('ah.', 1714),
 ('na.', 1673),
 ('an.', 1509),
 ('on.', 1503),
 ('.ma', 1453),
 ('a.a', 1277),
 ('.ja', 1255),
 ('.ka', 1254),
 ('en.', 1217),
 ('lyn', 976)]

_b = extract_ngrams(words, 2)
_t = extract_ngrams(words, 3)
len(_b), len(_t)

(228146, 196113)

_b[:5], _t[:5]

(['.e', 'em', 'mm', 'ma', 'a.'], ['.em', 'emm', 'mma', 'ma.', '.ol'])

len(nltk.FreqDist(_b)), len(nltk.FreqDist(_t))

(627, 6037)

print(f'The most common bigrams are: {nltk.FreqDist(_b).most_common(5)}\n')
print(f'The most common trigrams are: {nltk.FreqDist(_t).most_common(5)}')

The most common bigrams are: [('n.', 6763), ('a.', 6640), ('an', 5438), ('.a', 4410), ('e.', 3983)]

The most common trigrams are: [('ah.', 1714), ('na.', 1673), ('an.', 1509), ('on.', 1503), ('.ma', 1453)]

sorted(nltk.FreqDist(_b).items(), key=lambda kv: -kv[1])[:5]

[('n.', 6763), ('a.', 6640), ('an', 5438), ('.a', 4410), ('e.', 3983)]

sorted(nltk.FreqDist(_t).items(), key=lambda kv: -kv[1])[:5]

[('ah.', 1714), ('na.', 1673), ('an.', 1509), ('on.', 1503), ('.ma', 1453)]

## Creating FreqDist for whole BoW, keeping the 20 most common tokens
all_fdist = nltk.FreqDist(_t).most_common(20)

## Conversion to Pandas series via Python Dictionary for easier plotting
all_fdist = pd.Series(dict(all_fdist))

## Setting figure, ax into variables
fig, ax = plt.subplots(figsize=(10,10))

## Seaborn plotting using Pandas attributes + xtick rotation for ease of viewing
all_plot = sns.barplot(x=all_fdist.index, y=all_fdist.values, ax=ax)
plt.xticks(rotation=30);
plt.show()

## Join bigram tokens with '_' + maintain sorting
ngram_joined = {'_'.join(k):v for k,v in sorted(all_fdist.items(), key=lambda item:-item[1])}

## Convert to Pandas series for easy plotting
ngram_freqdist = pd.Series(ngram_joined)
ngram_freqdist_percent = 100 * ngram_freqdist / sum(ngram_freqdist)

## Setting figure & ax for plots
fig, ax = plt.subplots(figsize=(10,10))

## Setting plot to horizontal for easy viewing + setting title + display  
bar_plot = sns.barplot(x=ngram_freqdist_percent.values, y=ngram_freqdist.index, orient='h', ax=ax)
plt.title('Frequency Distribution')
plt.show()

sum(ngram_freqdist), sum(ngram_freqdist_percent)

(21728, 100.00000000000001)

ngram_freqdist_percent.head()

a_h_.    7.888439
n_a_.    7.699742
a_n_.    6.944956
o_n_.    6.917342
._m_a    6.687224
dtype: float64

N[0][0], N[0][0].shape

(tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
         0, 0, 0], dtype=torch.int32),
 torch.Size([27]))

# Broadcasting rules check
# 27, 27
# 27, 1   (column vector)

P = (N[0]+1).float()                  
P /= P.sum(1, keepdim=True)          
P[0].sum(), P.shape, P.sum(1).shape, P.sum(1, keepdim=True).shape

(tensor(1.), torch.Size([27, 27]), torch.Size([27]), torch.Size([27, 1]))

[P[i].sum() for i in range(P.sum(1, keepdim=True).shape[0])],\
len([P[i].sum() for i in range(P.sum(1, keepdim=True).shape[0])])

([tensor(1.),
  tensor(1.0000),
  tensor(1.),
  tensor(1.),
  tensor(1.0000),
  tensor(1.0000),
  tensor(1.),
  tensor(1.),
  tensor(1.),
  tensor(1.),
  tensor(1.0000),
  tensor(1.),
  tensor(1.0000),
  tensor(1.0000),
  tensor(1.0000),
  tensor(1.),
  tensor(1.),
  tensor(1.),
  tensor(1.),
  tensor(1.0000),
  tensor(1.),
  tensor(1.),
  tensor(1.),
  tensor(1.),
  tensor(1.0000),
  tensor(1.0000),
  tensor(1.)],
 27)

P.sum(1, keepdim=True)

tensor([[1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000],
        [1.0000]])

# Broadcasting rules check
# 27, 27, 27
# 27, 27, 1

P = (N+1).float() 
# P.sum(2) because we want the third level of the array to be normalized to get the probabilities
P /= P.sum(2, keepdim=True) 
P[0][0].sum(), P.shape, P.sum(2).shape, P.sum(2, keepdim=True).shape

(tensor(1.),
 torch.Size([27, 27, 27]),
 torch.Size([27, 27]),
 torch.Size([27, 27, 1]))

list(P.sum(2, keepdim=True).shape)

[27, 27, 1]

[x,y,z] = list(P.sum(2, keepdim=True).shape)
len([P[i][j].sum() for i in range(x) for j in range(y)]),\
[P[i][j].sum() for i in range(x) for j in range(y)][:10]
# for i in range(P.sum(2, keepdim=True).shape[0]):
#     for j in range(P.sum(2, keepdim=True).shape[1]):
#         print(P[i][j].sum())

(729,
 [tensor(1.),
  tensor(1.0000),
  tensor(1.),
  tensor(1.),
  tensor(1.0000),
  tensor(1.0000),
  tensor(1.),
  tensor(1.),
  tensor(1.),
  tensor(1.)])

out = ['.', '.', 'a', 'c', 'e']
out[-2], out[-1], stoi[out[-2]], stoi[out[-1]]

('c', 'e', 3, 5)

pp = P[stoi[out[-2]], stoi[out[-1]]]
pp

tensor([0.4066, 0.0225, 0.0035, 0.0190, 0.0173, 0.0381, 0.0017, 0.0017, 0.0017,
        0.0121, 0.0017, 0.0035, 0.1609, 0.0087, 0.0830, 0.0225, 0.0069, 0.0017,
        0.0536, 0.0450, 0.0190, 0.0035, 0.0035, 0.0017, 0.0017, 0.0554, 0.0035])

# finally, sample from the 'neural net' model
g = torch.Generator().manual_seed(1478)
ix = torch.multinomial(pp, num_samples=1, replacement=True, generator=g).item()
itos[ix]

't'

torch.multinomial(pp, num_samples=100, replacement=True, generator=g)

tensor([12, 12,  0,  0,  0, 15, 12, 15,  0, 14,  0, 18, 14,  3, 14,  0, 19,  0,
         5, 12, 24, 12,  0,  5, 12,  2, 18,  0,  0, 12, 25, 18, 19,  0,  0, 26,
        18,  0, 18, 20, 14, 14, 14, 18, 14,  0,  1,  0,  0, 26, 14,  0,  0,  0,
        12, 18,  0,  0,  0, 15,  0, 14, 13,  0,  0,  0,  0, 19,  0, 14,  0,  0,
        25,  0, 18,  0,  0,  5,  0,  0, 12,  0,  5, 14,  0, 12,  0,  0, 12, 14,
        16,  5,  0,  5,  0,  0,  0,  1,  0,  0])

g = torch.Generator().manual_seed(1478)

for i in range(10):
    out = ['.', '.']
    #ix = 0
    while True:
        p = P[stoi[out[-2]], stoi[out[-1]]]  # Plug the last two chars into our probabilities table
        ix = torch.multinomial(p, num_samples=1, replacement=True, generator=g).item()
        out.append(itos[ix])
        if ix == 0:
            break
    print(''.join(out))

..xina.
..josmaratresarielyn.
..mann.
..fiydracyr.
..vonerrichaizi.
..inese.
..grani.
...
..ilyim.
..dy.

for w in words[:3]:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        prob = P[ix1, ix2, ix3]
        print(f'{ch1}{ch2}{ch3}: {prob:.4f}')

.em: 0.1855
emm: 0.1269
mma: 0.3744
ma.: 0.0669
.ol: 0.2494
oli: 0.1084
liv: 0.0219
ivi: 0.2669
via: 0.1578
ia.: 0.3657
.av: 0.0550
ava: 0.1882
va.: 0.1405

log_likelihood = 0.0
n = 0
for w in words[:5]:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        prob = P[ix1, ix2, ix3]
        logprob = torch.log(prob)
        log_likelihood += logprob
        n += 1
        print(f'{ch1}{ch2}{ch3}: {prob:.4f}')
        
print(f'\nlog_likelihood = {log_likelihood:.4f}')
nll = -log_likelihood
print(f'negative log_likelihood = {nll:.4f}\n')
avg_nll = nll/n
print(f'average negative log_likelihood = {avg_nll:.4f}')

.em: 0.1855
emm: 0.1269
mma: 0.3744
ma.: 0.0669
.ol: 0.2494
oli: 0.1084
liv: 0.0219
ivi: 0.2669
via: 0.1578
ia.: 0.3657
.av: 0.0550
ava: 0.1882
va.: 0.1405
.is: 0.2023
isa: 0.1065
sab: 0.0627
abe: 0.3063
bel: 0.2962
ell: 0.2513
lla: 0.2464
la.: 0.2585
.so: 0.0735
sop: 0.0394
oph: 0.3115
phi: 0.2684
hia: 0.1085
ia.: 0.3657

log_likelihood = -50.2693
negative log_likelihood = 50.2693

average negative log_likelihood = 1.8618

N = torch.zeros((27, 27, 27), dtype=torch.int32)
chars = sorted(list(set(''.join(words))))
stoi = {s:i+1 for i,s in enumerate(chars)}
stoi['.'] = 0
itos = {i:s for s,i in stoi.items()}

for w in words:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        N[ix1, ix2, ix3] +=1
        
P = (N+1).float() 
P /= P.sum(2, keepdim=True)

g = torch.Generator().manual_seed(1478)

for i in range(10):
    out = ['.', '.']
    ix = 0
    while True:
        p = P[stoi[out[-2]], stoi[out[-1]]]  # Plug the last two chars into our probabilities table
        ix = torch.multinomial(p, num_samples=1, replacement=True, generator=g).item()
        out.append(itos[ix])
        if ix == 0:
            break
    print(''.join(out))

..xina.
..josmaratresarielyn.
..mann.
..fiydracyr.
..vonerrichaizi.
..inese.
..grani.
...
..ilyim.
..dy.

log_likelihood = 0.0
n = 0
for w in words:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        prob = P[ix1, ix2, ix3]
        logprob = torch.log(prob)
        log_likelihood += logprob
        n += 1
        
print(f'\nlog_likelihood = {log_likelihood:.4f}')
nll = -log_likelihood
print(f'negative log_likelihood = {nll:.4f}\n')
avg_nll = nll/n
print(f'average negative log_likelihood = {avg_nll:.4f}')

log_likelihood = -410414.9688
negative log_likelihood = 410414.9688

average negative log_likelihood = 2.0927

xs, ys = [], []

for w in words:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        #print(ch1, ch2, ch3)
        xs.append((ix1, ix2))
        ys.append(ix3)
        
xs = torch.tensor(xs) 
ys = torch.tensor(ys)

g = torch.Generator().manual_seed(1478)
W = torch.randn((27*2, 27), generator=g, requires_grad=True)

xenc = F.one_hot(xs, num_classes=27).float()  # input to the network: one-hot encoding
logits = xenc.view(-1, 27*2) @ W              # predict log-counts
counts = logits.exp()                         # counts, equivalent to N
probs = counts / counts.sum(1, keepdim=True)
#print(xenc.shape, counts.shape, probs.shape)
#loss = -probs[torch.arange(ys.shape[0]), ys].log().mean() # ys.shape[0] is the number of examples (allows us to avoid using a count variable)

xs.shape, xs.dtype, ys.shape, ys.nelement()

(torch.Size([196113, 2]), torch.int64, torch.Size([196113]), 196113)

print(xenc.shape, W.shape, counts.shape, probs.shape)

torch.Size([196113, 2, 27]) torch.Size([54, 27]) torch.Size([196113, 27]) torch.Size([196113, 27])

#xs, ys
#xs.view(-1, 27*2).shape
xenc.shape, xenc.view(-1, 27*2).shape, W.shape

(torch.Size([196113, 2, 27]), torch.Size([196113, 54]), torch.Size([54, 27]))

probs[1]

tensor([0.1189, 0.0051, 0.0123, 0.0038, 0.3363, 0.0004, 0.0195, 0.0013, 0.0135,
        0.0617, 0.0037, 0.0161, 0.0078, 0.0083, 0.0203, 0.0176, 0.1262, 0.0057,
        0.0155, 0.0389, 0.0120, 0.0134, 0.0027, 0.0081, 0.0130, 0.0018, 0.1161],
       grad_fn=<SelectBackward0>)

num = 5
nlls = torch.zeros(num)
for i in range(num):
    # i-th trigram:
    x = xs.numpy().tolist()[i] 
    y = ys[i].item() 
    #print(x)
    #print(y)
    print('--------\n')
    print(f'trigram example {i+1}: {itos[x[0]]}{itos[x[1]]}{itos[y]} (indexes {x},{y})')
    print('input to the neural net:', x)
    print('output probabilities from the neural net:', probs[i])
    print('label (actual next character):', y)
    p = probs[i, y]
    print(f'probability assigned by the net to the the correct character: {p.item():.6f}')
    logp = torch.log(p)
    #print('log likelihood:', logp.item()%.4f)
    print(f'log likelihood: {logp.item():.4f}')

    nll = -logp
    print(f'negative log likelihood: {nll.item():.4f}')
    nlls[i] = nll

print('\n=========')
print(f'average negative log likelihood, i.e. loss = {nlls.mean().item():.4f}')

--------

trigram example 1: .em (indexes [0, 5],13)
input to the neural net: [0, 5]
output probabilities from the neural net: tensor([0.0076, 0.0152, 0.0289, 0.0012, 0.0266, 0.0049, 0.0239, 0.0417, 0.0110,
        0.1575, 0.0446, 0.1141, 0.0203, 0.0036, 0.0024, 0.0013, 0.0078, 0.2472,
        0.0183, 0.0078, 0.0031, 0.0197, 0.0050, 0.0279, 0.0620, 0.0048, 0.0916],
       grad_fn=<SelectBackward0>)
label (actual next character): 13
probability assigned by the net to the the correct character: 0.003583
log likelihood: -5.6314
negative log likelihood: 5.6314
--------

trigram example 2: emm (indexes [5, 13],13)
input to the neural net: [5, 13]
output probabilities from the neural net: tensor([0.1189, 0.0051, 0.0123, 0.0038, 0.3363, 0.0004, 0.0195, 0.0013, 0.0135,
        0.0617, 0.0037, 0.0161, 0.0078, 0.0083, 0.0203, 0.0176, 0.1262, 0.0057,
        0.0155, 0.0389, 0.0120, 0.0134, 0.0027, 0.0081, 0.0130, 0.0018, 0.1161],
       grad_fn=<SelectBackward0>)
label (actual next character): 13
probability assigned by the net to the the correct character: 0.008272
log likelihood: -4.7949
negative log likelihood: 4.7949
--------

trigram example 3: mma (indexes [13, 13],1)
input to the neural net: [13, 13]
output probabilities from the neural net: tensor([0.1742, 0.0041, 0.0109, 0.0006, 0.3083, 0.0055, 0.0069, 0.0041, 0.0139,
        0.0202, 0.0037, 0.0025, 0.0301, 0.0141, 0.0050, 0.0066, 0.1897, 0.0050,
        0.0031, 0.0042, 0.0096, 0.1362, 0.0009, 0.0072, 0.0004, 0.0068, 0.0263],
       grad_fn=<SelectBackward0>)
label (actual next character): 1
probability assigned by the net to the the correct character: 0.004090
log likelihood: -5.4992
negative log likelihood: 5.4992
--------

trigram example 4: ma. (indexes [13, 1],0)
input to the neural net: [13, 1]
output probabilities from the neural net: tensor([0.1586, 0.0470, 0.0041, 0.0005, 0.0343, 0.0201, 0.0364, 0.0063, 0.0067,
        0.0046, 0.0051, 0.0185, 0.1030, 0.0176, 0.0148, 0.0308, 0.0474, 0.2389,
        0.0055, 0.0006, 0.0427, 0.0298, 0.0059, 0.0180, 0.0008, 0.0558, 0.0460],
       grad_fn=<SelectBackward0>)
label (actual next character): 0
probability assigned by the net to the the correct character: 0.158585
log likelihood: -1.8415
negative log likelihood: 1.8415
--------

trigram example 5: .ol (indexes [0, 15],12)
input to the neural net: [0, 15]
output probabilities from the neural net: tensor([0.0015, 0.0114, 0.0107, 0.0013, 0.0155, 0.0133, 0.0090, 0.0069, 0.0031,
        0.0385, 0.0191, 0.3142, 0.0075, 0.0022, 0.0133, 0.0039, 0.0239, 0.2257,
        0.0097, 0.0171, 0.0057, 0.0400, 0.0031, 0.0138, 0.0391, 0.0063, 0.1443],
       grad_fn=<SelectBackward0>)
label (actual next character): 12
probability assigned by the net to the the correct character: 0.007525
log likelihood: -4.8896
negative log likelihood: 4.8896

=========
average negative log likelihood, i.e. loss = 4.5313

xs, ys = [], []

for w in words:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        #print(ch1, ch2, ch3)
        xs.append((ix1, ix2))
        ys.append(ix3)
        
xs = torch.tensor(xs) 
ys = torch.tensor(ys)
num = ys.nelement()
print('number of examples: ', num)

# initialize the 'network'
g = torch.Generator().manual_seed(1478)
W = torch.randn((27*2, 27), generator=g, requires_grad=True)
# Got stuck a bit on this part and didn't realize how many neurons I'd be needing, 
# started with (27x27, 27) but I think this was not necessary
# You just need 27x2, the first 27 will activate for the first input, the second 27 for the second one
# Double the inputs, double the neurons, that makes sense!

number of examples:  196113

%%time
n_iter = 250
lr = 50
C = 0.01
losses = []
# gradient descent
for k in range(n_iter):

    # forward pass
    xenc = F.one_hot(xs, num_classes=27).float()       # input to the network: one-hot encoding
    logits = xenc.view(-1, 27*2) @ W                   # predict log-counts
    counts = logits.exp()                              # counts, equivalent to N
    probs = counts / counts.sum(1, keepdim=True)      # probabilities for next character
    loss = -probs[torch.arange(num), ys].log().mean() + C*(W**2).mean() 
    #print(loss.item())
    losses.append(loss)

    # backward pass
    W.grad = None # set to zero the gradient
    loss.backward()

    # update
    W.data += -lr * W.grad
print(f'\naverage negative log_likelihood = {loss.item():.4f}\n')

average negative log_likelihood = 2.2573

CPU times: user 51.9 s, sys: 7.93 s, total: 59.8 s
Wall time: 26.7 s

print(f"train_loss (final): {losses[-1]:.4f}")
plt.plot(losses)
plt.xlabel("epoch")
plt.ylabel("loss")
plt.title("Loss on training data")
plt.grid() 
plt.plot()

train_loss (final): 2.2573

[]

_xenc = xenc.view(-1, 27*2)
sns.heatmap(_xenc==0, vmin=0, vmax=1, cbar=False)
plt.title("Feature matrix of X_enc")
plt.show()

sns.heatmap(W.detach().numpy(), vmin=0, vmax=1, cbar=False)
plt.title("Feature matrix of W")
plt.show()

# logits = x_enc @ w # predict log-counts
counts = W.exp()
probas = counts / counts.sum(1, keepdims=True)
probas_t = probas.T 

plt.figure(figsize=(30, 20))
plt.imshow(probas_t.detach().numpy())
#plt.imshow(probas_t.tolist())
plt.title("(row) normalized Exp(w.T)")
plt.show()

probas_t[:, 0:27].shape

torch.Size([27, 27])

random.seed(1245)
words = open('../data/names.txt', 'r').read().splitlines()
random.shuffle(words)

words[:10]

['urban',
 'tal',
 'myrah',
 'sanya',
 'anelle',
 'daniel',
 'naika',
 'paitynn',
 'kierstin',
 'mayah']

train = words[ : int(len(words) * 0.8)]
val = words[int(len(words) * 0.8) : int(len(words) * 0.9)]
test = words[int(len(words) * 0.9) : ]

len(train), len(val), len(test), len(words)

(25626, 3203, 3204, 32033)

print(f'{len(train)/len(words)*100:.0f}% training set')
print(f'{len(val)/len(words)*100:.0f}% validation set')
print(f'{len(test)/len(words)*100:.0f}% test set')

80% training set
10% validation set
10% test set

xs_tr, ys_tr = [], []

for w in train:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        xs_tr.append((ix1, ix2))
        ys_tr.append(ix3)
        
xs_tr = torch.tensor(xs_tr) 
ys_tr = torch.tensor(ys_tr)
num_tr = ys_tr.nelement()
print('number of examples: ', num_tr)

# initialize the 'network'
g = torch.Generator().manual_seed(1478)
W = torch.randn((27*2, 27), generator=g, requires_grad=True)

number of examples:  156751

%%time
n_iter = 250
lr = 50
C = 0.01
losses_tr = []
# gradient descent
for k in range(n_iter):

    # forward pass
    xenc_tr = F.one_hot(xs_tr, num_classes=27).float()      
    logits = xenc_tr.view(-1, 27*2) @ W               
    counts = logits.exp()                             
    probs = counts / counts.sum(1, keepdim=True)      
    loss_tr = -probs[torch.arange(num_tr), ys_tr].log().mean() + C*(W**2).mean() 
    #print(loss.item())
    losses_tr.append(loss_tr)

    # backward pass
    W.grad = None # set to zero the gradient
    loss_tr.backward()

    # update
    W.data += -lr * W.grad
print(f'\naverage negative log_likelihood = {loss_tr.item():.4f}\n')

print(f"train_loss (final): {losses_tr[-1]:.4f}")
plt.plot(losses_tr)
plt.xlabel("epoch")
plt.ylabel("loss")
plt.title("Loss on training data")
plt.grid() 
plt.plot()

average negative log_likelihood = 2.2576

train_loss (final): 2.2576
CPU times: user 34.4 s, sys: 6.21 s, total: 40.6 s
Wall time: 19 s

[]

xenc_tr.shape, W.shape

(torch.Size([156751, 2, 27]), torch.Size([54, 27]))

xs_val, ys_val = [], []
losses_val = []

for w in val:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        xs_val.append((ix1, ix2))
        ys_val.append(ix3)
        
xs_val2 = torch.tensor(xs_val) 
ys_val2 = torch.tensor(ys_val)
num_val = ys_val2.nelement()

xenc_val = F.one_hot(xs_val2, num_classes=27).float()      
logits = xenc_val.view(-1, 27*2) @ W                   
counts = logits.exp()                             
probs = counts / counts.sum(1, keepdim=True)     
loss_val = -probs[torch.arange(num_val), ys_val2].log().mean()
    
print(f'The loss on the validation set is {loss_val.item():.4f}')

The loss on the validation set is 2.2514

xs_test, ys_test = [], []
losses_test = []

for w in test:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        xs_test.append((ix1, ix2))
        ys_test.append(ix3)
        
xs_test2 = torch.tensor(xs_test) 
ys_test2 = torch.tensor(ys_test)
num_test = ys_test2.nelement()

xenc_test = F.one_hot(xs_test2, num_classes=27).float()      
logits = xenc_test.view(-1, 27*2) @ W                   
counts = logits.exp()                             
probs = counts / counts.sum(1, keepdim=True)     
loss_test = -probs[torch.arange(num_test), ys_test2].log().mean()

print(f'The loss on the test set is {loss_test.item():.4f}')

The loss on the test set is 2.2460

#reg_param_list = [0, 0.001, 0.01, 0.02, 0.05, 0.1, 0.25, 0.5, 1.0]
#reg_param_list = [0.5, 0.375, 0.25, 0.1, 0.05, 0.01, 0.005, 0.001, 0.0005, 0.0001, 0.00005, 0.00001][::-1]
#reg_param_list = [0, 0.02, 0.05, 0.1]
reg_param_list = np.geomspace(0.001, 0.5, 12)

loss_dict = {} # <reg_strength: [final train loss, final val loss]>
n_iter = 100
lr = 50

for C in tqdm(reg_param_list):
    # Initialize model & losses
    W = torch.randn((27*2, 27), generator=g, requires_grad=True)
    #tr_losses, val_losses= [], []
    
    # gradient descent
    for k in range(n_iter):

        # forward pass
        xenc = F.one_hot(xs_tr, num_classes=27).float()       
        logits = xenc.view(-1, 27*2) @ W                 
        counts = logits.exp()                             
        probs = counts / counts.sum(1, keepdim=True)     
        tr_loss = -probs[torch.arange(num_tr), ys_tr].log().mean() + C*(W**2).mean() 
        #print(loss.item())
        #tr_losses.append(tr_loss)

        # Zero grad & backward pass
        W.grad = None 
        tr_loss.backward()

        # update
        W.data += -lr * W.grad     

    for w in val:
        chs = ['.'] + list(w) + ['.']
        for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
            ix1 = stoi[ch1]
            ix2 = stoi[ch2]
            ix3 = stoi[ch3]
            xs_val.append((ix1, ix2))
            ys_val.append(ix3)

    xs_val2 = torch.tensor(xs_val) 
    ys_val2 = torch.tensor(ys_val)
    num_val = ys_val2.nelement()

    xenc_val = F.one_hot(xs_val2, num_classes=27).float()      
    logits = xenc_val.view(-1, 27*2) @ W                   
    counts = logits.exp()                             
    probs = counts / counts.sum(1, keepdim=True)     
    val_loss = -probs[torch.arange(num_val), ys_val2].log().mean()

    print(f'For regularization parameter = {C:.4f} ---> train loss: {tr_loss.item():.4f},\
    validation loss: {val_loss.item():.4f}')
    
    _tr_loss = np.round(tr_loss.item(), decimals=4)
    _val_loss = np.round(val_loss.item(), decimals=4)
    _C = np.round(C, decimals=4)
    loss_dict[_C] = [_tr_loss, _val_loss]

  8%|▊         | 1/12 [00:08<01:30,  8.21s/it]

For regularization parameter = 0.0010 ---> train loss: 2.2633,    validation loss: 2.2658

 17%|█▋        | 2/12 [00:16<01:21,  8.19s/it]

For regularization parameter = 0.0018 ---> train loss: 2.2632,    validation loss: 2.2667

 25%|██▌       | 3/12 [00:25<01:16,  8.54s/it]

For regularization parameter = 0.0031 ---> train loss: 2.2656,    validation loss: 2.2654

 33%|███▎      | 4/12 [00:35<01:10,  8.78s/it]

For regularization parameter = 0.0054 ---> train loss: 2.2686,    validation loss: 2.2677

 42%|████▏     | 5/12 [00:43<01:00,  8.69s/it]

For regularization parameter = 0.0096 ---> train loss: 2.2722,    validation loss: 2.2664

loss_dict

{0.001: [2.2633, 2.2658],
 0.0018: [2.2632, 2.2667],
 0.0031: [2.2656, 2.2654],
 0.0054: [2.2686, 2.2677],
 0.0096: [2.2722, 2.2664],
 0.0169: [2.2779, 2.2674],
 0.0297: [2.2877, 2.2672],
 0.0522: [2.3015, 2.2689],
 0.0918: [2.3235, 2.2745],
 0.1615: [2.3535, 2.286],
 0.2842: [2.3949, 2.3056],
 0.5: [2.4526, 2.3375]}

plt.title(f"Effect of Regularization/Smoothing on Trigram Model")
plt.plot(list(loss_dict.keys()), [train_loss for train_loss, _ in loss_dict.values()], "o--", label="train")
plt.plot(list(loss_dict.keys()), [val_loss for _, val_loss in loss_dict.values()], "o--", label="val")
plt.xlabel("Regularization Strength, C")
plt.ylabel("Loss")
plt.grid()
plt.legend()
plt.plot()
#plt.show()

[]

#sorted_dicts_by_train_loss = dict(sorted(loss_dict.items(), key=lambda item: min(item[1])))
#sorted_dicts_by_train_loss = dict(sorted(loss_dict.items(), key=lambda item: min(item[1][1])))
#{key:t[key] for idx, key in enumerate(t) if idx<10}
for k, v in loss_dict.items():
    print(k,v)

0.001 [2.2633, 2.2658]
0.0018 [2.2632, 2.2667]
0.0031 [2.2656, 2.2654]
0.0054 [2.2686, 2.2677]
0.0096 [2.2722, 2.2664]
0.0169 [2.2779, 2.2674]
0.0297 [2.2877, 2.2672]
0.0522 [2.3015, 2.2689]
0.0918 [2.3235, 2.2745]
0.1615 [2.3535, 2.286]
0.2842 [2.3949, 2.3056]
0.5 [2.4526, 2.3375]

train_loss_dict = {k:v[0] for k,v in loss_dict.items() }
val_loss_dict = {k:v[1] for k,v in loss_dict.items() }

#train_loss_dict, val_loss_dict

min(train_loss_dict.values()), min(train_loss_dict, key=train_loss_dict.get)

(2.2632, 0.0018)

min(val_loss_dict.values()), min(val_loss_dict, key=val_loss_dict.get)

(2.2654, 0.0031)

C_test = 0
logits_test = xenc_test.view(-1, 27*2) @ W                   
counts_test = logits_test.exp()                             
probs_test = counts_test / counts_test.sum(1, keepdim=True)     
loss_test2 = -probs_test[torch.arange(num_test), ys_test2].log().mean() + C_test*(W**2).mean()
print(f'The test loss after applying a C=0 regularization parameter: {loss_test2.item():.4f}')

The test loss after applying a C=0 regularization parameter: 2.3323

xs, ys = [], []

for w in words:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        #print(ch1, ch2, ch3)
        xs.append((ix1, ix2))
        ys.append(ix3)
        
xs = torch.tensor(xs) 
ys = torch.tensor(ys)
num = ys.nelement()

x_enc = F.one_hot(xs, num_classes=27).float()
x_enc.shape

torch.Size([196113, 2, 27])

x_enc[100]

tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 1., 0.]])

x100_enc_i = torch.tensor([x_enc[100].tolist()])
x100_enc_i.argmax(), x100_enc_i.shape, x100_enc_i.view(-1, 27*2).shape

(tensor(11), torch.Size([1, 2, 27]), torch.Size([1, 54]))

x100_enc_i.view(-1, 27*2)

tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.]])

x100_enc_i.view(-1, 27*2) @ W

tensor([[ 1.3609,  2.0429, -0.6675, -0.4255,  0.0137,  0.3082, -0.7364, -0.7811,
         -0.7451,  0.4934, -0.7190, -0.6170,  1.6086,  0.1379,  1.8832,  0.2683,
         -0.6588, -0.7640,  0.5670,  0.4279, -0.1699, -0.2077, -0.4730, -0.7689,
         -0.7119, -0.0456, -0.4161]], grad_fn=<MmBackward0>)

xW_100_enc_i = W[xs[100, 0]] + W[xs[100, 1] + 27]
xW_100_enc_i

tensor([ 1.3609,  2.0429, -0.6675, -0.4255,  0.0137,  0.3082, -0.7364, -0.7811,
        -0.7451,  0.4934, -0.7190, -0.6170,  1.6086,  0.1379,  1.8832,  0.2683,
        -0.6588, -0.7640,  0.5670,  0.4279, -0.1699, -0.2077, -0.4730, -0.7689,
        -0.7119, -0.0456, -0.4161], grad_fn=<AddBackward0>)

(x100_enc_i.view(-1, 27*2) @ W).shape, (W[xs[100, 0]] + W[xs[100, 1] + 27]).shape

(torch.Size([1, 27]), torch.Size([27]))

(x_enc.view(-1, 27*2) @ W).shape, (W[xs[:, 0]] + W[xs[:, 1] + 27]).shape

(torch.Size([196113, 27]), torch.Size([196113, 27]))

xW_100_enc_i == x100_enc_i.view(-1, 27*2) @ W

tensor([[True, True, True, True, True, True, True, True, True, True, True, True,
         True, True, True, True, True, True, True, True, True, True, True, True,
         True, True, True]])

logits = x_enc.view(-1, 2*27) @ W
counts = logits.exp()
probs = counts / counts.sum(1, keepdim=True)
loss = -probs[torch.arange(x_enc.shape[0]), ys].log().mean()
print(f"Average Negative Log Likelihood Loss: {loss.item():.4f}")

Average Negative Log Likelihood Loss: 2.3353

loss_ce = F.cross_entropy(logits, ys)
print(f"Cross Entropy Loss: {loss_ce.item():.4f}")

Cross Entropy Loss: 2.3354

xs, ys = [], []

for w in words:
    chs = ['.'] + list(w) + ['.']
    for ch1, ch2, ch3 in zip(chs, chs[1:], chs[2:]):
        ix1 = stoi[ch1]
        ix2 = stoi[ch2]
        ix3 = stoi[ch3]
        #print(ch1, ch2, ch3)
        xs.append([ix1, ix2])
        ys.append(ix3)

xs = torch.tensor(xs) 
ys = torch.tensor(ys)

class MLP(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = torch.nn.Linear(2*27, 27) # first fully connected layer of MLP
        torch.nn.init.normal_(self.fc1.weight, mean=0, std=1)
        #torch.nn.init.xavier_uniform(self.fc1.weight)

    def forward(self, x):
        logits = W[x[:, 0]] + W[x[:, 1] + 27]
        return logits

C = 0
model = MLP()
# gradient descent
for k in range(250):

    # forward pass
    logits = model(xs)
    loss = F.cross_entropy(logits, ys) 
    loss += C * (model.fc1.weight ** 2).mean() # added regularization term
    
#     if k % 10 == 0:
#         print(f'After {k} iterations, the loss is {loss.item():.4f}')

    # backward pass
    model.fc1.zero_grad() # set to zero the gradient (reset gradients of the layer)
    loss.backward()

    # update
    model.fc1.weight.data += -50 * model.fc1.weight.grad
print(f"After {k} iterations, the loss is {loss.item():.4f}")

After 249 iterations, the loss is 2.3354

makemore: bigram exercises¶

Exercises¶

1. Trigram Language Model¶

a. Counting Approach¶

detour to figure out how to visualize trigrams¶

function for making n-grams for visualization input¶

back to counting approach¶

combining it all together¶

b. Neural Network Approach¶

combining it all together¶

2. Validate Trigram Language Model¶

Evaluate model on validation set¶

Evaluate model on test set¶

3. Tune Regularization/Smoothing with Validation Set¶

4. Indexing into Rows of `W` (Discard One-Hot Encoding)¶

5. Indexing into Rows of `W` (Discard One-Hot Encoding)¶

6. meta-exercise: PyTorch Base Class for all Neural Network Modules¶

	Input	Target	Count
0	.e	m	288
1	em	m	100
2	mm	a	72
3	ma	.	174
4	.o	l	104

	Count
Input
an	5438
.a	4410
ar	3264
el	3248
ri	3033
...	...
sq	1
jb	1
wz	1
cp	1
mf	1

makemore: bigram exercises¶

Exercises¶

1. Trigram Language Model¶

a. Counting Approach¶

detour to figure out how to visualize trigrams¶

function for making n-grams for visualization input¶

back to counting approach¶

combining it all together¶

b. Neural Network Approach¶

combining it all together¶

2. Validate Trigram Language Model¶

Evaluate model on validation set¶

Evaluate model on test set¶

3. Tune Regularization/Smoothing with Validation Set¶

4. Indexing into Rows of W (Discard One-Hot Encoding)¶

5. Indexing into Rows of W (Discard One-Hot Encoding)¶

6. meta-exercise: PyTorch Base Class for all Neural Network Modules¶

4. Indexing into Rows of `W` (Discard One-Hot Encoding)¶

5. Indexing into Rows of `W` (Discard One-Hot Encoding)¶