Backpropagation: Manual Example

In [1]:
# define activation function
import numpy as np
def sigmoid(x):
    return(1.0 / (1.0 + np.exp(-x)))
In [2]:
# observe an input vector and a label
bias =  1    # adding a bias
input1 =  1
input2 = -1
target = 1
In [3]:
# initialize weights to small random values
weight_bias_hidden1 = 0.01
weight_input1_hidden1 = 0.02
weight_input2_hidden1 = 0.03

weight_bias_hidden2 = 0.04
weight_input1_hidden2 = 0.05
weight_input2_hidden2 = 0.06

weight_bias_output = 0.07
weight_hidden1_output = 0.08
weight_hidden2_output = 0.09
from IPython.display import Image
Image("http://cross-entropy.net/PyData/Simple_MLP_Example.png")
Out[3]:
In [4]:
# forward propagation of input features through hidden layer
sum_hidden1 = weight_bias_hidden1 * bias + weight_input1_hidden1 * input1 + weight_input2_hidden1 * input2
activation_hidden1 = sigmoid(sum_hidden1)
sum_hidden2 = weight_bias_hidden2 * bias + weight_input1_hidden2 * input1 + weight_input2_hidden2 * input2
activation_hidden2 = sigmoid(sum_hidden2)
print(activation_hidden1)
print(activation_hidden2)

0.5
0.5074994375506203
In [5]:
# forward propagation of hidden features through output layer
sum_output = weight_bias_output * bias + weight_hidden1_output * activation_hidden1 + weight_hidden2_output * activation_hidden2
activation_output = sigmoid(sum_output)
print(activation_output)

0.5388403287327925
In [6]:
# back propagation of error: updating the output layer weights
learningRate = 0.1

derivative_loss = (activation_output - target) / (activation_output * (1 - activation_output))
derivative_output = activation_output * (1 - activation_output)

previous_weight_hidden1_output = weight_hidden1_output
previous_weight_hidden2_output = weight_hidden2_output

weight_bias_output -= learningRate * bias * derivative_loss * derivative_output
weight_hidden1_output -= learningRate * activation_hidden1 * derivative_loss * derivative_output
weight_hidden2_output -= learningRate * activation_hidden2 * derivative_loss * derivative_output
print(weight_bias_output)
print(weight_hidden1_output)
print(weight_hidden2_output)

0.11611596712672076
0.10305798356336038
0.11340382737891368
In [7]:
# back propagation of error: updating the hidden layer weights
derivative_hidden1 = activation_hidden1 * (1 - activation_hidden1)
derivative_hidden2 = activation_hidden2 * (1 - activation_hidden2)

weight_bias_hidden1 -= learningRate * bias * derivative_hidden1 * previous_weight_hidden1_output * derivative_loss * derivative_output
weight_input1_hidden1 -= learningRate * input1 * derivative_hidden1 * previous_weight_hidden1_output * derivative_loss * derivative_output
weight_input2_hidden1 -= learningRate * input2 * derivative_hidden1 * previous_weight_hidden1_output * derivative_loss * derivative_output

weight_bias_hidden2 -= learningRate * bias * derivative_hidden2 * previous_weight_hidden2_output * derivative_loss * derivative_output
weight_input1_hidden2 -= learningRate * input1 * derivative_hidden2 * previous_weight_hidden2_output * derivative_loss * derivative_output
weight_input2_hidden2 -= learningRate * input2 * derivative_hidden2 * previous_weight_hidden2_output * derivative_loss * derivative_output

print(weight_bias_hidden1)
print(weight_input1_hidden1)
print(weight_input2_hidden1)

print(weight_bias_hidden2)
print(weight_input1_hidden2)
print(weight_input2_hidden2)

0.010922319342534416
0.020922319342534416
0.029077680657465584
0.041037375833282486
0.05103737583328249
0.05896262416671751
In [8]:
# check prediction with updated weights: it's closer to the target ... it's learning!
sum_hidden1 = weight_bias_hidden1 * bias + weight_input1_hidden1 * input1 + weight_input2_hidden1 * input2
activation_hidden1 = sigmoid(sum_hidden1)
sum_hidden2 = weight_bias_hidden2 * bias + weight_input1_hidden2 * input1 + weight_input2_hidden2 * input2
activation_hidden2 = sigmoid(sum_hidden2)
sum_output = weight_bias_output * bias + weight_hidden1_output * activation_hidden1 + weight_hidden2_output * activation_hidden2
activation_output = sigmoid(sum_output)
print(activation_hidden1)
print(activation_hidden2)
print(activation_output)

0.5006917390655681
0.508277275612741
0.5561019783297847
In [ ]: