Multi Layer Perceptron (MLP) Example with the Cognitive Toolkit (CNTK)¶
In [1]:
# initialize the environment
import cntk as C
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np
np.random.seed(123)
In [2]:
# visualize the data
X = np.concatenate([ np.random.multivariate_normal([-3, -3], [[0.1, 0], [0, 0.1]], 100),
np.random.multivariate_normal([ 3, 3], [[0.1, 0], [0, 0.1]], 100),
np.random.multivariate_normal([-3, 3], [[0.1, 0], [0, 0.1]], 100),
np.random.multivariate_normal([ 3, -3], [[0.1, 0], [0, 0.1]], 100) ]).astype(np.float32)
class_vector = np.concatenate([[0] * 200, [1] * 200]).reshape(400, 1)
plt.scatter(X[:,0], X[:,1], c = class_vector, marker = 'x')
plt.show()
In [3]:
# declare the input and output variables
input_feature_count = 2
output_class_count = 2
input = C.ops.input_variable(input_feature_count)
label = C.ops.input_variable(output_class_count)
In [4]:
# declare the model
hidden_feature_count = 2
model = C.layers.Sequential([ C.layers.Dense(hidden_feature_count, init = C.layers.glorot_uniform(), activation = C.layers.sigmoid, name = "hidden_layer"),
C.layers.Dense(output_class_count, init = C.layers.glorot_uniform(), activation = None, name = "output_layer") ])(input)
loss = C.losses.cross_entropy_with_softmax(model, label)
eval = C.metrics.classification_error(model, label)
learning_rate = 0.5
lr_schedule = C.learners.learning_rate_schedule(learning_rate, C.UnitType.minibatch)
learner = C.learners.sgd(model.parameters, lr_schedule)
trainer = C.train.trainer.Trainer(model, (loss, eval), [learner])
In [5]:
# train the model
for i in range(100):
X = np.concatenate([ np.random.multivariate_normal([-3, -3], [[0.1, 0], [0, 0.1]], 100),
np.random.multivariate_normal([ 3, 3], [[0.1, 0], [0, 0.1]], 100),
np.random.multivariate_normal([-3, 3], [[0.1, 0], [0, 0.1]], 100),
np.random.multivariate_normal([ 3, -3], [[0.1, 0], [0, 0.1]], 100) ]).astype(np.float32)
class_vector = np.concatenate([[0] * 200, [1] * 200]).reshape(400, 1)
class_indicators = [class_vector == class_index for class_index in range(output_class_count)]
Y = np.asarray(np.hstack(class_indicators)).astype(np.float32)
trainer.train_minibatch({input : X, label : Y})
print ("Minibatch: {0}, Loss: {1:.4f}, Error: {2:.2f}".format(i, trainer.previous_minibatch_loss_average, trainer.previous_minibatch_evaluation_average))
In [6]:
# view the weights and biases
print(model.hidden_layer.W.value)
print(model.hidden_layer.b.value)
print(model.output_layer.W.value)
print(model.output_layer.b.value)
In [7]:
# visualize the hidden features
def sigmoid(x):
return (1.0 / (1.0 + np.exp(-x)))
z1 = sigmoid(np.dot(X, model.hidden_layer.W.value[:,0]) + model.hidden_layer.b.value[0])
z2 = sigmoid(np.dot(X, model.hidden_layer.W.value[:,1]) + model.hidden_layer.b.value[1])
plt.scatter(z1, z2, c = class_vector, marker = 'x')
plt.show()
In [8]:
# check out the mappings: from input features to hidden features
print([np.mean(z1[0:100]), np.mean(z2[0:100])])
print([np.mean(z1[100:200]), np.mean(z2[100:200])])
print([np.mean(z1[200:300]), np.mean(z2[200:300])])
print([np.mean(z1[300:400]), np.mean(z2[300:400])])
In [ ]: