Canadian Institute for Advanced Research 10 Class (CIFAR10) Data: Convolutional Neural Network (CNN)

In [1]:
'''Train a simple deep CNN on the CIFAR10 small images dataset.

GPU run command with Theano backend (with TensorFlow, the GPU is automatically used):
    THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatx=float32 python cifar10_cnn.py

It gets down to 0.65 test logloss in 25 epochs, and down to 0.55 after 50 epochs.
(it's still underfitting at that point, though).
'''

from __future__ import print_function
import keras
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D

batch_size = 32
num_classes = 10
epochs = 30
data_augmentation = True

Using CNTK backend
In [2]:
# The data, shuffled and split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

# Convert class vectors to binary class matrices.
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)

x_train shape: (50000, 32, 32, 3)
50000 train samples
10000 test samples
In [3]:
model = Sequential()

model.add(Conv2D(32, (3, 3), padding='same',
                 input_shape=x_train.shape[1:]))
model.add(Activation('relu'))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Conv2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.summary()
# 32 * ( 3 * 3 * 3 + 1) =       896
# 32 * (32 * 3 * 3 + 1) =     9,248
# 64 * (32 * 3 * 3 + 1) =    18,496
# 64 * (64 * 3 * 3 + 1) =    36,928
# 64 * 6 * 6            =     2,304
# (2304 + 1) * 512      = 1,180,160
# (512 + 1) * 10        =     5,130

# initiate RMSprop optimizer
opt = keras.optimizers.rmsprop(lr=0.0001, decay=1e-6)

# Let's train the model using RMSprop
model.compile(loss='categorical_crossentropy',
              optimizer=opt,
              metrics=['accuracy'])

_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_1 (Conv2D)            (None, 32, 32, 32)        896       
_________________________________________________________________
activation_1 (Activation)    (None, 32, 32, 32)        0         
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 30, 30, 32)        9248      
_________________________________________________________________
activation_2 (Activation)    (None, 30, 30, 32)        0         
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 15, 15, 32)        0         
_________________________________________________________________
dropout_1 (Dropout)          (None, 15, 15, 32)        0         
_________________________________________________________________
conv2d_3 (Conv2D)            (None, 15, 15, 64)        18496     
_________________________________________________________________
activation_3 (Activation)    (None, 15, 15, 64)        0         
_________________________________________________________________
conv2d_4 (Conv2D)            (None, 13, 13, 64)        36928     
_________________________________________________________________
activation_4 (Activation)    (None, 13, 13, 64)        0         
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 6, 6, 64)          0         
_________________________________________________________________
dropout_2 (Dropout)          (None, 6, 6, 64)          0         
_________________________________________________________________
flatten_1 (Flatten)          (None, 2304)              0         
_________________________________________________________________
dense_1 (Dense)              (None, 512)               1180160   
_________________________________________________________________
activation_5 (Activation)    (None, 512)               0         
_________________________________________________________________
dropout_3 (Dropout)          (None, 512)               0         
_________________________________________________________________
dense_2 (Dense)              (None, 10)                5130      
_________________________________________________________________
activation_6 (Activation)    (None, 10)                0         
=================================================================
Total params: 1,250,858
Trainable params: 1,250,858
Non-trainable params: 0
_________________________________________________________________
In [4]:
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
In [5]:
if not data_augmentation:
    print('Not using data augmentation.')
    model.fit(x_train, y_train,
              batch_size=batch_size,
              epochs=epochs,
              validation_data=(x_test, y_test),
              shuffle=True)
else:
    print('Using real-time data augmentation.')
    # This will do preprocessing and realtime data augmentation:
    datagen = ImageDataGenerator(
        featurewise_center=False,  # set input mean to 0 over the dataset
        samplewise_center=False,  # set each sample mean to 0
        featurewise_std_normalization=False,  # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        zca_whitening=False,  # apply ZCA whitening
        rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)
        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
        horizontal_flip=True,  # randomly flip images
        vertical_flip=False)  # randomly flip images

    # Compute quantities required for feature-wise normalization
    # (std, mean, and principal components if ZCA whitening is applied).
    datagen.fit(x_train)

    # Fit the model on the batches generated by datagen.flow().
    model.fit_generator(datagen.flow(x_train, y_train,
                                     batch_size=batch_size),
                        steps_per_epoch=x_train.shape[0] // batch_size,
                        epochs=epochs,
                        validation_data=(x_test, y_test))

Using real-time data augmentation.
Epoch 1/30
   4/1562 [..............................] - ETA: 203s - loss: 2.3190 - acc: 0.0703
/home/dadebarr/anaconda3/lib/python3.5/site-packages/cntk/core.py:351: UserWarning: your data is of type "float64", but your input variable (uid "Input178") expects "<class 'numpy.float32'>". Please convert your data beforehand to speed up training.
  (sample.dtype, var.uid, str(var.dtype)))

1560/1562 [============================>.] - ETA: 0s - loss: 1.8659 - acc: 0.3104
/home/dadebarr/anaconda3/lib/python3.5/site-packages/cntk/core.py:74: RuntimeWarning: data is not C contiguous; rearrange your data/computation to avoid costly data conversions
  RuntimeWarning)

1562/1562 [==============================] - 46s - loss: 1.8659 - acc: 0.3104 - val_loss: 1.5824 - val_acc: 0.4284
Epoch 2/30
1562/1562 [==============================] - 41s - loss: 1.5716 - acc: 0.4246 - val_loss: 1.3884 - val_acc: 0.4917
Epoch 3/30
1562/1562 [==============================] - 41s - loss: 1.4387 - acc: 0.4790 - val_loss: 1.2952 - val_acc: 0.5307
Epoch 4/30
1562/1562 [==============================] - 41s - loss: 1.3511 - acc: 0.5153 - val_loss: 1.2348 - val_acc: 0.5694
Epoch 5/30
1562/1562 [==============================] - 41s - loss: 1.2814 - acc: 0.5432 - val_loss: 1.1131 - val_acc: 0.6056
Epoch 6/30
1562/1562 [==============================] - 39s - loss: 1.2213 - acc: 0.5665 - val_loss: 1.0614 - val_acc: 0.6267
Epoch 7/30
1562/1562 [==============================] - 42s - loss: 1.1654 - acc: 0.5866 - val_loss: 1.0090 - val_acc: 0.6484
Epoch 8/30
1562/1562 [==============================] - 40s - loss: 1.1222 - acc: 0.6029 - val_loss: 0.9579 - val_acc: 0.6722
Epoch 9/30
1562/1562 [==============================] - 41s - loss: 1.0802 - acc: 0.6160 - val_loss: 1.0059 - val_acc: 0.6475
Epoch 10/30
1562/1562 [==============================] - 41s - loss: 1.0485 - acc: 0.6325 - val_loss: 0.9175 - val_acc: 0.6809
Epoch 11/30
1562/1562 [==============================] - 41s - loss: 1.0142 - acc: 0.6438 - val_loss: 0.8739 - val_acc: 0.6995
Epoch 12/30
1562/1562 [==============================] - 40s - loss: 0.9916 - acc: 0.6540 - val_loss: 0.8548 - val_acc: 0.7029
Epoch 13/30
1562/1562 [==============================] - 41s - loss: 0.9684 - acc: 0.6612 - val_loss: 0.8467 - val_acc: 0.7131
Epoch 14/30
1562/1562 [==============================] - 41s - loss: 0.9482 - acc: 0.6687 - val_loss: 0.8151 - val_acc: 0.7197
Epoch 15/30
1562/1562 [==============================] - 41s - loss: 0.9296 - acc: 0.6739 - val_loss: 0.8027 - val_acc: 0.7246
Epoch 16/30
1562/1562 [==============================] - 41s - loss: 0.9160 - acc: 0.6783 - val_loss: 0.8000 - val_acc: 0.7219
Epoch 17/30
1562/1562 [==============================] - 41s - loss: 0.8977 - acc: 0.6861 - val_loss: 0.7794 - val_acc: 0.7291
Epoch 18/30
1562/1562 [==============================] - 41s - loss: 0.8863 - acc: 0.6916 - val_loss: 0.7581 - val_acc: 0.7385
Epoch 19/30
1562/1562 [==============================] - 41s - loss: 0.8756 - acc: 0.6957 - val_loss: 0.7573 - val_acc: 0.7394
Epoch 20/30
1562/1562 [==============================] - 41s - loss: 0.8631 - acc: 0.7006 - val_loss: 0.7423 - val_acc: 0.7448
Epoch 21/30
1562/1562 [==============================] - 40s - loss: 0.8560 - acc: 0.7019 - val_loss: 0.7315 - val_acc: 0.7477
Epoch 22/30
1562/1562 [==============================] - 41s - loss: 0.8432 - acc: 0.7078 - val_loss: 0.7212 - val_acc: 0.7486
Epoch 23/30
1562/1562 [==============================] - 41s - loss: 0.8373 - acc: 0.7101 - val_loss: 0.7516 - val_acc: 0.7387
Epoch 24/30
1562/1562 [==============================] - 39s - loss: 0.8365 - acc: 0.7090 - val_loss: 0.7377 - val_acc: 0.7456
Epoch 25/30
1562/1562 [==============================] - 42s - loss: 0.8286 - acc: 0.7177 - val_loss: 0.7710 - val_acc: 0.7407
Epoch 26/30
1562/1562 [==============================] - 41s - loss: 0.8247 - acc: 0.7163 - val_loss: 0.6911 - val_acc: 0.7627
Epoch 27/30
1562/1562 [==============================] - 40s - loss: 0.8195 - acc: 0.7195 - val_loss: 0.7022 - val_acc: 0.7566
Epoch 28/30
1562/1562 [==============================] - 41s - loss: 0.8138 - acc: 0.7211 - val_loss: 0.6865 - val_acc: 0.7615
Epoch 29/30
1562/1562 [==============================] - 41s - loss: 0.8143 - acc: 0.7211 - val_loss: 0.7287 - val_acc: 0.7505
Epoch 30/30
1562/1562 [==============================] - 41s - loss: 0.8079 - acc: 0.7215 - val_loss: 0.6975 - val_acc: 0.7570
In [ ]: