1. Add descrioptinsp
  2. add drop out
  3. add normalization of data
In [2]:
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from tensorflow.contrib.layers import flatten


EPOCHS = 10
BATCH_SIZE = 50
In [13]:
def conv2d(input, weight_tuple, strides_list, depth):
    # Filter (weights and bias)
    F_W = tf.Variable(tf.truncated_normal(weight_tuple))
    F_b = tf.Variable(tf.zeros(depth))
    strides = strides_list
    padding = 'VALID'
    return tf.nn.conv2d(input, F_W, strides, padding) + F_b

def max_pool(input, ksize, strides):
    padding = 'VALID'
    return tf.nn.max_pool(input, ksize, strides, padding)
In [14]:
# LeNet architecture:
# INPUT -> CONV -> ACT -> POOL -> CONV -> ACT -> POOL -> FLATTEN -> FC -> ACT -> FC
#
# Don't worry about anything else in the file too much, all you have to do is
# create the LeNet and return the result of the last fully connected layer.
def LeNet(x):
    # Reshape from 2D to 4D. This prepares the data for
    # convolutional and pooling layers.
    x = tf.reshape(x, (-1, 28, 28, 1))
    # Pad 0s to 32x32. Centers the digit further.
    # Add 2 rows/columns on each side for height and width dimensions.
    x = tf.pad(x, [[0, 0], [2, 2], [2, 2], [0, 0]], mode="CONSTANT")
    
    x = conv2d(x, (5,5,1,6), [1,1,1,1], 6)
    x = tf.nn.relu(x)
    x = max_pool(x, [1,2,2,1], [1,2,2,1])
    
    x = conv2d(x, (2,2,6,16), [1,2,2,1], 16)
    x = tf.nn.relu(x)
    x = max_pool(x, [1,2,2,1], [1,2,2,1])
    
    # Flatten
    fc1 = flatten(x)
    # (5 * 5 * 16, 120)
    fc1_shape = (fc1.get_shape().as_list()[-1], 120)
    
    fc1_W = tf.Variable(tf.truncated_normal(shape=(fc1_shape)))
    fc1_b = tf.Variable(tf.zeros(120))
    fc1 = tf.matmul(fc1, fc1_W) + fc1_b
    fc1 = tf.nn.relu(fc1)
    
    fc2_W = tf.Variable(tf.truncated_normal(shape=(120, 10)))
    fc2_b = tf.Variable(tf.zeros(10))
    return tf.matmul(fc1, fc2_W) + fc2_b
In [18]:
# MNIST consists of 28x28x1, grayscale images
x = tf.placeholder(tf.float32, (None, 784))
# Classify over 10 digits 0-9
y = tf.placeholder(tf.float32, (None, 10))
fc2 = LeNet(x)

loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(fc2, y))
opt = tf.train.AdamOptimizer()
train_op = opt.minimize(loss_op)
correct_prediction = tf.equal(tf.argmax(fc2, 1), tf.argmax(y, 1))
accuracy_op = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))


def eval_data(dataset):
    """
    Given a dataset as input returns the loss and accuracy.
    """
    # If dataset.num_examples is not divisible by BATCH_SIZE
    # the remainder will be discarded.
    # Ex: If BATCH_SIZE is 64 and training set has 55000 examples
    # steps_per_epoch = 55000 // 64 = 859
    # num_examples = 859 * 64 = 54976
    #
    # So in that case we go over 54976 examples instead of 55000.
    steps_per_epoch = dataset.num_examples // BATCH_SIZE
    num_examples = steps_per_epoch * BATCH_SIZE
    total_acc, total_loss = 0, 0
    sess = tf.get_default_session()
    for step in range(steps_per_epoch):
        batch_x, batch_y = dataset.next_batch(BATCH_SIZE)
        loss, acc = sess.run([loss_op, accuracy_op], feed_dict={x: batch_x, y: batch_y})
        total_acc += (acc * batch_x.shape[0])
        total_loss += (loss * batch_x.shape[0])
    return total_loss/num_examples, total_acc/num_examples


if __name__ == '__main__':
    # Load data
    mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
        steps_per_epoch = mnist.train.num_examples // BATCH_SIZE
        num_examples = steps_per_epoch * BATCH_SIZE

        # Train model
        for i in range(EPOCHS):
            for step in range(steps_per_epoch):
                batch_x, batch_y = mnist.train.next_batch(BATCH_SIZE)
                loss = sess.run(train_op, feed_dict={x: batch_x, y: batch_y})

            val_loss, val_acc = eval_data(mnist.validation)
            print("EPOCH {} ...".format(i+1))
            print("Validation loss = {:.3f}".format(val_loss))
            print("Validation accuracy = {:.3f}".format(val_acc))
            print()

        # Evaluate on the test data
        test_loss, test_acc = eval_data(mnist.test)
        print("Test loss = {:.3f}".format(test_loss))
        print("Test accuracy = {:.3f}".format(test_acc))


 
Extracting MNIST_data/train-images-idx3-ubyte.gz
Extracting MNIST_data/train-labels-idx1-ubyte.gz
Extracting MNIST_data/t10k-images-idx3-ubyte.gz
Extracting MNIST_data/t10k-labels-idx1-ubyte.gz
EPOCH 1 ...
Validation loss = 16.058
Validation accuracy = 0.739

EPOCH 2 ...
Validation loss = 7.659
Validation accuracy = 0.828

EPOCH 3 ...
Validation loss = 4.818
Validation accuracy = 0.871

EPOCH 4 ...
Validation loss = 3.415
Validation accuracy = 0.885

EPOCH 5 ...
Validation loss = 2.489
Validation accuracy = 0.895

EPOCH 6 ...
Validation loss = 1.897
Validation accuracy = 0.908

EPOCH 7 ...
Validation loss = 1.512
Validation accuracy = 0.911

EPOCH 8 ...
Validation loss = 1.170
Validation accuracy = 0.921

EPOCH 9 ...
Validation loss = 0.991
Validation accuracy = 0.921

EPOCH 10 ...
Validation loss = 0.827
Validation accuracy = 0.926

Test loss = 0.832
Test accuracy = 0.920
In [ ]: