Very important

Before starting to read and copy/paste parts of the code we should ask our selves the next questions
  1. How many layers do I have in my NN?
  2. What is the type of each layer?
  3. What is the rank and shape of the input and output of each layer?
    Please, apply basic matricial operations.

Build and Compare Different Neural Networks models

We will focus on the specific problem of the classification of handwritten digits.

  1. Build a One Layer Neural Network classifier

    This simple NN is build using one hidden softmax layer. This layer will have 10 neurons and uses softmax function as tansfer function.
    Layer Number of neurons Ativation funtion
    One 10 softmax

    import tensorflow as tf from tensorflow.examples.tutorials.mnist import input_data import numpy as np batch_size = 100 learning_rate = 0.5 training_epochs = 10 mnist = input_data.read_data_sets("data", one_hot=True) #First, we create the necessary tensors
    #The set of input images of size 28x28 pixels each one> X = tf.placeholder(tf.float32, [None, 784], name="input") #The output of of the network, a tensor of 10 elements> Y_ = tf.placeholder(tf.float32, [None, 10]) W = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) XX = tf.reshape(X, [-1, 784]) #Then, we define our flowgraph
    Y = tf.nn.softmax(tf.matmul(XX, W) + b, name="output") cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y_, logits=Y)) correct_prediction = tf.equal(tf.argmax(Y, 1), tf.argmax(Y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) train_step = tf.train.GradientDescentOptimizer(0.005).minimize(cross_entropy) #Finally, we crearte a session to run our flowgraph
    with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for epoch in range(training_epochs): batch_count = int(mnist.train.num_examples/batch_size) for i in range(batch_count): batch_x, batch_y = mnist.train.next_batch(batch_size) sess.run([train_step],feed_dict={X: batch_x,Y_: batch_y}) print("Epoch: ", epoch) print("Accuracy: ", accuracy.eval(feed_dict={X: mnist.test.images, Y_: mnist.test.labels})) print("done")

  2. Build Four layers Neural Network classifier

    Layer Number of neurons Ativation funtion
    First 200 ReLU
    Second 100 ReLU
    Third 60 ReLU
    Fourth 30 ReLU
    Fifth 10 softmax

    L = 200 M = 100 N = 60 O = 30 W1 = tf.Variable(tf.truncated_normal([784, L], stddev=0.1)) B1 = tf.Variable(tf.ones([L])/10) W2 = tf.Variable(tf.truncated_normal([L, M], stddev=0.1)) B2 = tf.Variable(tf.ones([M])/10) W3 = tf.Variable(tf.truncated_normal([M, N], stddev=0.1)) B3 = tf.Variable(tf.ones([N])/10) W4 = tf.Variable(tf.truncated_normal([N, O], stddev=0.1)) B4 = tf.Variable(tf.ones([O])/10) W5 = tf.Variable(tf.truncated_normal([O, 10], stddev=0.1)) B5 = tf.Variable(tf.zeros([10])) Y1 = tf.nn.relu(tf.matmul(XX, W1) + B1) Y2 = tf.nn.relu(tf.matmul(Y1, W2) + B2) Y3 = tf.nn.relu(tf.matmul(Y2, W3) + B3) Y4 = tf.nn.relu(tf.matmul(Y3, W4) + B4) Ylogits = tf.matmul(Y4, W5) + B5 Y = tf.nn.softmax(Ylogits) cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=Y_, logits=Y))*100

    The accuracy of this model can be improved by including the droput optimization after each ReLU layer and by tuning the network parameters.

  3. Build a Five layer Convolutional Neural Network classifier


    import tensorflow as tf import numpy as np from tensorflow.examples.tutorials.mnist import input_data batch_size = 128 test_size = 256 img_size = 28 num_classes = 10 def init_weights(shape): return tf.Variable(tf.random_normal(shape, stddev=0.01)) def model(X, w, w2, w3, w4, w_o, p_keep_conv, p_keep_hidden): conv1 = tf.nn.conv2d(X, w,strides=[1, 1, 1, 1],padding='SAME') conv1_a = tf.nn.relu(conv1) conv1 = tf.nn.max_pool(conv1_a, ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1],padding='SAME') conv1 = tf.nn.dropout(conv1, p_keep_conv) conv2 = tf.nn.conv2d(conv1, w2,strides=[1, 1, 1, 1],padding='SAME') conv2_a = tf.nn.relu(conv2) conv2 = tf.nn.max_pool(conv2_a, ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1],padding='SAME') conv2 = tf.nn.dropout(conv2, p_keep_conv) conv3=tf.nn.conv2d(conv2, w3,strides=[1, 1, 1, 1],padding='SAME') conv3 = tf.nn.relu(conv3) FC_layer = tf.nn.max_pool(conv3, ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1],padding='SAME') FC_layer = tf.reshape(FC_layer, [-1, w4.get_shape().as_list()[0]]) FC_layer = tf.nn.dropout(FC_layer, p_keep_conv) output_layer = tf.nn.relu(tf.matmul(FC_layer, w4)) output_layer = tf.nn.dropout(output_layer, p_keep_hidden) result = tf.matmul(output_layer, w_o) return result mnist = input_data.read_data_sets("MNIST_data", one_hot=True) trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels trX = trX.reshape(-1, img_size, img_size, 1) teX = teX.reshape(-1, img_size, img_size, 1) # 28x28x1 input img # 28x28x1 input img X = tf.placeholder("float", [None, img_size, img_size, 1]) Y = tf.placeholder("float", [None, num_classes]) w = init_weights([3, 3, 1, 32]) w2 = init_weights([3, 3, 32, 64]) w3 = init_weights([3, 3, 64, 128]) w4 = init_weights([128 *4 * 4, 625]) w_o = init_weights([625,num_classes]) p_keep_conv = tf.placeholder("float") p_keep_hidden = tf.placeholder("float") py_x = model(X, w, w2, w3, w4, w_o, p_keep_conv, p_keep_hidden) Y_ = tf.nn.softmax_cross_entropy_with_logits(logits=py_x, labels=Y) cost = tf.reduce_mean(Y_) optimizer = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cost) predict_op = tf.argmax(py_x, 1) with tf.Session() as sess: init_g = tf.global_variables_initializer() sess.run(init_g) for i in range(100): training_batch =zip(range(0, len(trX), batch_size), range(batch_size, len(trX)+1, batch_size)) for start, end in training_batch: sess.run(optimizer , feed_dict={X: trX[start:end], Y: trY[start:end], p_keep_conv: 0.8, p_keep_hidden: 0.5}) test_indices = np.arange(len(teX)) # Get A Test Batch np.random.shuffle(test_indices) test_indices = test_indices[0:test_size] print(i, np.mean(np.argmax(teY[test_indices], axis=1) == sess.run (predict_op, feed_dict={X: teX[test_indices], Y: teY[test_indices], p_keep_conv: 1.0, p_keep_hidden: 1.0})))