コンボリューションニューラルネットワーク構造およびtensorflow実装(CNN,ResNet,DenseNet)

本稿では,CNN,ResNet,DenseNetの3種類のボリュームニューラルネットワーク構造と,pythonとtensorflowを用いてそれぞれ3種類のニューラルネットワーク構造を実現し,MNIST手書きデジタルセットで画像分類を実現した.
一.CNN

from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf

mnist = input_data.read_data_sets('MNIST_data', one_hot=True)   #import MNIST
print("start")

inputs_ = tf.placeholder(tf.float32, (None, 28, 28, 1), name='inputs_')#train data
y = tf.placeholder(tf.float32, [None, 10])#train label
sess = tf.InteractiveSession()

def weight_variable(shape):
#initial variable
  initial = tf.truncated_normal(shape, mean=0,stddev=0.1)
  return tf.Variable(initial)

def bias_variable(shape):
  initial = tf.constant(0.1, shape=shape)
  return tf.Variable(initial)

x = tf.reshape(inputs_, (-1,28,28,1))   #x -1*28*28*1

conv1 = tf.layers.conv2d(x, 32, (3,3), padding='same', activation=tf.nn.relu)
pooling1 = tf.layers.max_pooling2d(conv1, (2,2), (2,2), padding='same')
#conv1 turn to 14*14*32
conv2 = tf.layers.conv2d(pooling1, 64, (3,3), padding='same', activation=tf.nn.relu)
pooling2 = tf.layers.max_pooling2d(conv2, (2,2), (2,2), padding='same')
#7*7*64
conv3 = tf.layers.conv2d(pooling2, 64, (3,3), padding='same', activation=tf.nn.relu)
pooling3 = tf.layers.max_pooling2d(conv3, (2,2), (2,2), padding='same')
#4*4*64
#fully connected network
flat = tf.reshape(pooling3, [-1,4*4*64])  #turn to 1 dim

w_fc1 = weight_variable([4 * 4 *64, 1024])
b_fc1 = bias_variable([1024])

h_fc1 = tf.nn.relu(tf.matmul(flat, w_fc1) + b_fc1)
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
w_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])
y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2

prediction = tf.nn.softmax(y_conv,name='prediction')    #softmax        
#      ，          
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_conv))
#     
train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y,1))  #  bool (  ，  )
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))  #   ，     （tf.cast:bool-float  ）
#     

#sess = tf.Session()
sess.run(tf.global_variables_initializer())
pos = 0
trains = []
print("houyu")
for i in range(6001):
    batch = mnist.train.next_batch(100)
    images = batch[0].reshape((-1,28,28,1)) #      [0,1]    
    #print("images shape:",np.shape(images))
    labels = batch[1]
    if i%100 == 0:
        train_accuracy = accuracy.eval(feed_dict={
        inputs_:images, y: labels, keep_prob: 1.0})
        print("step %d, training accuracy %g"%(i, train_accuracy))
    train_step.run(feed_dict={x: images, y: labels, keep_prob: 0.5})

二.ResNet(残差ニューラルネットワーク)

from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf

mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

inputs_ = tf.placeholder(tf.float32, (None, 28, 28, 1), name='inputs_')
y = tf.placeholder(tf.float32, [None, 10])
sess = tf.InteractiveSession()

def weight_variable(shape):
#         
  initial = tf.truncated_normal(shape, mean=0,stddev=0.1) #s          
  return tf.Variable(initial)

def bias_variable(shape):
  initial = tf.constant(0.1, shape=shape)
  return tf.Variable(initial)

#       identity_block (         )
def identity_block(X_input, kernel_size, in_filter, out_filters, stage, block):
        """
        Arguments:
        X_input -- input tensor of shape (m, H, W, C)
        kernel_size -- integer, specifying the shape of the middle CONV's window for the main path
        filter -- python list of integers, defining the number of filters in the CONV layers of the main path
        stage -- integer, used to name the layers, depending on their position in the network
        block -- string/character, used to name the layers, depending on their position in the network

        Returns:
        X -- output of the identity block, tensor of shape (n, H, W, C)
        """

        # defining name basis
        block_name = 'res' + str(stage) + block
        f1, f2, f3 = out_filters
        with tf.variable_scope(block_name):
            X_shortcut = X_input

            #first
            X = tf.layers.conv2d(X_input, f1, (1,1), padding='SAME', activation=tf.nn.relu)

            #second
            X = tf.layers.conv2d(X, f2, (kernel_size, kernel_size), padding='SAME', activation=tf.nn.relu)

            #third
            X = tf.layers.conv2d(X, f3, (1,1), padding='SAME', activation=tf.nn.relu)
            #final step
            add = tf.add(X, X_shortcut)
            b_conv_fin = bias_variable([f3])
            add_result = tf.nn.relu(add+b_conv_fin)

        return add_result

#  conv_block  ，                 ，             ，    
def convolutional_block( X_input, kernel_size, in_filter,
                            out_filters, stage, block, stride=1):
        """
        Arguments:
        X -- input tensor of shape (m, H, W, C)
        kernel_size -- integer, specifying the shape of the middle CONV's window for the main path
        filters -- python list of integers, defining the number of filters in the CONV layers of the main path
        stage -- integer, used to name the layers, depending on their position in the network
        block -- string/character, used to name the layers, depending on their position in the network
        stride -- Integer, specifying the stride to be used

        Returns:
        X -- output of the convolutional block, tensor of shape (n, H, W, C)
        """

        # defining name basis
        block_name = 'res' + str(stage) + block
        with tf.variable_scope(block_name):
            f1, f2, f3 = out_filters

            x_shortcut = X_input
            #filter 1*1*f1
            X = tf.layers.conv2d(X_input, f1, (1,1), padding='SAME', activation=tf.nn.relu)

            #second 3*3*f2
            X = tf.layers.conv2d(X, f2, (kernel_size, kernel_size), padding='SAME', activation=tf.nn.relu)

            #third 1*1*f3
            X = tf.layers.conv2d(X, f3, (1, 1), padding='SAME', activation=tf.nn.relu)
            #shortcut path
            W_shortcut =weight_variable([1, 1, in_filter, f3])
            x_shortcut = tf.nn.conv2d(x_shortcut, W_shortcut, strides=[1, stride, stride, 1], padding='VALID')

            #final
            add = tf.add(x_shortcut, X)
            #         
            b_conv_fin = bias_variable([f3])
            add_result = tf.nn.relu(add + b_conv_fin)

        return add_result

x = tf.reshape(inputs_, (-1,28,28,1))

x1 = convolutional_block(x, 3, 1, [32, 32, 64], stage=1, block='a' )
pooling1 = tf.layers.max_pooling2d(x1, pool_size=(2, 2), strides=(2, 2), padding='SAME')
#       14x14x64

x2 = convolutional_block(X_input=pooling1, kernel_size=3, in_filter=64,  out_filters=[64, 64, 128], stage=2, block='b', stride=1)
pooling2 = tf.layers.max_pooling2d(x2, pool_size=(2, 2), strides=(2, 2), padding='SAME')
#  conv_block   ，    7x7x128

x3 = identity_block(pooling2, 3, 128, [64, 64, 128], stage=2, block='c' )
pooling3 = tf.layers.max_pooling2d(x3, pool_size=(2, 2), strides=(2, 2), padding='SAME')
#           4*4*128

flat = tf.reshape(pooling3, [-1,4*4*128])   #     

w_fc1 = weight_variable([4 * 4 * 128, 1024])
b_fc1 = bias_variable([1024])
h_fc1 = tf.nn.relu(tf.matmul(flat, w_fc1) + b_fc1)

keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
w_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])
y_conv = tf.matmul(h_fc1_drop, w_fc2) + b_fc2

#      ，          
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_conv))

train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) #cast bool   float，        
#     

sess.run(tf.global_variables_initializer())

print("start")
for i in range(2000):
    batch = mnist.train.next_batch(100)
    input = batch[0].reshape((-1,28,28,1))
    if i % 100 == 0:
        train_accuracy = accuracy.eval(feed_dict={
        inputs_:input, y: batch[1], keep_prob: 1.0})
        print("step %d, training accuracy %g"%(i, train_accuracy))
    train_step.run(feed_dict={x: input, y: batch[1], keep_prob: 0.5})

三.DenseNet(稠密ニューラルネットワーク)

from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf

def weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev=0.01)
    return tf.Variable(initial)

def bias_variable(shape):
    initial = tf.constant(0.01, shape=shape)
    return tf.Variable(initial)

def conv2d(input, in_features, out_features, kernel_size, with_bias=False):
    W = weight_variable([ kernel_size, kernel_size, in_features, out_features ])
    conv = tf.nn.conv2d(input, W, [ 1, 1, 1, 1 ], padding='SAME')
    if with_bias:
        return conv + bias_variable([ out_features ])
    return conv

def avg_pool(input, s):
    return tf.nn.avg_pool(input, [ 1, s, s, 1 ], [1, s, s, 1 ], 'SAME')

def batch_activ_conv(current, in_features, out_features, kernel_size, is_training, keep_prob):
    current = tf.contrib.layers.batch_norm(current, scale=True, is_training=is_training, updates_collections=None)
    current = tf.nn.relu(current)
    current = conv2d(current, in_features, out_features, kernel_size)
    current = tf.nn.dropout(current, keep_prob)
    return current
#            
def block(input, layers, in_features, growth, is_training, keep_prob):
    current = input
    features = in_features
    for idx in range(layers):
        tmp = batch_activ_conv(current, features, growth, 3, is_training, keep_prob)
        current = tf.concat((current, tmp), axis=3)
        features += growth
    return current, features

mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

inputs_ = tf.placeholder(tf.float32, (None, 28, 28, 1), name='inputs_')
y = tf.placeholder(tf.float32, [None, 10])

keep_prob = tf.placeholder(tf.float32)
#is_training = tf.placeholder("bool", shape=[])
is_training = True
sess = tf.InteractiveSession()

x = tf.reshape(inputs_, (-1,28,28,1))
conv1 = conv2d(x, 1, 16, 3)
#       28*28*16
current, features = block(conv1, layers=4, in_features = 16, growth = 12, is_training = is_training, keep_prob = keep_prob)
current = batch_activ_conv(current, features, features, 1, is_training = is_training, keep_prob = keep_prob)
current = avg_pool(current, 2)

current, features = block(current, layers = 4, in_features = features, growth = 12, is_training = is_training, keep_prob = keep_prob)
current = batch_activ_conv(current, features, features, 1, is_training = is_training, keep_prob = keep_prob)
current = avg_pool(current, 2)

current, features = block(current, layers = 4, in_features = features, growth = 12, is_training = is_training, keep_prob = keep_prob)
current = tf.contrib.layers.batch_norm(current, scale=True, is_training=is_training, updates_collections=None)
current = tf.nn.relu(current)
# current = avg_pool(current, 8)
current = avg_pool(current, 2)
final_dim = features
label_count = 1024
flat = tf.reshape(current, [-1, 4*4*final_dim])
Wfc = weight_variable([4*4*final_dim, label_count])
bfc = bias_variable([label_count])

h_fc1 = tf.nn.relu(tf.matmul(flat, Wfc) + bfc)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
w_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])

ys_ = tf.matmul(h_fc1_drop, w_fc2) + b_fc2

#      ，          
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=ys_))

train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(ys_,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#     

sess.run(tf.global_variables_initializer())

print("start")
for i in range(2000):
    batch = mnist.train.next_batch(100)
    input = batch[0].reshape((-1,28,28,1))
    if i % 100 == 0:
        train_accuracy = accuracy.eval(feed_dict={
        inputs_:input, y: batch[1], keep_prob: 1.0})
        print("step %d, training accuracy %g"%(i, train_accuracy))
    train_step.run(feed_dict={x: input, y: batch[1], keep_prob: 1.0})

残差ニューラルネットワーク: