# Author:凌逆战
# -*- encoding:utf-8 -*-
# 修改时间：2020年5月31日
import time
from tensorflow.examples.tutorials.mnist import input_data
from nets.my_alex import alexNet
from ops import *
tf.flags.DEFINE_integer('batch_size', 50, 'batch size, default: 1')
tf.flags.DEFINE_integer('class_num', 10, 'batch size, default: 1')
tf.flags.DEFINE_integer('epochs', 10, 'batch size, default: 1')
tf.flags.DEFINE_float('learning_rate', 1e-4, '初始学习率, 默认: 0.0002')
tf.flags.DEFINE_string('checkpoints_dir', "checkpoints", '保存检查点的地址')
FLAGS = tf.flags.FLAGS
# 从MNIST_data/中读取MNIST数据。当数据不存在时，会自动执行下载
mnist = input_data.read_data_sets('./data', one_hot=True, reshape=False)
# reshape=False  (None, 28,28,1)    # 用于第一层是卷积层
# reshape=False  (None, 784)        # 用于第一层是全连接层
# 我们看一下数据的shape
print(mnist.train.images.shape)  # 训练数据图片(55000, 28, 28, 1)
print(mnist.train.labels.shape)  # 训练数据标签(55000, 10)
print(mnist.test.images.shape)  # 测试数据图片(10000, 28, 28, 1)
print(mnist.test.labels.shape)  # 测试数据图片(10000, 10)
print(mnist.validation.images.shape)  # 验证数据图片(5000, 28, 28, 1)
print(mnist.validation.labels.shape)  # 验证数据图片(5000, 784)
def train():
batch_size = FLAGS.batch_size  # 一个batch训练多少个样本
batch_nums = mnist.train.images.shape[0] // batch_size  # 一个epoch中应该包含多少batch数据
class_num = FLAGS.class_num  # 分类类别数
epochs = FLAGS.epochs  # 训练周期数
learning_rate = FLAGS.learning_rate  # 初始学习率
############    保存检查点的地址   ############
checkpoints_dir = FLAGS.checkpoints_dir  # checkpoints
# 如果检查点不存在，则创建
if not os.path.exists(checkpoints_dir):
os.makedirs(FLAGS.checkpoints_dir)
######################################################
#                    创建图                          #
######################################################
graph = tf.Graph()  # 自定义图
# 在自己的图中定义数据和操作
with graph.as_default():
inputs = tf.placeholder(dtype="float", shape=[None, 28, 28, 1], name='inputs')
labels = tf.placeholder(dtype="float", shape=[None, class_num], name='labels')
# 看个人喜欢，有的人在初始化定义中就定义了learning_rate，有的人喜欢通过feed传learning_rate
learning_rate = tf.placeholder("float", None, name='learning_rate')
# 如果网络结构有dropout层，需要定义keep_probn,如果没有则不需要
# 训练的时候需要，测试的时候需要设置成1
keep_prob = tf.placeholder(dtype="float", name='keep_prob')
############    搭建模型   ############
logits = alexNet(inputs, class_num, keep_prob=keep_prob)  # 使用placeholder搭建模型
############    损失函数   ############
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits))
tf.add_to_collection('losses', loss)
total_loss = tf.add_n(tf.get_collection("loss"))  # total_loss=模型损失+权重正则化损失
############    模型精度   ############
predict = tf.argmax(logits, 1)  # 模型预测结果
accuracy = tf.reduce_mean(tf.cast(tf.equal(predict, tf.argmax(labels, 1)), tf.float32))
############    优化器   ############
variable_to_train = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)  # 可训练变量列表
# 创建优化器，更新网络参数，最小化loss，
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(learning_rate=learning_rate,  # 初始学习率
global_step=global_step,
decay_steps=batch_nums,  # 多少步衰减一次
decay_rate=0.1,  # 衰减率
staircase=True)  # 以阶梯的形式衰减
# 移动平均值更新参数
# train_op = moving_average(loss, learning_rate, global_step)
# adam优化器,adam算法好像会自动衰减学习率，
train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss=total_loss,
global_step=global_step,
var_list=variable_to_train)
############    TensorBoard可视化 summary  ############
summary_writer = tf.summary.FileWriter(logdir="./logs", graph=graph)  # 创建事件文件
tf.summary.scalar(name="losses", tensor=total_loss)  # 收集损失值变量
tf.summary.scalar(name="acc", tensor=accuracy)  # 收集精度值变量
tf.summary.scalar(name='learning_rate', tensor=learning_rate)
merged_summary_op = tf.summary.merge_all()  # 将所有的summary合并为一个op
############    模型保存和恢复 Saver   ############
saver = tf.train.Saver(max_to_keep=5)
######################################################
#                   创建会话                          #
######################################################
max_acc = 0.
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)
with tf.Session(config=config, graph=graph) as sess:
# 加载模型，如果模型存在返回 是否加载成功和训练步数
could_load, checkpoint_step = load_model(sess, saver, FLAGS.checkpoints_dir)
if could_load:
print(" [*] 模型加载成功")
else:
print(" [!] 模型加载失败")
try:
tf.global_variables_initializer().run()
except:
tf.initialize_all_variables().run()
for epoch in range(epochs):
for i in range(batch_nums):
start_time = time.time()
# batch_images = data_X[i * batch_size:(i + 1) * batch_size]
# batch_labels = data_y[i * batch_size:(i + 1) * batch_size]
train_batch_x, train_batch_y = mnist.train.next_batch(batch_size)
# 使用真实数据填充placeholder，运行训练模型和合并变量操作
_, summary, loss, step = sess.run([train_op, merged_summary_op, total_loss, global_step],
feed_dict={inputs: train_batch_x,
labels: train_batch_y,
keep_prob: 0.5})
if step % 100 == 0:
summary_writer.add_summary(summary, step)  # 将每次迭代后的变量写入事件文件
summary_writer.flush()  # 强制summary_writer将缓存中的数据写入到日志文件中（可选）
############    可视化打印   ############
print("Epoch：[%2d] [%4d/%4d] time：%4.4f，loss：%.8f" % (
epoch, i, batch_nums, time.time() - start_time, loss))
# 打印一些可视化的数据，损失...
if step % 100 == 0:
acc = sess.run(accuracy, feed_dict={inputs: mnist.validation.images,
labels: mnist.validation.labels,
keep_prob: 1.0})
print("Epoch：[%2d] [%4d/%4d] accuracy：%.8f" % (epoch, i, batch_nums, acc))
############    保存模型   ############
if acc > max_acc:
max_acc = acc
save_path = saver.save(sess,
save_path=os.path.join(checkpoints_dir, "model.ckpt"),
global_step=step)
tf.logging.info("模型保存在: %s" % save_path)
print("优化完成!")
def main(argv=None):
train()
if __name__ == '__main__':
# logging.basicConfig(level=logging.INFO)
tf.logging.set_verbosity(tf.logging.INFO)
tf.app.run()

# Author:凌逆战
# -*- encoding:utf-8 -*-
# 修改时间：2020年5月31日
import time
from tensorflow.examples.tutorials.mnist import input_data
from nets.my_vgg import VGG16Net
from ops import *
tf.flags.DEFINE_integer('batch_size', 100, 'batch size, default: 1')
tf.flags.DEFINE_integer('class_num', 10, 'batch size, default: 1')
tf.flags.DEFINE_integer('epochs', 10, 'batch size, default: 1')
tf.flags.DEFINE_float('learning_rate', 2e-4, '初始学习率, 默认: 0.0001')
tf.flags.DEFINE_string('checkpoints_dir', "checkpoint", '保存检查点的地址')
FLAGS = tf.flags.FLAGS
# 从MNIST_data/中读取MNIST数据。当数据不存在时，会自动执行下载
mnist = input_data.read_data_sets('./MNIST_data', one_hot=True, reshape=False)
# reshape=False  (None, 28,28,1)    # 用于第一层是卷积层
# reshape=False  (None, 784)        # 用于第一层是全连接层
# 我们看一下数据的shape
print(mnist.train.images.shape)  # 训练数据图片(55000, 28, 28, 1)
print(mnist.train.labels.shape)  # 训练数据标签(55000, 10)
print(mnist.test.images.shape)  # 测试数据图片(10000, 28, 28, 1)
print(mnist.test.labels.shape)  # 测试数据图片(10000, 10)
print(mnist.validation.images.shape)  # 验证数据图片(5000, 28, 28, 1)
print(mnist.validation.labels.shape)  # 验证数据图片(5000, 784)
def train():
batch_size = FLAGS.batch_size
batch_nums = mnist.train.images.shape[0] // batch_size  # 一个epoch中应该包含多少batch数据
class_num = FLAGS.class_num
epochs = FLAGS.epochs
learning_rate = FLAGS.learning_rate
############    保存检查点的地址   ############
checkpoints_dir = FLAGS.checkpoints_dir  # checkpoints
# 如果检查点不存在，则创建
if not os.path.exists(checkpoints_dir):
os.makedirs(FLAGS.checkpoints_dir)
######################################################
#                    创建图                          #
######################################################
graph = tf.Graph()  # 自定义图
# 在自己的图中定义数据和操作
with graph.as_default():
inputs = tf.placeholder(dtype="float", shape=[None, 28, 28, 1], name='inputs')
labels = tf.placeholder(dtype="float", shape=[None, class_num], name='labels')
############    搭建模型   ############
logits = VGG16Net(inputs, class_num)  # 使用placeholder搭建模型
############    损失函数   ############
# 计算预测值和真实值之间的误差
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits))
tf.add_to_collection('losses', loss)
total_loss = tf.add_n(tf.get_collection("loss"))  # total_loss=模型损失+权重正则化损失
############    模型精度   ############
predict = tf.argmax(logits, axis=1)
accuracy = tf.reduce_mean(tf.cast(tf.equal(predict, tf.argmax(labels, axis=1)), tf.float32))
############    优化器   ############
variable_to_train = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)  # 可训练变量列表
# 创建优化器，更新网络参数，最小化loss，
train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss=total_loss,
var_list=variable_to_train)
############    TensorBoard可视化 summary  ############
summary_writer = tf.summary.FileWriter("./logs", graph=graph)  # 创建事件文件
tf.summary.scalar(name="loss", tensor=total_loss)  # 收集损失值变量
tf.summary.scalar(name='accuracy', tensor=accuracy)  # 收集精度值变量
tf.summary.scalar(name='learning_rate', tensor=learning_rate)
merged_summary_op = tf.summary.merge_all()  # 将所有的summary合并为一个op
############    模型保存和恢复 Saver   ############
saver = tf.train.Saver(max_to_keep=5)
######################################################
#                   创建会话                          #
######################################################
max_acc = 0.
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)
with tf.Session(config=config, graph=graph) as sess:
# 加载模型，如果模型存在返回 是否加载成功和训练步数
could_load, checkpoint_step = load_model(sess, saver, FLAGS.checkpoints_dir)
if could_load:
step = checkpoint_step
print(" [*] 模型加载成功")
else:
print(" [!] 模型加载失败")
try:
tf.global_variables_initializer().run()
except:
tf.initialize_all_variables().run()
step = 0
for epoch in range(epochs):
for i in range(batch_nums):
start_time = time.time()    # 记录一下开始训练的时间
# batch_images = data_X[i * batch_size:(i + 1) * batch_size]
# batch_labels = data_y[i * batch_size:(i + 1) * batch_size]
train_batch_x, train_batch_y = mnist.train.next_batch(batch_size)
# 使用真实数据填充placeholder，运行训练模型和合并变量操作
_, summary, loss = sess.run([train_op, merged_summary_op, total_loss],
feed_dict={inputs: train_batch_x,
labels: train_batch_y})
if step % 100 == 0:
summary_writer.add_summary(summary, step)  # 将每次迭代后的变量写入事件文件
summary_writer.flush()  # 强制summary_writer将缓存中的数据写入到日志文件中（可选）
############    可视化打印   ############
print("Epoch：[%2d] [%4d/%4d] time：%4.4f，loss：%.8f" % (
epoch, i, batch_nums, time.time() - start_time, loss))
# 打印一些可视化的数据，损失...
# if np.mod(step, 100) == 1
if step % 100 == 0:
acc = sess.run(accuracy, {inputs: mnist.validation.images,
labels: mnist.validation.labels})
print("Epoch：[%2d] [%4d/%4d]，acc：%.8f" % (epoch, i, batch_nums, acc))
############    保存模型   ############
if acc > max_acc:
max_acc = acc
save_path = saver.save(sess,
save_path=os.path.join(checkpoints_dir, "model.ckpt"),
global_step=step)
# logging.info("模型保存在: %s" % save_path)
tf.logging.info("模型保存在: %s" % save_path)
step += 1
print("优化完成!")
def main(argv=None):
train()
if __name__ == '__main__':
# logging.basicConfig(level=logging.INFO)
tf.logging.set_verbosity(tf.logging.INFO)
tf.app.run()

def conv(input, kernel_size, output_size, stride, init_bias=0.0, padding="SAME", name=None, wd=None):
input_size = input.shape[-1]
conv_weights = tf.get_variable(name='weights',
shape=[kernel_size, kernel_size, input_size, output_size],
initializer=tf.truncated_normal_initializer(stddev=0.1),
dtype=tf.float32)
conv_biases = tf.get_variable(name='biases',
shape=[output_size],
initializer=tf.constant_initializer(init_bias),
dtype=tf.float32)
if wd is not None:
# wd 0.004
# tf.nn.l2_loss(var)=sum(t**2)/2
weight_decay = tf.multiply(tf.nn.l2_loss(conv_weights), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
conv_layer = tf.nn.conv2d(input, conv_weights, [1, stride, stride, 1], padding=padding, name=name)  # 卷积操作
conv_layer = tf.nn.bias_add(conv_layer, conv_biases)  # 加上偏置项
conv_layer = tf.nn.relu(conv_layer)  # relu激活函数
return conv_layer
def fc(input, output_size, init_bias=0.0, activeation_func=True, wd=None):
input_shape = input.get_shape().as_list()
# 创建 全连接权重 变量
fc_weights = tf.get_variable(name="weights",
shape=[input_shape[-1], output_size],
initializer=tf.truncated_normal_initializer(stddev=0.1),
dtype=tf.float32)
if wd is not None:
# wd 0.004
# tf.nn.l2_loss(var)=sum(t**2)/2
weight_decay = tf.multiply(tf.nn.l2_loss(fc_weights), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
# 创建 全连接偏置 变量
fc_biases = tf.get_variable(name="biases",
shape=[output_size],
initializer=tf.constant_initializer(init_bias),
dtype=tf.float32)
fc_layer = tf.matmul(input, fc_weights)  # 全连接计算
fc_layer = tf.nn.bias_add(fc_layer, fc_biases)  # 加上偏置项
if activeation_func:
fc_layer = tf.nn.relu(fc_layer)  # rele激活函数
return fc_layer

def conv(input, kernel_size, output_size, stride, init_bias=0.0, padding="SAME", name=None, wd=None):
input_size = input.shape[-1]
conv_weights = tf.get_variable(name='weights',
shape=[kernel_size, kernel_size, input_size, output_size],
initializer=tf.random_normal_initializer(mean=0, stddev=0.01),
dtype=tf.float32)
if wd is not None:
# wd 0.004
# tf.nn.l2_loss(var)=sum(t**2)/2
weight_decay = tf.multiply(tf.nn.l2_loss(conv_weights), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
conv_biases = tf.get_variable(name='biases',
shape=[output_size],
initializer=tf.constant_initializer(init_bias),
dtype=tf.float32)
conv_layer = tf.nn.conv2d(input, conv_weights, [1, stride, stride, 1], padding=padding, name=name)  # 卷积操作
conv_layer = tf.nn.bias_add(conv_layer, conv_biases)  # 加上偏置项
conv_layer = tf.nn.relu(conv_layer)  # relu激活函数
return conv_layer

def fc(input, output_size, init_bias=0.0, activeation_func=True, wd=None):
input_shape = input.get_shape().as_list()
# 创建 全连接权重 变量
fc_weights = tf.get_variable(name="weights",
shape=[input_shape[-1], output_size],
initializer=tf.random_normal_initializer(mean=0.0, stddev=0.01),
dtype=tf.float32)
if wd is not None:
# wd 0.004
# tf.nn.l2_loss(var)=sum(t**2)/2
weight_decay = tf.multiply(tf.nn.l2_loss(fc_weights), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
# 创建 全连接偏置 变量
fc_biases = tf.get_variable(name="biases",
shape=[output_size],
initializer=tf.constant_initializer(init_bias),
dtype=tf.float32)
fc_layer = tf.matmul(input, fc_weights)  # 全连接计算
fc_layer = tf.nn.bias_add(fc_layer, fc_biases)  # 加上偏置项
if activeation_func:
fc_layer = tf.nn.relu(fc_layer)  # rele激活函数
return fc_layer

def LRN(input, depth_radius=2, alpha=0.0001, beta=0.75, bias=1.0):
"""Local Response Normalization 局部响应归一化"""
return tf.nn.local_response_normalization(input, depth_radius=depth_radius, alpha=alpha,
beta=beta, bias=bias)

def conv(inputs, scope_name, kernel_size, output_size, stride, init_bias=0.0, padding="SAME", wd=None):
input_size = int(inputs.get_shape()[-1])
with tf.variable_scope(scope_name):
conv_weights = tf.get_variable(name='weights',
shape=[kernel_size, kernel_size, input_size, output_size],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=1e-1))
if wd is not None:
# tf.nn.l2_loss(var)=sum(t**2)/2
weight_decay = tf.multiply(tf.nn.l2_loss(conv_weights), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
conv_biases = tf.get_variable(name='biases',
shape=[output_size],
dtype=tf.float32,
initializer=tf.constant_initializer(init_bias))
conv_layer = tf.nn.conv2d(inputs, conv_weights, [1, stride, stride, 1], padding=padding, name=scope_name)
conv_layer = tf.nn.bias_add(conv_layer, conv_biases)
conv_layer = tf.nn.relu(conv_layer)
return conv_layer

def fc(inputs, scope_name, output_size, init_bias=0.0, activeation_func=True, wd=None):
input_shape = inputs.get_shape().as_list()
with tf.variable_scope(scope_name):
# 创建 全连接权重 变量
fc_weights = tf.get_variable(name="weights",
shape=[input_shape[-1], output_size],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=1e-1))
if wd is not None:
# wd 0.004
# tf.nn.l2_loss(var)=sum(t**2)/2
weight_decay = tf.multiply(tf.nn.l2_loss(fc_weights), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
# 创建 全连接偏置 变量
fc_biases = tf.get_variable(name="biases",
shape=[output_size],
dtype=tf.float32,
initializer=tf.constant_initializer(init_bias),
trainable=True)
fc_layer = tf.matmul(inputs, fc_weights)  # 全连接计算
fc_layer = tf.nn.bias_add(fc_layer, fc_biases)  # 加上偏置项
if activeation_func:
fc_layer = tf.nn.relu(fc_layer)  # rele激活函数
return fc_layer

def batch_normalization(inputs, output_size):
mean, variance = tf.nn.moments(inputs, axes=[0, 1, 2])  # 计算均值和方差
beta = tf.get_variable('beta', output_size, tf.float32, initializer=tf.zeros_initializer)
gamma = tf.get_variable('gamma', output_size, tf.float32, initializer=tf.ones_initializer)
bn_layer = tf.nn.batch_normalization(inputs, mean, variance, beta, gamma, 0.001)
return bn_layer

def conv(input, kernel_size, output_size, stride, padding="SAME", wd=None):
input_size = input.shape[-1]
conv_weights = tf.get_variable(name='weights',
shape=[kernel_size, kernel_size, input_size, output_size],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
regularizer=tf.contrib.layers.l2_regularizer(0.00004)) # 正则损失衰减率0.000004
conv_layer = tf.nn.conv2d(input, conv_weights, [1, stride, stride, 1], padding=padding)  # 卷积操作
batch_norm = batch_normalization(conv_layer, output_size)
conv_output = tf.nn.relu(batch_norm)  # relu激活函数
return conv_output

def fc(input, output_size, activeation_func=True):
input_shape = input.shape[-1]
# 创建 全连接权重 变量
fc_weights = tf.get_variable(name="weights",
shape=[input_shape, output_size],
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32,
regularizer=tf.contrib.layers.l2_regularizer(0.01))
# 创建 全连接偏置 变量
fc_biases = tf.get_variable(name="biases",
shape=[output_size],
initializer=tf.zeros_initializer,
dtype=tf.float32)
fc_layer = tf.matmul(input, fc_weights)  # 全连接计算
fc_layer = tf.nn.bias_add(fc_layer, fc_biases)  # 加上偏置项
if activeation_func:
fc_layer = tf.nn.relu(fc_layer)  # rele激活函数
return fc_layer

def block(input, n, output_size, change_first_stride, bottleneck):
if n == 0 and change_first_stride:
stride = 2
else:
stride = 1
if bottleneck:
with tf.variable_scope('a'):
conv_a = conv(input=input, kernel_size=1, output_size=output_size, stride=stride, padding="SAME")
conv_a = batch_normalization(conv_a, output_size)
conv_a = tf.nn.relu(conv_a)
with tf.variable_scope('b'):
conv_b = conv(input=conv_a, kernel_size=3, output_size=output_size, stride=1, padding="SAME")
conv_b = batch_normalization(conv_b, output_size)
conv_b = tf.nn.relu(conv_b)
with tf.variable_scope('c'):
conv_c = conv(input=conv_b, kernel_size=1, output_size=output_size * 4, stride=1, padding="SAME")
output = batch_normalization(conv_c, output_size * 4)
else:
with tf.variable_scope('A'):
conv_A = conv(input=input, kernel_size=3, output_size=output_size, stride=stride, padding="SAME")
conv_A = batch_normalization(conv_A, output_size)
conv_A = tf.nn.relu(conv_A)
with tf.variable_scope('B'):
conv_B = conv(input=conv_A, kernel_size=3, output_size=output_size, stride=1, padding="SAME")
output = batch_normalization(conv_B, output_size)
if input.shape == output.shape:
with tf.variable_scope('shortcut'):
shortcut = input  # shortcut
else:
with tf.variable_scope('shortcut'):
shortcut = conv(input=input, kernel_size=1, output_size=output_size * 4, stride=1, padding="SAME")
shortcut = batch_normalization(shortcut, output_size * 4)
return tf.nn.relu(output + shortcut)

def fc(input, output_size, activeation_func=tf.nn.relu):
input_shape = input.shape[-1]
# 创建 全连接权重 变量
fc_weights = tf.get_variable(name="weights",
shape=[input_shape, output_size],
initializer=tf.truncated_normal_initializer(stddev=0.01),
dtype=tf.float32,
regularizer=tf.contrib.layers.l2_regularizer(0.01))
# 创建 全连接偏置 变量
fc_biases = tf.get_variable(name="biases",
shape=[output_size],
initializer=tf.zeros_initializer,
dtype=tf.float32)
fc_layer = tf.matmul(input, fc_weights)  # 全连接计算
fc_layer = tf.nn.bias_add(fc_layer, fc_biases)  # 加上偏置项
if activeation_func:
fc_layer = activeation_func(fc_layer)  # rele激活函数
return fc_layer

#######################################
#       单层 静态/动态 LSTM/GRU        #
#######################################
# 单层静态LSTM
def single_layer_static_lstm(input_x, time_steps, hidden_size):
"""
:param input_x: 输入张量 形状为[batch_size, n_steps, input_size]
:param n_steps: 时序总数
:param n_hidden: LSTM单元输出的节点个数 即隐藏层节点数
"""
# 把输入input_x按列拆分，并返回一个有n_steps个张量组成的list
# 如batch_sizex28x28的输入拆成[(batch_size,28),((batch_size,28))....]
# 如果是调用的是静态rnn函数，需要这一步处理   即相当于把序列作为第一维度
input_x1 = tf.unstack(input_x, num=time_steps, axis=1)
lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size)  # 创建LSTM_cell
# 静态rnn函数传入的是一个张量list  每一个元素都是一个(batch_size,input_size)大小的张量
output, states = tf.nn.static_rnn(cell=lstm_cell, inputs=input_x1, dtype=tf.float32)  # 通过cell类构建RNN
return output, states
# 单层静态gru
def single_layer_static_gru(input, time_steps, hidden_size):
"""
:param input: 输入张量 形状为[batch_size, n_steps, input_size]
:param n_steps: 时序总数
:param n_hidden: gru单元输出的节点个数 即隐藏层节点数
:return: 返回静态单层GRU单元的输出，以及cell状态
"""
# 把输入input_x按列拆分，并返回一个有n_steps个张量组成的list
# 如batch_sizex28x28的输入拆成[(batch_size,28),((batch_size,28))....]
# 如果是调用的是静态rnn函数，需要这一步处理   即相当于把序列作为第一维度
input_x = tf.unstack(input, num=time_steps, axis=1)
gru_cell = tf.nn.rnn_cell.GRUCell(num_units=hidden_size)  # 创建GRU_cell
# 静态rnn函数传入的是一个张量list  每一个元素都是一个(batch_size,input_size)大小的张量
output, states = tf.nn.static_rnn(cell=gru_cell, inputs=input_x, dtype=tf.float32)  # 通过cell类构建RNN
return output, states
# 单层动态LSTM
def single_layer_dynamic_lstm(input, time_steps, hidden_size):
"""
:param input_x: 输入张量 形状为[batch_size, time_steps, input_size]
:param time_steps: 时序总数
:param hidden_size: LSTM单元输出的节点个数 即隐藏层节点数
:return: 返回动态单层LSTM单元的输出，以及cell状态
"""
lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size)  # 创建LSTM_cell
# 动态rnn函数传入的是一个三维张量，[batch_size,time_steps, input_size]  输出也是这种形状
output, states = tf.nn.dynamic_rnn(cell=lstm_cell, inputs=input, dtype=tf.float32)  # 通过cell类构建RNN
output = tf.transpose(output, [1, 0, 2])  # 注意这里输出需要转置  转换为时序优先的
return output, states
# 单层动态gru
def single_layer_dynamic_gru(input, time_steps, hidden_size):
"""
:param input: 输入张量 形状为[batch_size, time_steps, input_size]
:param time_steps: 时序总数
:param hidden_size: GRU单元输出的节点个数 即隐藏层节点数
:return: 返回动态单层GRU单元的输出，以及cell状态
"""
gru_cell = tf.nn.rnn_cell.GRUCell(num_units=hidden_size)  # 创建GRU_cell
# 动态rnn函数传入的是一个三维张量，[batch_size,n_steps,input_size]  输出也是这种形状
output, states = tf.nn.dynamic_rnn(cell=gru_cell, inputs=input, dtype=tf.float32)  # 通过cell类构建RNN
output = tf.transpose(output, [1, 0, 2])  # 注意这里输出需要转置  转换为时序优先的
return output, states

#######################################
#       多层 静态/动态 LSTM/GRU        #
#######################################
# 多层静态LSTM网络
def multi_layer_static_lstm(input, time_steps, hidden_size):
"""
:param input: 输入张量 形状为[batch_size,time_steps,input_size]
:param time_steps: 时序总数
:param n_hidden: LSTM单元输出的节点个数 即隐藏层节点数
:return: 返回静态多层LSTM单元的输出，以及cell状态
"""
# 把输入input_x按列拆分，并返回一个有n_steps个张量组成的list
# 如batch_sizex28x28的输入拆成[(batch_size,28),((batch_size,28))....]
# 如果是调用的是静态rnn函数，需要这一步处理   即相当于把序列作为第一维度
input_x1 = tf.unstack(input, num=time_steps, axis=1)
# 多层RNN的实现 例如cells=[cell1,cell2,cell3]，则表示一共有三层
mcell = tf.nn.rnn_cell.MultiRNNCell(
[tf.nn.rnn_cell.LSTMCell(num_units=hidden_size) for _ in range(3)])
# 静态rnn函数传入的是一个张量list  每一个元素都是一个(batch_size,input_size)大小的张量
output, states = tf.nn.static_rnn(cell=mcell, inputs=input_x1, dtype=tf.float32)
return output, states
# 多层静态GRU
def multi_layer_static_gru(input, time_steps, hidden_size):
"""
:param input_x: 输入张量 形状为[batch_size,n_steps,input_size]
:param time_steps: 时序总数
:param hidden_size: gru单元输出的节点个数 即隐藏层节点数
:return: 返回静态多层GRU单元的输出，以及cell状态
"""
# 把输入input_x按列拆分，并返回一个有n_steps个张量组成的list
# 如batch_sizex28x28的输入拆成[(batch_size,28),((batch_size,28))....]
# 如果是调用的是静态rnn函数，需要这一步处理   即相当于把序列作为第一维度
input_x = tf.unstack(input, num=time_steps, axis=1)
# 多层RNN的实现 例如cells=[cell1,cell2,cell3]，则表示一共有三层
mcell = tf.nn.rnn_cell.MultiRNNCell(
[tf.nn.rnn_cell.GRUCell(num_units=hidden_size) for _ in range(3)])
# 静态rnn函数传入的是一个张量list  每一个元素都是一个(batch_size,input_size)大小的张量
output, states = tf.nn.static_rnn(cell=mcell, inputs=input_x, dtype=tf.float32)
return output, states
# 多层静态GRU和LSTM 混合
def multi_layer_static_mix(input, time_steps, hidden_size):
"""
:param input: 输入张量 形状为[batch_size,n_steps,input_size]
:param time_steps: 时序总数
:param hidden_size: gru单元输出的节点个数 即隐藏层节点数
:return: 返回静态多层GRU和LSTM混合单元的输出，以及cell状态
"""
# 把输入input_x按列拆分，并返回一个有n_steps个张量组成的list
# 如batch_sizex28x28的输入拆成[(batch_size,28),((batch_size,28))....]
# 如果是调用的是静态rnn函数，需要这一步处理   即相当于把序列作为第一维度
input_x = tf.unstack(input, num=time_steps, axis=1)
# 可以看做2个隐藏层
lstm_cell = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size)
gru_cell = tf.nn.rnn_cell.GRUCell(num_units=hidden_size * 2)
# 多层RNN的实现 例如cells=[cell1,cell2]，则表示一共有两层，数据经过cell1后还要经过cells
mcell = tf.nn.rnn_cell.MultiRNNCell(cells=[lstm_cell, gru_cell])
# 静态rnn函数传入的是一个张量list  每一个元素都是一个(batch_size,input_size)大小的张量
output, states = tf.nn.static_rnn(cell=mcell, inputs=input_x, dtype=tf.float32)
return output, states
# 多层动态LSTM
def multi_layer_dynamic_lstm(input, time_steps, hidden_size):
"""
:param input: 输入张量  形状为[batch_size,n_steps,input_size]
:param time_steps: 时序总数
:param hidden_size: LSTM单元输出的节点个数 即隐藏层节点数
:return: 返回动态多层LSTM单元的输出，以及cell状态
"""
# 多层RNN的实现 例如cells=[cell1,cell2]，则表示一共有两层，数据经过cell1后还要经过cells
mcell = tf.nn.rnn_cell.MultiRNNCell(
[tf.nn.rnn_cell.LSTMCell(num_units=hidden_size) for _ in range(3)])
# 动态rnn函数传入的是一个三维张量，[batch_size,n_steps,input_size]  输出也是这种形状
output, states = tf.nn.dynamic_rnn(cell=mcell, inputs=input, dtype=tf.float32)
# 注意这里输出需要转置  转换为时序优先的
output = tf.transpose(output, [1, 0, 2])
return output, states
# 多层动态GRU
def multi_layer_dynamic_gru(input, time_steps, hidden_size):
"""
:param input: 输入张量 形状为[batch_size,n_steps,input_size]
:param time_steps: 时序总数
:param hidden_size: gru单元输出的节点个数 即隐藏层节点数
:return: 返回动态多层GRU单元的输出，以及cell状态
"""
# 多层RNN的实现 例如cells=[cell1,cell2]，则表示一共有两层，数据经过cell1后还要经过cells
mcell = tf.nn.rnn_cell.MultiRNNCell(
[tf.nn.rnn_cell.GRUCell(num_units=hidden_size) for _ in range(3)])
# 动态rnn函数传入的是一个三维张量，[batch_size,n_steps,input_size]  输出也是这种形状
output, states = tf.nn.dynamic_rnn(cell=mcell, inputs=input, dtype=tf.float32)
# 注意这里输出需要转置  转换为时序优先的
output = tf.transpose(output, [1, 0, 2])
return output, states
# 多层动态GRU和LSTM 混合
def multi_layer_dynamic_mix(input, time_steps, hidden_size):
"""
:param input: 输入张量 形状为[batch_size,n_steps,input_size]
:param time_steps: 时序总数
:param hidden_size: gru单元输出的节点个数 即隐藏层节点数
:return: 返回动态多层GRU和LSTM混合单元的输出，以及cell状态
"""
# 可以看做2个隐藏层
gru_cell = tf.nn.rnn_cell.GRUCell(num_units=hidden_size * 2)
lstm_cell = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size)
# 多层RNN的实现 例如cells=[cell1,cell2]，则表示一共有两层，数据经过cell1后还要经过cells
mcell = tf.nn.rnn_cell.MultiRNNCell(cells=[lstm_cell, gru_cell])
# 动态rnn函数传入的是一个三维张量，[batch_size,n_steps,input_size]  输出也是这种形状
output, states = tf.nn.dynamic_rnn(cell=mcell, inputs=input, dtype=tf.float32)
# 注意这里输出需要转置  转换为时序优先的
output = tf.transpose(output, [1, 0, 2])
return output, states

#######################################
#   单层/多层 双向 静态/动态 LSTM/GRU   #
#######################################
# 单层静态双向LSTM
def single_layer_static_bi_lstm(input, time_steps, hidden_size):
"""
:param input: 输入张量 形状为[batch_size,time_steps,input_size]
:param time_steps: 时序总数
:param hidden_size: LSTM单元输出的节点个数 即隐藏层节点数
:return: 返回单层静态双向LSTM单元的输出，以及cell状态
"""
# 把输入input_x按列拆分，并返回一个有n_steps个张量组成的list
# 如batch_sizex28x28的输入拆成[(batch_size,28),((batch_size,28))....]
# 如果是调用的是静态rnn函数，需要这一步处理   即相当于把序列作为第一维度
input_x = tf.unstack(input, num=time_steps, axis=1)
lstm_fw_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size)  # 正向
lstm_bw_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size)  # 反向
# 静态rnn函数传入的是一个张量list  每一个元素都是一个(batch_size,input_size)大小的张量
# 这里的输出output是一个list 每一个元素都是前向输出,后向输出的合并
output, fw_state, bw_state = tf.nn.static_bidirectional_rnn(cell_fw=lstm_fw_cell,
cell_bw=lstm_bw_cell,
inputs=input_x,
dtype=tf.float32)
print(type(output))  # <class 'list'>
print(len(output))  # 28
print(output[0].shape)  # (?, 256)
return output, fw_state, bw_state
# 单层动态双向LSTM
def single_layer_dynamic_bi_lstm(input, time_steps, hidden_size):
"""
:param input: 输入张量 形状为[batch_size,time_steps,input_size]
:param time_steps: 时序总数
:param hidden_size: gru单元输出的节点个数 即隐藏层节点数
:return: 返回单层动态双向LSTM单元的输出，以及cell状态
"""
lstm_fw_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size)  # 正向
lstm_bw_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size)  # 反向
# 动态rnn函数传入的是一个三维张量，[batch_size,time_steps,input_size]  输出是一个元组 每一个元素也是这种形状
output, state = tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,
cell_bw=lstm_bw_cell,
inputs=input,
dtype=tf.float32)
print(type(output))  # <class 'tuple'>
print(len(output))  # 2
print(output[0].shape)  # (?, 28, 128)
print(output[1].shape)  # (?, 28, 128)
output = tf.concat(output, axis=2)  # 按axis=2合并 (?,28,128) (?,28,128)按最后一维合并(?,28,256)
output = tf.transpose(output, [1, 0, 2])  # 注意这里输出需要转置  转换为时序优先的
return output, state
# 多层静态双向LSTM
def multi_layer_static_bi_lstm(input, time_steps, hidden_size):
"""
:param input: 输入张量 形状为[batch_size,time_steps,input_size]
:param time_steps: 时序总数
:param hidden_size: LSTM单元输出的节点个数 即隐藏层节点数
:return: 返回多层静态双向LSTM单元的输出，以及cell状态
"""
# 把输入input_x按列拆分，并返回一个有n_steps个张量组成的list
# 如batch_sizex28x28的输入拆成[(batch_size,28),((batch_size,28))....]
# 如果是调用的是静态rnn函数，需要这一步处理   即相当于把序列作为第一维度
input_x = tf.unstack(input, num=time_steps, axis=1)
stacked_fw_rnn = []
stacked_bw_rnn = []
for i in range(3):
stacked_fw_rnn.append(tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size))  # 正向
stacked_bw_rnn.append(tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size))  # 反向
# 静态rnn函数传入的是一个张量list  每一个元素都是一个(batch_size,input_size)大小的张量
# 这里的输出output是一个list 每一个元素都是前向输出,后向输出的合并
output, fw_state, bw_state = tf.contrib.rnn.stack_bidirectional_rnn(stacked_fw_rnn,
stacked_bw_rnn,
inputs=input_x,
dtype=tf.float32)
print(type(output))  # <class 'list'>
print(len(output))  # 28
print(output[0].shape)  # (?, 256)
return output, fw_state, bw_state
# 多层动态双向LSTM
def multi_layer_dynamic_bi_lstm(input, time_steps, hidden_size):
"""
:param input: 输入张量 形状为[batch_size,n_steps,input_size]
:param time_steps: 时序总数
:param hidden_size: gru单元输出的节点个数 即隐藏层节点数
:return: 返回多层动态双向LSTM单元的输出，以及cell状态
"""
stacked_fw_rnn = []
stacked_bw_rnn = []
for i in range(3):
stacked_fw_rnn.append(tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size))  # 正向
stacked_bw_rnn.append(tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_size))  # 反向
# 动态rnn函数传入的是一个三维张量，[batch_size,n_steps,input_size]  输出也是这种形状，
# input_size变成了正向和反向合并之后的 即input_size*2
output, fw_state, bw_state = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(stacked_fw_rnn,
stacked_bw_rnn,
inputs=input,
dtype=tf.float32)
print(type(output))  # <class 'tensorflow.python.framework.ops.Tensor'>
print(output.shape)  # (?, 28, 256)
output = tf.transpose(output, [1, 0, 2])  # 注意这里输出需要转置  转换为时序优先的
return output, fw_state, bw_state

def train(total_loss, global_step):
lr = tf.train.exponential_decay(0.01, global_step, decay_steps=350, decay_rate=0.1, staircase=True)
# 采用滑动平均的方法更新损失值
loss_averages = tf.train.ExponentialMovingAverage(decay=0.9, name='avg')
losses = tf.get_collection('losses')  # losses的列表
loss_averages_op = loss_averages.apply(losses + [total_loss])  # 计算损失值的影子变量op
# 计算梯度
with tf.control_dependencies([loss_averages_op]):  # 控制计算指定，只有执行了括号中的语句才能执行下面的语句
opt = tf.train.GradientDescentOptimizer(lr)  # 创建优化器
grads = opt.compute_gradients(total_loss)  # 计算梯度
# 应用梯度
apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
# 采用滑动平均的方法更新参数
variable_averages = tf.train.ExponentialMovingAverage(0.999, num_updates=global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
with tf.control_dependencies([apply_gradient_op, variables_averages_op]):
# tf.no_op()表示执行完apply_gradient_op, variable_averages_op操作之后什么都不做
train_op = tf.no_op(name='train')
return train_op

def load_model(sess, saver, checkpoint_dir):
"""加载模型，看看还能不能加一个功能，必须现在的检查检点是1000，但是我的train是100，要报错
还有就是读取之前的模型继续训练的问题
checkpoint_dir = checkpoint"""
# 通过checkpoint找到模型文件名
ckpt = tf.train.get_checkpoint_state(checkpoint_dir=checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
ckpt_name = os.path.basename(ckpt.model_checkpoint_path)  # 返回最新的chechpoint文件名 model.ckpt-1000
print("新的chechpoint文件名", ckpt_name)  # model.ckpt-2
saver.restore(sess, os.path.join(checkpoint_dir, ckpt_name))
# 现在不知道checkpoint文件名时怎样的，因此不知道里面如何运行
counter = int(next(re.finditer("(\d+)(?!.*\d)", ckpt_name)).group(0))  # 2
print(" [*] 成功模型 {}".format(ckpt_name))
return True, counter
else:
print(" [*] 找不到checkpoint")
return False, 0