实例55：RNN

# -*- coding: utf-8 -*-# 1. 定义基本函数
import copy, numpy as np
np.random.seed(0) #随机数生成器的种子，可以每次得到一样的值
# compute sigmoid nonlinearity
def sigmoid(x): #激活函数output = 1/(1+np.exp(-x))return output
# convert output of sigmoid function to its derivative
def sigmoid_output_to_derivative(output):#激活函数的导数return output*(1-output)# 2. 建立二进制映射
int2binary = {} #整数到其二进制表示的映射
binary_dim = 8 #暂时制作256以内的减法
## 计算0-256的二进制表示
largest_number = pow(2,binary_dim)
binary = np.unpackbits(np.array([range(largest_number)],dtype=np.uint8).T,axis=1)
for i in range(largest_number):int2binary[i] = binary[i]# 3. 定义参数
# input variables
alpha = 0.9 #学习速率
input_dim = 2 #输入的维度是2
hidden_dim = 16
output_dim = 1 #输出维度为1# initialize neural network weights
synapse_0 = (2*np.random.random((input_dim,hidden_dim)) - 1)*0.05 #维度为2*16， 2是输入维度，16是隐藏层维度
synapse_1 = (2*np.random.random((hidden_dim,output_dim)) - 1)*0.05
synapse_h = (2*np.random.random((hidden_dim,hidden_dim)) - 1)*0.05
# => [-0.05, 0.05)，# 用于存放反向传播的权重更新值
synapse_0_update = np.zeros_like(synapse_0)
synapse_1_update = np.zeros_like(synapse_1)
synapse_h_update = np.zeros_like(synapse_h)# 4. 准备样本数据
# training
for j in range(10000):#生成一个数字aa_int = np.random.randint(largest_number) #生成一个数字b,b的最大值取的是largest_number/2,作为被减数，让它小一点。b_int = np.random.randint(largest_number/2) #如果生成的b大了，那么交换一下if a_int<b_int:tt = a_intb_int = a_inta_int=tta = int2binary[a_int] # binary encodingb = int2binary[b_int] # binary encoding    # true answerc_int = a_int - b_intc = int2binary[c_int]# 5. 模型初始化# 存储神经网络的预测值d = np.zeros_like(c)overallError = 0 #每次把总误差清零layer_2_deltas = list() #存储每个时间点输出层的误差layer_1_values = list() #存储每个时间点隐藏层的值layer_1_values.append(np.ones(hidden_dim)*0.1) # 一开始没有隐藏层，所以初始化一下原始值为0.1# 6. 正向传播# moving along the positions in the binary encodingfor position in range(binary_dim):#循环遍历每一个二进制位# generate input and outputX = np.array([[a[binary_dim - position - 1],b[binary_dim - position - 1]]])#从右到左，每次去两个输入数字的一个bit位y = np.array([[c[binary_dim - position - 1]]]).T#正确答案# hidden layer (input ~+ prev_hidden)layer_1 = sigmoid(np.dot(X,synapse_0) + np.dot(layer_1_values[-1],synapse_h))#（输入层 + 之前的隐藏层） -> 新的隐藏层，这是体现循环神经网络的最核心的地方！！！# output layer (new binary representation)layer_2 = sigmoid(np.dot(layer_1,synapse_1)) #隐藏层 * 隐藏层到输出层的转化矩阵synapse_1 -> 输出层layer_2_error = y - layer_2 #预测误差layer_2_deltas.append((layer_2_error)*sigmoid_output_to_derivative(layer_2)) #把每一个时间点的误差导数都记录下来overallError += np.abs(layer_2_error[0])#总误差d[binary_dim - position - 1] = np.round(layer_2[0][0]) #记录下每一个预测bit位# store hidden layer so we can use it in the next timesteplayer_1_values.append(copy.deepcopy(layer_1))#记录下隐藏层的值，在下一个时间点用future_layer_1_delta = np.zeros(hidden_dim)# 7. 反向传播#反向传播，从最后一个时间点到第一个时间点for position in range(binary_dim):X = np.array([[a[position],b[position]]]) #最后一次的两个输入layer_1 = layer_1_values[-position-1] #当前时间点的隐藏层prev_layer_1 = layer_1_values[-position-2] #前一个时间点的隐藏层# error at output layerlayer_2_delta = layer_2_deltas[-position-1] #当前时间点输出层导数# error at hidden layer# 通过后一个时间点（因为是反向传播）的隐藏层误差和当前时间点的输出层误差，计算当前时间点的隐藏层误差layer_1_delta = (future_layer_1_delta.dot(synapse_h.T) + layer_2_delta.dot(synapse_1.T)) * sigmoid_output_to_derivative(layer_1)# 等到完成了所有反向传播误差计算， 才会更新权重矩阵，先暂时把更新矩阵存起来。synapse_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)synapse_h_update += np.atleast_2d(prev_layer_1).T.dot(layer_1_delta)synapse_0_update += X.T.dot(layer_1_delta)future_layer_1_delta = layer_1_delta# 完成所有反向传播之后，更新权重矩阵。并把矩阵变量清零synapse_0 += synapse_0_update * alphasynapse_1 += synapse_1_update * alphasynapse_h += synapse_h_update * alphasynapse_0_update *= 0synapse_1_update *= 0synapse_h_update *= 0# 8. 输出结果# print out progressif(j % 800 == 0):#print(synapse_0,synapse_h,synapse_1)print("总误差:" + str(overallError))print("Pred:" + str(d))print("True:" + str(c))out = 0for index,x in enumerate(reversed(d)):out += x*pow(2,index)print(str(a_int) + " - " + str(b_int) + " = " + str(out))print("------------")

实例56：使用RNN网络拟合回声信号序列

# -*- coding: utf-8 -*-# 1. 定义参数生成样本数据
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as pltnum_epochs = 5
total_series_length = 50000
truncated_backprop_length = 15
state_size = 4
num_classes = 2
echo_step = 3
batch_size = 5
num_batches = total_series_length//batch_size//truncated_backprop_lengthdef generateData():x = np.array(np.random.choice(2, total_series_length, p=[0.5, 0.5]))#在0 和1 中选择total_series_length个数y = np.roll(x, echo_step)#向右循环移位【1111000】---【0001111】y[0:echo_step] = 0x = x.reshape((batch_size, -1))  # 5,10000y = y.reshape((batch_size, -1))return (x, y)# 2. 定义占位符处理输入数据
batchX_placeholder = tf.placeholder(tf.float32, [batch_size, truncated_backprop_length])
batchY_placeholder = tf.placeholder(tf.int32, [batch_size, truncated_backprop_length])
init_state = tf.placeholder(tf.float32, [batch_size, state_size])# Unpack columns
inputs_series = tf.unstack(batchX_placeholder, axis=1)#truncated_backprop_length个序列
labels_series = tf.unstack(batchY_placeholder, axis=1)# 3. 定义网络结构
current_state = init_state
predictions_series = []
losses =[]
for current_input, labels in zip(inputs_series,labels_series):
#for current_input in inputs_series:current_input = tf.reshape(current_input, [batch_size, 1])input_and_state_concatenated = tf.concat([current_input, current_state],1)  # current_state 4  +1next_state = tf.contrib.layers.fully_connected(input_and_state_concatenated,state_size,activation_fn=tf.tanh)current_state = next_statelogits =tf.contrib.layers.fully_connected(next_state,num_classes,activation_fn=None)loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels,logits=logits)losses.append(loss)predictions = tf.nn.softmax(logits)predictions_series.append(predictions)total_loss = tf.reduce_mean(losses)
train_step = tf.train.AdagradOptimizer(0.3).minimize(total_loss)# plot 函数
def plot(loss_list, predictions_series, batchX, batchY):plt.subplot(2, 3, 1)plt.cla()plt.plot(loss_list)for batch_series_idx in range(batch_size):one_hot_output_series = np.array(predictions_series)[:, batch_series_idx, :]single_output_series = np.array([(1 if out[0] < 0.5 else 0) for out in one_hot_output_series])plt.subplot(2, 3, batch_series_idx + 2)plt.cla()plt.axis([0, truncated_backprop_length, 0, 2])left_offset = range(truncated_backprop_length)left_offset2 = range(echo_step,truncated_backprop_length+echo_step)label1 = "past values"label2 = "True echo values" label3 = "Predictions"      plt.plot(left_offset2, batchX[batch_series_idx, :]*0.2+1.5, "o--b", label=label1)plt.plot(left_offset, batchY[batch_series_idx, :]*0.2+0.8,"x--b", label=label2)plt.plot(left_offset,  single_output_series*0.2+0.1 , "o--y", label=label3)plt.legend(loc='best')plt.draw()plt.pause(0.0001)# 4. 建立session训练数据
with tf.Session() as sess:sess.run(tf.global_variables_initializer())plt.ion()plt.figure()plt.show()loss_list = []for epoch_idx in range(num_epochs):x,y = generateData()_current_state = np.zeros((batch_size, state_size))print("New data, epoch", epoch_idx)for batch_idx in range(num_batches):#50000/ 5 /15=分成多少段start_idx = batch_idx * truncated_backprop_lengthend_idx = start_idx + truncated_backprop_lengthbatchX = x[:,start_idx:end_idx]batchY = y[:,start_idx:end_idx]_total_loss, _train_step, _current_state, _predictions_series = sess.run([total_loss, train_step, current_state, predictions_series],feed_dict={batchX_placeholder:batchX,batchY_placeholder:batchY,init_state:_current_state})loss_list.append(_total_loss)# 5. 测试模型及可视化if batch_idx%100 == 0:print("Step",batch_idx, "Loss", _total_loss)plot(loss_list, _predictions_series, batchX, batchY)plt.ioff()
plt.show()    

实例57：构造LSTM对MNIST分类(BasicCell & LSTMCell)

# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)#1 BasicLSTMCell
lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)#3 gru
#gru = tf.contrib.rnn.GRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)#4 创建动态RNN
#outputs,_  = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 2.100226, Training Accuracy= 0.35938
Iter 2560, Minibatch Loss= 1.727162, Training Accuracy= 0.46875
Iter 3840, Minibatch Loss= 1.655166, Training Accuracy= 0.41406
Iter 5120, Minibatch Loss= 1.134178, Training Accuracy= 0.60938
Iter 6400, Minibatch Loss= 1.077124, Training Accuracy= 0.70312
Iter 7680, Minibatch Loss= 0.919439, Training Accuracy= 0.74219
Iter 8960, Minibatch Loss= 0.661384, Training Accuracy= 0.77344
Iter 10240, Minibatch Loss= 0.611414, Training Accuracy= 0.79688
Iter 11520, Minibatch Loss= 0.649199, Training Accuracy= 0.77344
Iter 12800, Minibatch Loss= 0.732224, Training Accuracy= 0.75781
Iter 14080, Minibatch Loss= 0.520225, Training Accuracy= 0.84375
Iter 15360, Minibatch Loss= 0.435234, Training Accuracy= 0.86719
Iter 16640, Minibatch Loss= 0.354433, Training Accuracy= 0.91406
Iter 17920, Minibatch Loss= 0.391198, Training Accuracy= 0.85938
Iter 19200, Minibatch Loss= 0.574429, Training Accuracy= 0.80469
Iter 20480, Minibatch Loss= 0.508154, Training Accuracy= 0.83594
Iter 21760, Minibatch Loss= 0.281986, Training Accuracy= 0.91406
Iter 23040, Minibatch Loss= 0.427695, Training Accuracy= 0.86719
Iter 24320, Minibatch Loss= 0.286359, Training Accuracy= 0.92969
Iter 25600, Minibatch Loss= 0.278457, Training Accuracy= 0.92188
Iter 26880, Minibatch Loss= 0.228263, Training Accuracy= 0.95312
Iter 28160, Minibatch Loss= 0.280953, Training Accuracy= 0.90625
Iter 29440, Minibatch Loss= 0.234286, Training Accuracy= 0.93750
Iter 30720, Minibatch Loss= 0.310759, Training Accuracy= 0.89062
Iter 32000, Minibatch Loss= 0.365036, Training Accuracy= 0.91406
Iter 33280, Minibatch Loss= 0.264047, Training Accuracy= 0.93750
Iter 34560, Minibatch Loss= 0.290281, Training Accuracy= 0.90625
Iter 35840, Minibatch Loss= 0.196424, Training Accuracy= 0.92969
Iter 37120, Minibatch Loss= 0.198498, Training Accuracy= 0.94531
Iter 38400, Minibatch Loss= 0.342719, Training Accuracy= 0.89844
Iter 39680, Minibatch Loss= 0.319853, Training Accuracy= 0.90625
Iter 40960, Minibatch Loss= 0.209966, Training Accuracy= 0.94531
Iter 42240, Minibatch Loss= 0.109240, Training Accuracy= 0.96875
Iter 43520, Minibatch Loss= 0.245075, Training Accuracy= 0.92188
Iter 44800, Minibatch Loss= 0.169283, Training Accuracy= 0.95312
Iter 46080, Minibatch Loss= 0.141722, Training Accuracy= 0.94531
Iter 47360, Minibatch Loss= 0.139368, Training Accuracy= 0.96875
Iter 48640, Minibatch Loss= 0.129791, Training Accuracy= 0.95312
Iter 49920, Minibatch Loss= 0.135869, Training Accuracy= 0.95312
Iter 51200, Minibatch Loss= 0.167792, Training Accuracy= 0.92969
Iter 52480, Minibatch Loss= 0.219980, Training Accuracy= 0.94531
Iter 53760, Minibatch Loss= 0.253420, Training Accuracy= 0.89844
Iter 55040, Minibatch Loss= 0.310215, Training Accuracy= 0.91406
Iter 56320, Minibatch Loss= 0.140659, Training Accuracy= 0.94531
Iter 57600, Minibatch Loss= 0.147263, Training Accuracy= 0.96094
Iter 58880, Minibatch Loss= 0.145595, Training Accuracy= 0.96875
Iter 60160, Minibatch Loss= 0.140386, Training Accuracy= 0.94531
Iter 61440, Minibatch Loss= 0.112856, Training Accuracy= 0.96094
Iter 62720, Minibatch Loss= 0.066107, Training Accuracy= 0.98438
Iter 64000, Minibatch Loss= 0.141188, Training Accuracy= 0.94531
Iter 65280, Minibatch Loss= 0.124689, Training Accuracy= 0.97656
Iter 66560, Minibatch Loss= 0.136360, Training Accuracy= 0.96094
Iter 67840, Minibatch Loss= 0.127406, Training Accuracy= 0.94531
Iter 69120, Minibatch Loss= 0.190868, Training Accuracy= 0.94531
Iter 70400, Minibatch Loss= 0.146545, Training Accuracy= 0.94531
Iter 71680, Minibatch Loss= 0.144414, Training Accuracy= 0.95312
Iter 72960, Minibatch Loss= 0.139988, Training Accuracy= 0.93750
Iter 74240, Minibatch Loss= 0.078669, Training Accuracy= 0.97656
Iter 75520, Minibatch Loss= 0.194032, Training Accuracy= 0.93750
Iter 76800, Minibatch Loss= 0.123782, Training Accuracy= 0.96875
Iter 78080, Minibatch Loss= 0.153625, Training Accuracy= 0.96094
Iter 79360, Minibatch Loss= 0.125683, Training Accuracy= 0.97656
Iter 80640, Minibatch Loss= 0.146972, Training Accuracy= 0.93750
Iter 81920, Minibatch Loss= 0.175487, Training Accuracy= 0.96094
Iter 83200, Minibatch Loss= 0.078552, Training Accuracy= 0.98438
Iter 84480, Minibatch Loss= 0.136101, Training Accuracy= 0.95312
Iter 85760, Minibatch Loss= 0.141502, Training Accuracy= 0.95312
Iter 87040, Minibatch Loss= 0.068094, Training Accuracy= 0.98438
Iter 88320, Minibatch Loss= 0.289425, Training Accuracy= 0.89844
Iter 89600, Minibatch Loss= 0.127497, Training Accuracy= 0.95312
Iter 90880, Minibatch Loss= 0.090173, Training Accuracy= 0.97656
Iter 92160, Minibatch Loss= 0.135557, Training Accuracy= 0.95312
Iter 93440, Minibatch Loss= 0.122439, Training Accuracy= 0.96094
Iter 94720, Minibatch Loss= 0.171625, Training Accuracy= 0.94531
Iter 96000, Minibatch Loss= 0.140195, Training Accuracy= 0.94531
Iter 97280, Minibatch Loss= 0.156978, Training Accuracy= 0.96094
Iter 98560, Minibatch Loss= 0.050950, Training Accuracy= 0.99219
Iter 99840, Minibatch Loss= 0.086917, Training Accuracy= 0.97656Finished!
Testing Accuracy: 0.9921875

# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)#2 LSTMCell
lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)#3 gru
#gru = tf.contrib.rnn.GRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)#4 创建动态RNN
#outputs,_  = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 2.132512, Training Accuracy= 0.30469
Iter 2560, Minibatch Loss= 2.001817, Training Accuracy= 0.28125
Iter 3840, Minibatch Loss= 1.523742, Training Accuracy= 0.51562
Iter 5120, Minibatch Loss= 1.272301, Training Accuracy= 0.64844
Iter 6400, Minibatch Loss= 1.060405, Training Accuracy= 0.65625
Iter 7680, Minibatch Loss= 0.889486, Training Accuracy= 0.69531
Iter 8960, Minibatch Loss= 0.854698, Training Accuracy= 0.69531
Iter 10240, Minibatch Loss= 0.786288, Training Accuracy= 0.77344
Iter 11520, Minibatch Loss= 0.758092, Training Accuracy= 0.77344
Iter 12800, Minibatch Loss= 0.784820, Training Accuracy= 0.77344
Iter 14080, Minibatch Loss= 0.383889, Training Accuracy= 0.91406
Iter 15360, Minibatch Loss= 0.458686, Training Accuracy= 0.82031
Iter 16640, Minibatch Loss= 0.446535, Training Accuracy= 0.85156
Iter 17920, Minibatch Loss= 0.593214, Training Accuracy= 0.83594
Iter 19200, Minibatch Loss= 0.483660, Training Accuracy= 0.85156
Iter 20480, Minibatch Loss= 0.413878, Training Accuracy= 0.87500
Iter 21760, Minibatch Loss= 0.438487, Training Accuracy= 0.85938
Iter 23040, Minibatch Loss= 0.440690, Training Accuracy= 0.89844
Iter 24320, Minibatch Loss= 0.395759, Training Accuracy= 0.88281
Iter 25600, Minibatch Loss= 0.303816, Training Accuracy= 0.91406
Iter 26880, Minibatch Loss= 0.457287, Training Accuracy= 0.85156
Iter 28160, Minibatch Loss= 0.435749, Training Accuracy= 0.89062
Iter 29440, Minibatch Loss= 0.282319, Training Accuracy= 0.92188
Iter 30720, Minibatch Loss= 0.341908, Training Accuracy= 0.90625
Iter 32000, Minibatch Loss= 0.413945, Training Accuracy= 0.84375
Iter 33280, Minibatch Loss= 0.277253, Training Accuracy= 0.90625
Iter 34560, Minibatch Loss= 0.265192, Training Accuracy= 0.92188
Iter 35840, Minibatch Loss= 0.273202, Training Accuracy= 0.89844
Iter 37120, Minibatch Loss= 0.188953, Training Accuracy= 0.95312
Iter 38400, Minibatch Loss= 0.401897, Training Accuracy= 0.89844
Iter 39680, Minibatch Loss= 0.277849, Training Accuracy= 0.91406
Iter 40960, Minibatch Loss= 0.256338, Training Accuracy= 0.91406
Iter 42240, Minibatch Loss= 0.259410, Training Accuracy= 0.92188
Iter 43520, Minibatch Loss= 0.286601, Training Accuracy= 0.95312
Iter 44800, Minibatch Loss= 0.163870, Training Accuracy= 0.96094
Iter 46080, Minibatch Loss= 0.161053, Training Accuracy= 0.96094
Iter 47360, Minibatch Loss= 0.250546, Training Accuracy= 0.92969
Iter 48640, Minibatch Loss= 0.134387, Training Accuracy= 0.96094
Iter 49920, Minibatch Loss= 0.173047, Training Accuracy= 0.92969
Iter 51200, Minibatch Loss= 0.337741, Training Accuracy= 0.90625
Iter 52480, Minibatch Loss= 0.207449, Training Accuracy= 0.93750
Iter 53760, Minibatch Loss= 0.156933, Training Accuracy= 0.94531
Iter 55040, Minibatch Loss= 0.199040, Training Accuracy= 0.94531
Iter 56320, Minibatch Loss= 0.147556, Training Accuracy= 0.95312
Iter 57600, Minibatch Loss= 0.148920, Training Accuracy= 0.94531
Iter 58880, Minibatch Loss= 0.144406, Training Accuracy= 0.94531
Iter 60160, Minibatch Loss= 0.114759, Training Accuracy= 0.96875
Iter 61440, Minibatch Loss= 0.145373, Training Accuracy= 0.95312
Iter 62720, Minibatch Loss= 0.154870, Training Accuracy= 0.95312
Iter 64000, Minibatch Loss= 0.176493, Training Accuracy= 0.95312
Iter 65280, Minibatch Loss= 0.124761, Training Accuracy= 0.96094
Iter 66560, Minibatch Loss= 0.060221, Training Accuracy= 0.98438
Iter 67840, Minibatch Loss= 0.154883, Training Accuracy= 0.95312
Iter 69120, Minibatch Loss= 0.141949, Training Accuracy= 0.95312
Iter 70400, Minibatch Loss= 0.289304, Training Accuracy= 0.91406
Iter 71680, Minibatch Loss= 0.167781, Training Accuracy= 0.94531
Iter 72960, Minibatch Loss= 0.239404, Training Accuracy= 0.93750
Iter 74240, Minibatch Loss= 0.138790, Training Accuracy= 0.96094
Iter 75520, Minibatch Loss= 0.176216, Training Accuracy= 0.92969
Iter 76800, Minibatch Loss= 0.179790, Training Accuracy= 0.95312
Iter 78080, Minibatch Loss= 0.157953, Training Accuracy= 0.93750
Iter 79360, Minibatch Loss= 0.213766, Training Accuracy= 0.92188
Iter 80640, Minibatch Loss= 0.080645, Training Accuracy= 0.97656
Iter 81920, Minibatch Loss= 0.191703, Training Accuracy= 0.95312
Iter 83200, Minibatch Loss= 0.172994, Training Accuracy= 0.96094
Iter 84480, Minibatch Loss= 0.210067, Training Accuracy= 0.93750
Iter 85760, Minibatch Loss= 0.165411, Training Accuracy= 0.94531
Iter 87040, Minibatch Loss= 0.087349, Training Accuracy= 0.97656
Iter 88320, Minibatch Loss= 0.189240, Training Accuracy= 0.93750
Iter 89600, Minibatch Loss= 0.153909, Training Accuracy= 0.94531
Iter 90880, Minibatch Loss= 0.231399, Training Accuracy= 0.92188
Iter 92160, Minibatch Loss= 0.132192, Training Accuracy= 0.95312
Iter 93440, Minibatch Loss= 0.078557, Training Accuracy= 0.98438
Iter 94720, Minibatch Loss= 0.081286, Training Accuracy= 0.98438
Iter 96000, Minibatch Loss= 0.154948, Training Accuracy= 0.96094
Iter 97280, Minibatch Loss= 0.132285, Training Accuracy= 0.95312
Iter 98560, Minibatch Loss= 0.107341, Training Accuracy= 0.97656
Iter 99840, Minibatch Loss= 0.068848, Training Accuracy= 1.00000Finished!
Testing Accuracy: 0.984375

实例58：构造单层GRU网络对MNIST数据集分类

# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)#3 gru
gru = tf.contrib.rnn.GRUCell(n_hidden)
outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)#4 创建动态RNN
#outputs,_  = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 2.110982, Training Accuracy= 0.35156
Iter 2560, Minibatch Loss= 1.813759, Training Accuracy= 0.36719
Iter 3840, Minibatch Loss= 1.430715, Training Accuracy= 0.57812
Iter 5120, Minibatch Loss= 1.243752, Training Accuracy= 0.53906
Iter 6400, Minibatch Loss= 1.036775, Training Accuracy= 0.66406
Iter 7680, Minibatch Loss= 0.958580, Training Accuracy= 0.65625
Iter 8960, Minibatch Loss= 0.797127, Training Accuracy= 0.75000
Iter 10240, Minibatch Loss= 0.834968, Training Accuracy= 0.71094
Iter 11520, Minibatch Loss= 0.762974, Training Accuracy= 0.72656
Iter 12800, Minibatch Loss= 0.702314, Training Accuracy= 0.78125
Iter 14080, Minibatch Loss= 0.631187, Training Accuracy= 0.75000
Iter 15360, Minibatch Loss= 0.601635, Training Accuracy= 0.82031
Iter 16640, Minibatch Loss= 0.539469, Training Accuracy= 0.84375
Iter 17920, Minibatch Loss= 0.446828, Training Accuracy= 0.89844
Iter 19200, Minibatch Loss= 0.491736, Training Accuracy= 0.86719
Iter 20480, Minibatch Loss= 0.443790, Training Accuracy= 0.85156
Iter 21760, Minibatch Loss= 0.360417, Training Accuracy= 0.90625
Iter 23040, Minibatch Loss= 0.399639, Training Accuracy= 0.89844
Iter 24320, Minibatch Loss= 0.365533, Training Accuracy= 0.90625
Iter 25600, Minibatch Loss= 0.361865, Training Accuracy= 0.89062
Iter 26880, Minibatch Loss= 0.314257, Training Accuracy= 0.89844
Iter 28160, Minibatch Loss= 0.323813, Training Accuracy= 0.88281
Iter 29440, Minibatch Loss= 0.258019, Training Accuracy= 0.92969
Iter 30720, Minibatch Loss= 0.362180, Training Accuracy= 0.90625
Iter 32000, Minibatch Loss= 0.300678, Training Accuracy= 0.90625
Iter 33280, Minibatch Loss= 0.345444, Training Accuracy= 0.88281
Iter 34560, Minibatch Loss= 0.315548, Training Accuracy= 0.90625
Iter 35840, Minibatch Loss= 0.149648, Training Accuracy= 0.94531
Iter 37120, Minibatch Loss= 0.208241, Training Accuracy= 0.95312
Iter 38400, Minibatch Loss= 0.240656, Training Accuracy= 0.92188
Iter 39680, Minibatch Loss= 0.242481, Training Accuracy= 0.94531
Iter 40960, Minibatch Loss= 0.198655, Training Accuracy= 0.94531
Iter 42240, Minibatch Loss= 0.169093, Training Accuracy= 0.96094
Iter 43520, Minibatch Loss= 0.144338, Training Accuracy= 0.96094
Iter 44800, Minibatch Loss= 0.120352, Training Accuracy= 0.96875
Iter 46080, Minibatch Loss= 0.260023, Training Accuracy= 0.92969
Iter 47360, Minibatch Loss= 0.259179, Training Accuracy= 0.92188
Iter 48640, Minibatch Loss= 0.169763, Training Accuracy= 0.95312
Iter 49920, Minibatch Loss= 0.295729, Training Accuracy= 0.87500
Iter 51200, Minibatch Loss= 0.262093, Training Accuracy= 0.94531
Iter 52480, Minibatch Loss= 0.160087, Training Accuracy= 0.94531
Iter 53760, Minibatch Loss= 0.216548, Training Accuracy= 0.92969
Iter 55040, Minibatch Loss= 0.197999, Training Accuracy= 0.92969
Iter 56320, Minibatch Loss= 0.209844, Training Accuracy= 0.93750
Iter 57600, Minibatch Loss= 0.145806, Training Accuracy= 0.96094
Iter 58880, Minibatch Loss= 0.145609, Training Accuracy= 0.92969
Iter 60160, Minibatch Loss= 0.141855, Training Accuracy= 0.96875
Iter 61440, Minibatch Loss= 0.149774, Training Accuracy= 0.96094
Iter 62720, Minibatch Loss= 0.151638, Training Accuracy= 0.95312
Iter 64000, Minibatch Loss= 0.183334, Training Accuracy= 0.95312
Iter 65280, Minibatch Loss= 0.135904, Training Accuracy= 0.96094
Iter 66560, Minibatch Loss= 0.099675, Training Accuracy= 0.97656
Iter 67840, Minibatch Loss= 0.122692, Training Accuracy= 0.96094
Iter 69120, Minibatch Loss= 0.146226, Training Accuracy= 0.94531
Iter 70400, Minibatch Loss= 0.180213, Training Accuracy= 0.96094
Iter 71680, Minibatch Loss= 0.095388, Training Accuracy= 0.96875
Iter 72960, Minibatch Loss= 0.151607, Training Accuracy= 0.95312
Iter 74240, Minibatch Loss= 0.138024, Training Accuracy= 0.95312
Iter 75520, Minibatch Loss= 0.077417, Training Accuracy= 0.98438
Iter 76800, Minibatch Loss= 0.142044, Training Accuracy= 0.96094
Iter 78080, Minibatch Loss= 0.096436, Training Accuracy= 0.96875
Iter 79360, Minibatch Loss= 0.170768, Training Accuracy= 0.94531
Iter 80640, Minibatch Loss= 0.244333, Training Accuracy= 0.94531
Iter 81920, Minibatch Loss= 0.049825, Training Accuracy= 0.98438
Iter 83200, Minibatch Loss= 0.100651, Training Accuracy= 0.96094
Iter 84480, Minibatch Loss= 0.100706, Training Accuracy= 0.96875
Iter 85760, Minibatch Loss= 0.125473, Training Accuracy= 0.96094
Iter 87040, Minibatch Loss= 0.049680, Training Accuracy= 0.99219
Iter 88320, Minibatch Loss= 0.077539, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.124252, Training Accuracy= 0.96094
Iter 90880, Minibatch Loss= 0.039592, Training Accuracy= 0.99219
Iter 92160, Minibatch Loss= 0.108718, Training Accuracy= 0.96875
Iter 93440, Minibatch Loss= 0.085434, Training Accuracy= 0.97656
Iter 94720, Minibatch Loss= 0.164223, Training Accuracy= 0.95312
Iter 96000, Minibatch Loss= 0.174773, Training Accuracy= 0.92969
Iter 97280, Minibatch Loss= 0.062409, Training Accuracy= 0.98438
Iter 98560, Minibatch Loss= 0.065892, Training Accuracy= 0.98438
Iter 99840, Minibatch Loss= 0.109494, Training Accuracy= 0.96875Finished!
Testing Accuracy: 0.96875

实例59：创建动态单层RNN对MNIST数据集分类

# -*- coding: utf-8 -*-
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)#3 gru
gru = tf.contrib.rnn.GRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)#4 创建动态RNN
outputs,_  = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
outputs = tf.transpose(outputs, [1, 0, 2])pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 2.117727, Training Accuracy= 0.28906
Iter 2560, Minibatch Loss= 1.873549, Training Accuracy= 0.37500
Iter 3840, Minibatch Loss= 1.643477, Training Accuracy= 0.46875
Iter 5120, Minibatch Loss= 1.384699, Training Accuracy= 0.54688
Iter 6400, Minibatch Loss= 1.180172, Training Accuracy= 0.60938
Iter 7680, Minibatch Loss= 1.117720, Training Accuracy= 0.59375
Iter 8960, Minibatch Loss= 0.758485, Training Accuracy= 0.72656
Iter 10240, Minibatch Loss= 0.766036, Training Accuracy= 0.76562
Iter 11520, Minibatch Loss= 0.644146, Training Accuracy= 0.80469
Iter 12800, Minibatch Loss= 0.611225, Training Accuracy= 0.82812
Iter 14080, Minibatch Loss= 0.442834, Training Accuracy= 0.84375
Iter 15360, Minibatch Loss= 0.449818, Training Accuracy= 0.88281
Iter 16640, Minibatch Loss= 0.438875, Training Accuracy= 0.87500
Iter 17920, Minibatch Loss= 0.267779, Training Accuracy= 0.91406
Iter 19200, Minibatch Loss= 0.243440, Training Accuracy= 0.95312
Iter 20480, Minibatch Loss= 0.444132, Training Accuracy= 0.85938
Iter 21760, Minibatch Loss= 0.346954, Training Accuracy= 0.90625
Iter 23040, Minibatch Loss= 0.175107, Training Accuracy= 0.95312
Iter 24320, Minibatch Loss= 0.268770, Training Accuracy= 0.92969
Iter 25600, Minibatch Loss= 0.297496, Training Accuracy= 0.91406
Iter 26880, Minibatch Loss= 0.274186, Training Accuracy= 0.89844
Iter 28160, Minibatch Loss= 0.153380, Training Accuracy= 0.94531
Iter 29440, Minibatch Loss= 0.223070, Training Accuracy= 0.93750
Iter 30720, Minibatch Loss= 0.187295, Training Accuracy= 0.92188
Iter 32000, Minibatch Loss= 0.192284, Training Accuracy= 0.94531
Iter 33280, Minibatch Loss= 0.206862, Training Accuracy= 0.93750
Iter 34560, Minibatch Loss= 0.157154, Training Accuracy= 0.94531
Iter 35840, Minibatch Loss= 0.221760, Training Accuracy= 0.92188
Iter 37120, Minibatch Loss= 0.204846, Training Accuracy= 0.94531
Iter 38400, Minibatch Loss= 0.143488, Training Accuracy= 0.96094
Iter 39680, Minibatch Loss= 0.147712, Training Accuracy= 0.95312
Iter 40960, Minibatch Loss= 0.260166, Training Accuracy= 0.92188
Iter 42240, Minibatch Loss= 0.236482, Training Accuracy= 0.92188
Iter 43520, Minibatch Loss= 0.222228, Training Accuracy= 0.93750
Iter 44800, Minibatch Loss= 0.135309, Training Accuracy= 0.96094
Iter 46080, Minibatch Loss= 0.148444, Training Accuracy= 0.96094
Iter 47360, Minibatch Loss= 0.284828, Training Accuracy= 0.92188
Iter 48640, Minibatch Loss= 0.276086, Training Accuracy= 0.92188
Iter 49920, Minibatch Loss= 0.111534, Training Accuracy= 0.95312
Iter 51200, Minibatch Loss= 0.111628, Training Accuracy= 0.96094
Iter 52480, Minibatch Loss= 0.131188, Training Accuracy= 0.96875
Iter 53760, Minibatch Loss= 0.080626, Training Accuracy= 0.98438
Iter 55040, Minibatch Loss= 0.118804, Training Accuracy= 0.96094
Iter 56320, Minibatch Loss= 0.211226, Training Accuracy= 0.92969
Iter 57600, Minibatch Loss= 0.165107, Training Accuracy= 0.95312
Iter 58880, Minibatch Loss= 0.134429, Training Accuracy= 0.96094
Iter 60160, Minibatch Loss= 0.168475, Training Accuracy= 0.95312
Iter 61440, Minibatch Loss= 0.128811, Training Accuracy= 0.96094
Iter 62720, Minibatch Loss= 0.077244, Training Accuracy= 0.96094
Iter 64000, Minibatch Loss= 0.090807, Training Accuracy= 0.96875
Iter 65280, Minibatch Loss= 0.192585, Training Accuracy= 0.92188
Iter 66560, Minibatch Loss= 0.197945, Training Accuracy= 0.95312
Iter 67840, Minibatch Loss= 0.146314, Training Accuracy= 0.96875
Iter 69120, Minibatch Loss= 0.117953, Training Accuracy= 0.96875
Iter 70400, Minibatch Loss= 0.174820, Training Accuracy= 0.96875
Iter 71680, Minibatch Loss= 0.078789, Training Accuracy= 0.98438
Iter 72960, Minibatch Loss= 0.180256, Training Accuracy= 0.92188
Iter 74240, Minibatch Loss= 0.142464, Training Accuracy= 0.94531
Iter 75520, Minibatch Loss= 0.112260, Training Accuracy= 0.96875
Iter 76800, Minibatch Loss= 0.133794, Training Accuracy= 0.95312
Iter 78080, Minibatch Loss= 0.067401, Training Accuracy= 0.99219
Iter 79360, Minibatch Loss= 0.154360, Training Accuracy= 0.96094
Iter 80640, Minibatch Loss= 0.067086, Training Accuracy= 0.97656
Iter 81920, Minibatch Loss= 0.110198, Training Accuracy= 0.97656
Iter 83200, Minibatch Loss= 0.149606, Training Accuracy= 0.95312
Iter 84480, Minibatch Loss= 0.119215, Training Accuracy= 0.96875
Iter 85760, Minibatch Loss= 0.051235, Training Accuracy= 0.98438
Iter 87040, Minibatch Loss= 0.094870, Training Accuracy= 0.96875
Iter 88320, Minibatch Loss= 0.073898, Training Accuracy= 0.98438
Iter 89600, Minibatch Loss= 0.087323, Training Accuracy= 0.96094
Iter 90880, Minibatch Loss= 0.088403, Training Accuracy= 0.96094
Iter 92160, Minibatch Loss= 0.120743, Training Accuracy= 0.97656
Iter 93440, Minibatch Loss= 0.061005, Training Accuracy= 0.98438
Iter 94720, Minibatch Loss= 0.095413, Training Accuracy= 0.97656
Iter 96000, Minibatch Loss= 0.078382, Training Accuracy= 0.97656
Iter 97280, Minibatch Loss= 0.100441, Training Accuracy= 0.97656
Iter 98560, Minibatch Loss= 0.022794, Training Accuracy= 1.00000
Iter 99840, Minibatch Loss= 0.085918, Training Accuracy= 0.97656Finished!
Testing Accuracy: 0.984375

实例60：静态多层LSTM对MNIST数据集分类

# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)
batch_size = 128tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])stacked_rnn = []
for i in range(3):stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)x1 = tf.unstack(x, n_steps, 1)
outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))training_iters = 100000display_step = 10# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 100test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 1.892138, Training Accuracy= 0.27344
Iter 2560, Minibatch Loss= 1.520981, Training Accuracy= 0.42969
Iter 3840, Minibatch Loss= 1.271327, Training Accuracy= 0.53906
Iter 5120, Minibatch Loss= 0.980068, Training Accuracy= 0.65625
Iter 6400, Minibatch Loss= 0.812825, Training Accuracy= 0.74219
Iter 7680, Minibatch Loss= 0.807655, Training Accuracy= 0.74219
Iter 8960, Minibatch Loss= 0.761637, Training Accuracy= 0.77344
Iter 10240, Minibatch Loss= 0.580690, Training Accuracy= 0.82812
Iter 11520, Minibatch Loss= 0.802299, Training Accuracy= 0.72656
Iter 12800, Minibatch Loss= 0.557632, Training Accuracy= 0.82031
Iter 14080, Minibatch Loss= 0.580068, Training Accuracy= 0.85938
Iter 15360, Minibatch Loss= 0.365136, Training Accuracy= 0.88281
Iter 16640, Minibatch Loss= 0.554880, Training Accuracy= 0.83594
Iter 17920, Minibatch Loss= 0.392231, Training Accuracy= 0.88281
Iter 19200, Minibatch Loss= 0.239315, Training Accuracy= 0.92969
Iter 20480, Minibatch Loss= 0.266329, Training Accuracy= 0.92969
Iter 21760, Minibatch Loss= 0.239885, Training Accuracy= 0.93750
Iter 23040, Minibatch Loss= 0.308746, Training Accuracy= 0.89062
Iter 24320, Minibatch Loss= 0.405082, Training Accuracy= 0.86719
Iter 25600, Minibatch Loss= 0.306493, Training Accuracy= 0.92969
Iter 26880, Minibatch Loss= 0.297999, Training Accuracy= 0.92188
Iter 28160, Minibatch Loss= 0.205317, Training Accuracy= 0.94531
Iter 29440, Minibatch Loss= 0.209697, Training Accuracy= 0.92188
Iter 30720, Minibatch Loss= 0.224113, Training Accuracy= 0.92188
Iter 32000, Minibatch Loss= 0.202094, Training Accuracy= 0.93750
Iter 33280, Minibatch Loss= 0.209670, Training Accuracy= 0.92188
Iter 34560, Minibatch Loss= 0.275021, Training Accuracy= 0.92188
Iter 35840, Minibatch Loss= 0.141351, Training Accuracy= 0.95312
Iter 37120, Minibatch Loss= 0.172745, Training Accuracy= 0.92188
Iter 38400, Minibatch Loss= 0.228708, Training Accuracy= 0.92969
Iter 39680, Minibatch Loss= 0.245980, Training Accuracy= 0.92188
Iter 40960, Minibatch Loss= 0.216683, Training Accuracy= 0.92188
Iter 42240, Minibatch Loss= 0.183437, Training Accuracy= 0.96094
Iter 43520, Minibatch Loss= 0.249385, Training Accuracy= 0.92969
Iter 44800, Minibatch Loss= 0.275260, Training Accuracy= 0.91406
Iter 46080, Minibatch Loss= 0.174156, Training Accuracy= 0.92969
Iter 47360, Minibatch Loss= 0.261219, Training Accuracy= 0.90625
Iter 48640, Minibatch Loss= 0.187510, Training Accuracy= 0.93750
Iter 49920, Minibatch Loss= 0.090633, Training Accuracy= 0.98438
Iter 51200, Minibatch Loss= 0.061939, Training Accuracy= 0.97656
Iter 52480, Minibatch Loss= 0.075695, Training Accuracy= 0.97656
Iter 53760, Minibatch Loss= 0.322783, Training Accuracy= 0.92188
Iter 55040, Minibatch Loss= 0.126282, Training Accuracy= 0.96875
Iter 56320, Minibatch Loss= 0.119054, Training Accuracy= 0.96875
Iter 57600, Minibatch Loss= 0.213464, Training Accuracy= 0.94531
Iter 58880, Minibatch Loss= 0.083522, Training Accuracy= 0.96094
Iter 60160, Minibatch Loss= 0.094831, Training Accuracy= 0.98438
Iter 61440, Minibatch Loss= 0.096521, Training Accuracy= 0.96875
Iter 62720, Minibatch Loss= 0.173445, Training Accuracy= 0.93750
Iter 64000, Minibatch Loss= 0.119984, Training Accuracy= 0.96875
Iter 65280, Minibatch Loss= 0.180508, Training Accuracy= 0.95312
Iter 66560, Minibatch Loss= 0.068011, Training Accuracy= 0.98438
Iter 67840, Minibatch Loss= 0.210539, Training Accuracy= 0.95312
Iter 69120, Minibatch Loss= 0.097277, Training Accuracy= 0.96875
Iter 70400, Minibatch Loss= 0.196134, Training Accuracy= 0.95312
Iter 71680, Minibatch Loss= 0.091252, Training Accuracy= 0.96094
Iter 72960, Minibatch Loss= 0.144938, Training Accuracy= 0.95312
Iter 74240, Minibatch Loss= 0.101681, Training Accuracy= 0.96875
Iter 75520, Minibatch Loss= 0.057837, Training Accuracy= 0.97656
Iter 76800, Minibatch Loss= 0.095161, Training Accuracy= 0.97656
Iter 78080, Minibatch Loss= 0.091807, Training Accuracy= 0.98438
Iter 79360, Minibatch Loss= 0.063520, Training Accuracy= 0.99219
Iter 80640, Minibatch Loss= 0.087506, Training Accuracy= 0.96875
Iter 81920, Minibatch Loss= 0.264015, Training Accuracy= 0.93750
Iter 83200, Minibatch Loss= 0.163464, Training Accuracy= 0.95312
Iter 84480, Minibatch Loss= 0.074320, Training Accuracy= 0.98438
Iter 85760, Minibatch Loss= 0.087142, Training Accuracy= 0.96875
Iter 87040, Minibatch Loss= 0.061768, Training Accuracy= 0.97656
Iter 88320, Minibatch Loss= 0.129232, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.054522, Training Accuracy= 0.98438
Iter 90880, Minibatch Loss= 0.060756, Training Accuracy= 0.97656
Iter 92160, Minibatch Loss= 0.034073, Training Accuracy= 1.00000
Iter 93440, Minibatch Loss= 0.112970, Training Accuracy= 0.97656
Iter 94720, Minibatch Loss= 0.088832, Training Accuracy= 0.96875
Iter 96000, Minibatch Loss= 0.093209, Training Accuracy= 0.96875
Iter 97280, Minibatch Loss= 0.098466, Training Accuracy= 0.97656
Iter 98560, Minibatch Loss= 0.085677, Training Accuracy= 0.97656
Iter 99840, Minibatch Loss= 0.102284, Training Accuracy= 0.96875Finished!
Testing Accuracy: 1.0

实例61：静态多层RNN-LSTM连接GRU对MNIST数据集分类

# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)
batch_size = 128tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])#
#stacked_rnn = []
#for i in range(3):
#    stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
#mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)gru = tf.contrib.rnn.GRUCell(n_hidden*2)
lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden)
mcell = tf.contrib.rnn.MultiRNNCell([lstm_cell,gru])x1 = tf.unstack(x, n_steps, 1)
outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))training_iters = 100000display_step = 10# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 100test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 1.808671, Training Accuracy= 0.42188
Iter 2560, Minibatch Loss= 1.288418, Training Accuracy= 0.59375
Iter 3840, Minibatch Loss= 0.954410, Training Accuracy= 0.75781
Iter 5120, Minibatch Loss= 0.873409, Training Accuracy= 0.70312
Iter 6400, Minibatch Loss= 0.872713, Training Accuracy= 0.66406
Iter 7680, Minibatch Loss= 0.618704, Training Accuracy= 0.83594
Iter 8960, Minibatch Loss= 0.426509, Training Accuracy= 0.88281
Iter 10240, Minibatch Loss= 0.480236, Training Accuracy= 0.86719
Iter 11520, Minibatch Loss= 0.429108, Training Accuracy= 0.84375
Iter 12800, Minibatch Loss= 0.552932, Training Accuracy= 0.82812
Iter 14080, Minibatch Loss= 0.362248, Training Accuracy= 0.87500
Iter 15360, Minibatch Loss= 0.265852, Training Accuracy= 0.92969
Iter 16640, Minibatch Loss= 0.315052, Training Accuracy= 0.87500
Iter 17920, Minibatch Loss= 0.259191, Training Accuracy= 0.93750
Iter 19200, Minibatch Loss= 0.360970, Training Accuracy= 0.89844
Iter 20480, Minibatch Loss= 0.292491, Training Accuracy= 0.94531
Iter 21760, Minibatch Loss= 0.259818, Training Accuracy= 0.90625
Iter 23040, Minibatch Loss= 0.293232, Training Accuracy= 0.93750
Iter 24320, Minibatch Loss= 0.244462, Training Accuracy= 0.94531
Iter 25600, Minibatch Loss= 0.227099, Training Accuracy= 0.93750
Iter 26880, Minibatch Loss= 0.231890, Training Accuracy= 0.92969
Iter 28160, Minibatch Loss= 0.196979, Training Accuracy= 0.94531
Iter 29440, Minibatch Loss= 0.165492, Training Accuracy= 0.95312
Iter 30720, Minibatch Loss= 0.252359, Training Accuracy= 0.95312
Iter 32000, Minibatch Loss= 0.237196, Training Accuracy= 0.94531
Iter 33280, Minibatch Loss= 0.197849, Training Accuracy= 0.95312
Iter 34560, Minibatch Loss= 0.126519, Training Accuracy= 0.96094
Iter 35840, Minibatch Loss= 0.173198, Training Accuracy= 0.95312
Iter 37120, Minibatch Loss= 0.176549, Training Accuracy= 0.95312
Iter 38400, Minibatch Loss= 0.206727, Training Accuracy= 0.95312
Iter 39680, Minibatch Loss= 0.170020, Training Accuracy= 0.94531
Iter 40960, Minibatch Loss= 0.211021, Training Accuracy= 0.91406
Iter 42240, Minibatch Loss= 0.174285, Training Accuracy= 0.93750
Iter 43520, Minibatch Loss= 0.074847, Training Accuracy= 0.99219
Iter 44800, Minibatch Loss= 0.188610, Training Accuracy= 0.96875
Iter 46080, Minibatch Loss= 0.098489, Training Accuracy= 0.96875
Iter 47360, Minibatch Loss= 0.164228, Training Accuracy= 0.96094
Iter 48640, Minibatch Loss= 0.099976, Training Accuracy= 0.97656
Iter 49920, Minibatch Loss= 0.185240, Training Accuracy= 0.93750
Iter 51200, Minibatch Loss= 0.146018, Training Accuracy= 0.95312
Iter 52480, Minibatch Loss= 0.085702, Training Accuracy= 0.97656
Iter 53760, Minibatch Loss= 0.079165, Training Accuracy= 0.98438
Iter 55040, Minibatch Loss= 0.230488, Training Accuracy= 0.94531
Iter 56320, Minibatch Loss= 0.026857, Training Accuracy= 1.00000
Iter 57600, Minibatch Loss= 0.189286, Training Accuracy= 0.93750
Iter 58880, Minibatch Loss= 0.085770, Training Accuracy= 0.97656
Iter 60160, Minibatch Loss= 0.102213, Training Accuracy= 0.96094
Iter 61440, Minibatch Loss= 0.087951, Training Accuracy= 0.97656
Iter 62720, Minibatch Loss= 0.105146, Training Accuracy= 0.96094
Iter 64000, Minibatch Loss= 0.150521, Training Accuracy= 0.96094
Iter 65280, Minibatch Loss= 0.051553, Training Accuracy= 0.99219
Iter 66560, Minibatch Loss= 0.044942, Training Accuracy= 0.98438
Iter 67840, Minibatch Loss= 0.093745, Training Accuracy= 0.95312
Iter 69120, Minibatch Loss= 0.091557, Training Accuracy= 0.96094
Iter 70400, Minibatch Loss= 0.079020, Training Accuracy= 0.97656
Iter 71680, Minibatch Loss= 0.107035, Training Accuracy= 0.96875
Iter 72960, Minibatch Loss= 0.103098, Training Accuracy= 0.96094
Iter 74240, Minibatch Loss= 0.137766, Training Accuracy= 0.96875
Iter 75520, Minibatch Loss= 0.087706, Training Accuracy= 0.96875
Iter 76800, Minibatch Loss= 0.056394, Training Accuracy= 0.99219
Iter 78080, Minibatch Loss= 0.097080, Training Accuracy= 0.97656
Iter 79360, Minibatch Loss= 0.064108, Training Accuracy= 0.97656
Iter 80640, Minibatch Loss= 0.081608, Training Accuracy= 0.97656
Iter 81920, Minibatch Loss= 0.074628, Training Accuracy= 0.98438
Iter 83200, Minibatch Loss= 0.173194, Training Accuracy= 0.96094
Iter 84480, Minibatch Loss= 0.166146, Training Accuracy= 0.94531
Iter 85760, Minibatch Loss= 0.132719, Training Accuracy= 0.97656
Iter 87040, Minibatch Loss= 0.119572, Training Accuracy= 0.96875
Iter 88320, Minibatch Loss= 0.030019, Training Accuracy= 1.00000
Iter 89600, Minibatch Loss= 0.176066, Training Accuracy= 0.96875
Iter 90880, Minibatch Loss= 0.069413, Training Accuracy= 0.96875
Iter 92160, Minibatch Loss= 0.112738, Training Accuracy= 0.97656
Iter 93440, Minibatch Loss= 0.069958, Training Accuracy= 0.98438
Iter 94720, Minibatch Loss= 0.037432, Training Accuracy= 0.99219
Iter 96000, Minibatch Loss= 0.082820, Training Accuracy= 0.98438
Iter 97280, Minibatch Loss= 0.121944, Training Accuracy= 0.96875
Iter 98560, Minibatch Loss= 0.058363, Training Accuracy= 0.99219
Iter 99840, Minibatch Loss= 0.139464, Training Accuracy= 0.96875Finished!
Testing Accuracy: 0.97

实例62：动态多层RNN对MNIST分类

# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)
batch_size = 128tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])#
#stacked_rnn = []
#for i in range(3):
#    stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))
#mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)gru = tf.contrib.rnn.GRUCell(n_hidden*2)
lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden)
mcell = tf.contrib.rnn.MultiRNNCell([lstm_cell,gru])#x1 = tf.unstack(x, n_steps, 1)
#outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
#pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)outputs,states  = tf.nn.dynamic_rnn(mcell,x,dtype=tf.float32)#(?, 28, 256)
outputs = tf.transpose(outputs, [1, 0, 2])#(28, ?, 256) 28个时序，取最后一个时序outputs[-1]=(?,256)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))training_iters = 100000display_step = 10# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 100test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 1.786891, Training Accuracy= 0.36719
Iter 2560, Minibatch Loss= 1.360941, Training Accuracy= 0.55469
Iter 3840, Minibatch Loss= 1.208105, Training Accuracy= 0.60156
Iter 5120, Minibatch Loss= 1.075533, Training Accuracy= 0.62500
Iter 6400, Minibatch Loss= 0.780158, Training Accuracy= 0.72656
Iter 7680, Minibatch Loss= 0.785984, Training Accuracy= 0.71875
Iter 8960, Minibatch Loss= 0.459019, Training Accuracy= 0.87500
Iter 10240, Minibatch Loss= 0.699168, Training Accuracy= 0.78125
Iter 11520, Minibatch Loss= 0.549069, Training Accuracy= 0.82031
Iter 12800, Minibatch Loss= 0.412937, Training Accuracy= 0.83594
Iter 14080, Minibatch Loss= 0.359638, Training Accuracy= 0.88281
Iter 15360, Minibatch Loss= 0.358867, Training Accuracy= 0.89844
Iter 16640, Minibatch Loss= 0.384718, Training Accuracy= 0.85938
Iter 17920, Minibatch Loss= 0.344854, Training Accuracy= 0.90625
Iter 19200, Minibatch Loss= 0.320385, Training Accuracy= 0.89062
Iter 20480, Minibatch Loss= 0.424948, Training Accuracy= 0.89062
Iter 21760, Minibatch Loss= 0.237800, Training Accuracy= 0.92969
Iter 23040, Minibatch Loss= 0.153665, Training Accuracy= 0.96094
Iter 24320, Minibatch Loss= 0.269827, Training Accuracy= 0.92188
Iter 25600, Minibatch Loss= 0.209402, Training Accuracy= 0.94531
Iter 26880, Minibatch Loss= 0.191818, Training Accuracy= 0.96094
Iter 28160, Minibatch Loss= 0.222970, Training Accuracy= 0.91406
Iter 29440, Minibatch Loss= 0.114014, Training Accuracy= 0.98438
Iter 30720, Minibatch Loss= 0.237388, Training Accuracy= 0.92969
Iter 32000, Minibatch Loss= 0.107080, Training Accuracy= 0.97656
Iter 33280, Minibatch Loss= 0.160952, Training Accuracy= 0.94531
Iter 34560, Minibatch Loss= 0.153807, Training Accuracy= 0.95312
Iter 35840, Minibatch Loss= 0.162314, Training Accuracy= 0.93750
Iter 37120, Minibatch Loss= 0.158119, Training Accuracy= 0.94531
Iter 38400, Minibatch Loss= 0.075459, Training Accuracy= 0.97656
Iter 39680, Minibatch Loss= 0.145804, Training Accuracy= 0.96875
Iter 40960, Minibatch Loss= 0.099279, Training Accuracy= 0.99219
Iter 42240, Minibatch Loss= 0.171049, Training Accuracy= 0.94531
Iter 43520, Minibatch Loss= 0.231703, Training Accuracy= 0.93750
Iter 44800, Minibatch Loss= 0.102913, Training Accuracy= 0.97656
Iter 46080, Minibatch Loss= 0.091480, Training Accuracy= 0.98438
Iter 47360, Minibatch Loss= 0.101064, Training Accuracy= 0.96875
Iter 48640, Minibatch Loss= 0.187488, Training Accuracy= 0.94531
Iter 49920, Minibatch Loss= 0.115087, Training Accuracy= 0.96094
Iter 51200, Minibatch Loss= 0.059695, Training Accuracy= 0.99219
Iter 52480, Minibatch Loss= 0.112815, Training Accuracy= 0.95312
Iter 53760, Minibatch Loss= 0.177814, Training Accuracy= 0.94531
Iter 55040, Minibatch Loss= 0.082340, Training Accuracy= 0.98438
Iter 56320, Minibatch Loss= 0.110558, Training Accuracy= 0.96875
Iter 57600, Minibatch Loss= 0.103817, Training Accuracy= 0.97656
Iter 58880, Minibatch Loss= 0.084542, Training Accuracy= 0.97656
Iter 60160, Minibatch Loss= 0.173816, Training Accuracy= 0.95312
Iter 61440, Minibatch Loss= 0.084009, Training Accuracy= 0.96875
Iter 62720, Minibatch Loss= 0.103503, Training Accuracy= 0.96875
Iter 64000, Minibatch Loss= 0.054017, Training Accuracy= 0.97656
Iter 65280, Minibatch Loss= 0.040551, Training Accuracy= 0.99219
Iter 66560, Minibatch Loss= 0.156163, Training Accuracy= 0.95312
Iter 67840, Minibatch Loss= 0.082662, Training Accuracy= 0.96875
Iter 69120, Minibatch Loss= 0.038192, Training Accuracy= 1.00000
Iter 70400, Minibatch Loss= 0.167307, Training Accuracy= 0.96094
Iter 71680, Minibatch Loss= 0.112745, Training Accuracy= 0.97656
Iter 72960, Minibatch Loss= 0.053178, Training Accuracy= 0.98438
Iter 74240, Minibatch Loss= 0.150620, Training Accuracy= 0.95312
Iter 75520, Minibatch Loss= 0.086217, Training Accuracy= 0.98438
Iter 76800, Minibatch Loss= 0.108635, Training Accuracy= 0.98438
Iter 78080, Minibatch Loss= 0.078446, Training Accuracy= 0.98438
Iter 79360, Minibatch Loss= 0.117279, Training Accuracy= 0.96875
Iter 80640, Minibatch Loss= 0.046549, Training Accuracy= 0.97656
Iter 81920, Minibatch Loss= 0.082791, Training Accuracy= 0.97656
Iter 83200, Minibatch Loss= 0.047810, Training Accuracy= 0.98438
Iter 84480, Minibatch Loss= 0.048099, Training Accuracy= 0.99219
Iter 85760, Minibatch Loss= 0.109521, Training Accuracy= 0.96094
Iter 87040, Minibatch Loss= 0.054557, Training Accuracy= 0.98438
Iter 88320, Minibatch Loss= 0.035168, Training Accuracy= 0.99219
Iter 89600, Minibatch Loss= 0.077837, Training Accuracy= 0.98438
Iter 90880, Minibatch Loss= 0.162258, Training Accuracy= 0.94531
Iter 92160, Minibatch Loss= 0.121832, Training Accuracy= 0.97656
Iter 93440, Minibatch Loss= 0.040085, Training Accuracy= 0.98438
Iter 94720, Minibatch Loss= 0.047824, Training Accuracy= 0.98438
Iter 96000, Minibatch Loss= 0.112367, Training Accuracy= 0.96094
Iter 97280, Minibatch Loss= 0.077551, Training Accuracy= 0.99219
Iter 98560, Minibatch Loss= 0.093468, Training Accuracy= 0.95312
Iter 99840, Minibatch Loss= 0.088767, Training Accuracy= 0.98438Finished!
Testing Accuracy: 0.99

实例63：构建单层动态BiRNN对MNIST数据集分类

# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# 反向cell
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, output_states = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,lstm_bw_cell,x,dtype=tf.float32)
print(len(outputs),outputs[0].shape,outputs[1].shape)
outputs = tf.concat(outputs, 2)
outputs = tf.transpose(outputs, [1, 0, 2])pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 2.160943, Training Accuracy= 0.27344
Iter 2560, Minibatch Loss= 1.997462, Training Accuracy= 0.33594
Iter 3840, Minibatch Loss= 1.570268, Training Accuracy= 0.42188
Iter 5120, Minibatch Loss= 1.373489, Training Accuracy= 0.48438
Iter 6400, Minibatch Loss= 1.088359, Training Accuracy= 0.65625
Iter 7680, Minibatch Loss= 0.993419, Training Accuracy= 0.70312
Iter 8960, Minibatch Loss= 0.906906, Training Accuracy= 0.72656
Iter 10240, Minibatch Loss= 0.781511, Training Accuracy= 0.73438
Iter 11520, Minibatch Loss= 0.745537, Training Accuracy= 0.74219
Iter 12800, Minibatch Loss= 0.687559, Training Accuracy= 0.79688
Iter 14080, Minibatch Loss= 0.536104, Training Accuracy= 0.83594
Iter 15360, Minibatch Loss= 0.499159, Training Accuracy= 0.85938
Iter 16640, Minibatch Loss= 0.525461, Training Accuracy= 0.84375
Iter 17920, Minibatch Loss= 0.458517, Training Accuracy= 0.87500
Iter 19200, Minibatch Loss= 0.525781, Training Accuracy= 0.82031
Iter 20480, Minibatch Loss= 0.398223, Training Accuracy= 0.89062
Iter 21760, Minibatch Loss= 0.343273, Training Accuracy= 0.89062
Iter 23040, Minibatch Loss= 0.294032, Training Accuracy= 0.90625
Iter 24320, Minibatch Loss= 0.333711, Training Accuracy= 0.91406
Iter 25600, Minibatch Loss= 0.303608, Training Accuracy= 0.89062
Iter 26880, Minibatch Loss= 0.356443, Training Accuracy= 0.89844
Iter 28160, Minibatch Loss= 0.318978, Training Accuracy= 0.90625
Iter 29440, Minibatch Loss= 0.346513, Training Accuracy= 0.91406
Iter 30720, Minibatch Loss= 0.351226, Training Accuracy= 0.89062
Iter 32000, Minibatch Loss= 0.262708, Training Accuracy= 0.90625
Iter 33280, Minibatch Loss= 0.250729, Training Accuracy= 0.91406
Iter 34560, Minibatch Loss= 0.429823, Training Accuracy= 0.85156
Iter 35840, Minibatch Loss= 0.348004, Training Accuracy= 0.91406
Iter 37120, Minibatch Loss= 0.325374, Training Accuracy= 0.91406
Iter 38400, Minibatch Loss= 0.247374, Training Accuracy= 0.89062
Iter 39680, Minibatch Loss= 0.402448, Training Accuracy= 0.87500
Iter 40960, Minibatch Loss= 0.314637, Training Accuracy= 0.92188
Iter 42240, Minibatch Loss= 0.252825, Training Accuracy= 0.92969
Iter 43520, Minibatch Loss= 0.280104, Training Accuracy= 0.91406
Iter 44800, Minibatch Loss= 0.251025, Training Accuracy= 0.92969
Iter 46080, Minibatch Loss= 0.336563, Training Accuracy= 0.89062
Iter 47360, Minibatch Loss= 0.195152, Training Accuracy= 0.94531
Iter 48640, Minibatch Loss= 0.202264, Training Accuracy= 0.94531
Iter 49920, Minibatch Loss= 0.203700, Training Accuracy= 0.92188
Iter 51200, Minibatch Loss= 0.207661, Training Accuracy= 0.96094
Iter 52480, Minibatch Loss= 0.264310, Training Accuracy= 0.92969
Iter 53760, Minibatch Loss= 0.190808, Training Accuracy= 0.94531
Iter 55040, Minibatch Loss= 0.083677, Training Accuracy= 0.97656
Iter 56320, Minibatch Loss= 0.236587, Training Accuracy= 0.92188
Iter 57600, Minibatch Loss= 0.242121, Training Accuracy= 0.92969
Iter 58880, Minibatch Loss= 0.182118, Training Accuracy= 0.96094
Iter 60160, Minibatch Loss= 0.174736, Training Accuracy= 0.94531
Iter 61440, Minibatch Loss= 0.178520, Training Accuracy= 0.96875
Iter 62720, Minibatch Loss= 0.163360, Training Accuracy= 0.93750
Iter 64000, Minibatch Loss= 0.219186, Training Accuracy= 0.92969
Iter 65280, Minibatch Loss= 0.082264, Training Accuracy= 0.98438
Iter 66560, Minibatch Loss= 0.171661, Training Accuracy= 0.95312
Iter 67840, Minibatch Loss= 0.210177, Training Accuracy= 0.96094
Iter 69120, Minibatch Loss= 0.301095, Training Accuracy= 0.92969
Iter 70400, Minibatch Loss= 0.082820, Training Accuracy= 0.97656
Iter 71680, Minibatch Loss= 0.222693, Training Accuracy= 0.93750
Iter 72960, Minibatch Loss= 0.164202, Training Accuracy= 0.92969
Iter 74240, Minibatch Loss= 0.158589, Training Accuracy= 0.95312
Iter 75520, Minibatch Loss= 0.155147, Training Accuracy= 0.96094
Iter 76800, Minibatch Loss= 0.139993, Training Accuracy= 0.95312
Iter 78080, Minibatch Loss= 0.138517, Training Accuracy= 0.95312
Iter 79360, Minibatch Loss= 0.184054, Training Accuracy= 0.94531
Iter 80640, Minibatch Loss= 0.207642, Training Accuracy= 0.92188
Iter 81920, Minibatch Loss= 0.127576, Training Accuracy= 0.96094
Iter 83200, Minibatch Loss= 0.157001, Training Accuracy= 0.96094
Iter 84480, Minibatch Loss= 0.169914, Training Accuracy= 0.94531
Iter 85760, Minibatch Loss= 0.133477, Training Accuracy= 0.96094
Iter 87040, Minibatch Loss= 0.244336, Training Accuracy= 0.89844
Iter 88320, Minibatch Loss= 0.071679, Training Accuracy= 0.99219

Iter 89600, Minibatch Loss= 0.123343, Training Accuracy= 0.96094
Iter 90880, Minibatch Loss= 0.217972, Training Accuracy= 0.92188
Iter 92160, Minibatch Loss= 0.082326, Training Accuracy= 0.99219
Iter 93440, Minibatch Loss= 0.206144, Training Accuracy= 0.94531
Iter 94720, Minibatch Loss= 0.119195, Training Accuracy= 0.97656
Iter 96000, Minibatch Loss= 0.076865, Training Accuracy= 0.97656
Iter 97280, Minibatch Loss= 0.208217, Training Accuracy= 0.92969
Iter 98560, Minibatch Loss= 0.143691, Training Accuracy= 0.94531
Iter 99840, Minibatch Loss= 0.119417, Training Accuracy= 0.97656Finished!
Testing Accuracy: 0.9765625

实例64：构建单层静态BiRNN对MNIST数据集分类

# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
# 反向cell
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,dtype=tf.float32)
print(outputs[0].shape,len(outputs))
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 2.198292, Training Accuracy= 0.34375
Iter 2560, Minibatch Loss= 1.922280, Training Accuracy= 0.33594
Iter 3840, Minibatch Loss= 1.619660, Training Accuracy= 0.48438
Iter 5120, Minibatch Loss= 1.287644, Training Accuracy= 0.57812
Iter 6400, Minibatch Loss= 1.054544, Training Accuracy= 0.63281
Iter 7680, Minibatch Loss= 0.970391, Training Accuracy= 0.64844
Iter 8960, Minibatch Loss= 0.785846, Training Accuracy= 0.75000
Iter 10240, Minibatch Loss= 0.735042, Training Accuracy= 0.77344
Iter 11520, Minibatch Loss= 0.709633, Training Accuracy= 0.78906
Iter 12800, Minibatch Loss= 0.494802, Training Accuracy= 0.85938
Iter 14080, Minibatch Loss= 0.587061, Training Accuracy= 0.82812
Iter 15360, Minibatch Loss= 0.571160, Training Accuracy= 0.80469
Iter 16640, Minibatch Loss= 0.576181, Training Accuracy= 0.80469
Iter 17920, Minibatch Loss= 0.425966, Training Accuracy= 0.89062
Iter 19200, Minibatch Loss= 0.334115, Training Accuracy= 0.91406
Iter 20480, Minibatch Loss= 0.367955, Training Accuracy= 0.89844
Iter 21760, Minibatch Loss= 0.314265, Training Accuracy= 0.90625
Iter 23040, Minibatch Loss= 0.325072, Training Accuracy= 0.89844
Iter 24320, Minibatch Loss= 0.427572, Training Accuracy= 0.86719
Iter 25600, Minibatch Loss= 0.407501, Training Accuracy= 0.86719
Iter 26880, Minibatch Loss= 0.279054, Training Accuracy= 0.92969
Iter 28160, Minibatch Loss= 0.250573, Training Accuracy= 0.91406
Iter 29440, Minibatch Loss= 0.243975, Training Accuracy= 0.92188
Iter 30720, Minibatch Loss= 0.238282, Training Accuracy= 0.93750
Iter 32000, Minibatch Loss= 0.205076, Training Accuracy= 0.92188
Iter 33280, Minibatch Loss= 0.346559, Training Accuracy= 0.89844
Iter 34560, Minibatch Loss= 0.247907, Training Accuracy= 0.92188
Iter 35840, Minibatch Loss= 0.277912, Training Accuracy= 0.90625
Iter 37120, Minibatch Loss= 0.290173, Training Accuracy= 0.92188
Iter 38400, Minibatch Loss= 0.301018, Training Accuracy= 0.89062
Iter 39680, Minibatch Loss= 0.269850, Training Accuracy= 0.90625
Iter 40960, Minibatch Loss= 0.276790, Training Accuracy= 0.91406
Iter 42240, Minibatch Loss= 0.234881, Training Accuracy= 0.95312
Iter 43520, Minibatch Loss= 0.276319, Training Accuracy= 0.91406
Iter 44800, Minibatch Loss= 0.232567, Training Accuracy= 0.92188
Iter 46080, Minibatch Loss= 0.319510, Training Accuracy= 0.89062
Iter 47360, Minibatch Loss= 0.148028, Training Accuracy= 0.95312
Iter 48640, Minibatch Loss= 0.176917, Training Accuracy= 0.95312
Iter 49920, Minibatch Loss= 0.236975, Training Accuracy= 0.92969
Iter 51200, Minibatch Loss= 0.129818, Training Accuracy= 0.96094
Iter 52480, Minibatch Loss= 0.238299, Training Accuracy= 0.94531
Iter 53760, Minibatch Loss= 0.156861, Training Accuracy= 0.95312
Iter 55040, Minibatch Loss= 0.216879, Training Accuracy= 0.92969
Iter 56320, Minibatch Loss= 0.302488, Training Accuracy= 0.89844
Iter 57600, Minibatch Loss= 0.131714, Training Accuracy= 0.96094
Iter 58880, Minibatch Loss= 0.111539, Training Accuracy= 0.98438
Iter 60160, Minibatch Loss= 0.153051, Training Accuracy= 0.94531
Iter 61440, Minibatch Loss= 0.203167, Training Accuracy= 0.93750
Iter 62720, Minibatch Loss= 0.135487, Training Accuracy= 0.96875
Iter 64000, Minibatch Loss= 0.193364, Training Accuracy= 0.93750
Iter 65280, Minibatch Loss= 0.253839, Training Accuracy= 0.92969
Iter 66560, Minibatch Loss= 0.167367, Training Accuracy= 0.92969
Iter 67840, Minibatch Loss= 0.292977, Training Accuracy= 0.92188
Iter 69120, Minibatch Loss= 0.167282, Training Accuracy= 0.95312
Iter 70400, Minibatch Loss= 0.320158, Training Accuracy= 0.90625
Iter 71680, Minibatch Loss= 0.104524, Training Accuracy= 0.96875
Iter 72960, Minibatch Loss= 0.150813, Training Accuracy= 0.95312
Iter 74240, Minibatch Loss= 0.221704, Training Accuracy= 0.94531
Iter 75520, Minibatch Loss= 0.163385, Training Accuracy= 0.96094
Iter 76800, Minibatch Loss= 0.116737, Training Accuracy= 0.96875
Iter 78080, Minibatch Loss= 0.125065, Training Accuracy= 0.96094
Iter 79360, Minibatch Loss= 0.297941, Training Accuracy= 0.92188
Iter 80640, Minibatch Loss= 0.195890, Training Accuracy= 0.93750
Iter 81920, Minibatch Loss= 0.071490, Training Accuracy= 0.99219
Iter 83200, Minibatch Loss= 0.115205, Training Accuracy= 0.97656
Iter 84480, Minibatch Loss= 0.046660, Training Accuracy= 0.99219
Iter 85760, Minibatch Loss= 0.104727, Training Accuracy= 0.96875
Iter 87040, Minibatch Loss= 0.126642, Training Accuracy= 0.94531
Iter 88320, Minibatch Loss= 0.112083, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.084982, Training Accuracy= 0.98438
Iter 90880, Minibatch Loss= 0.184768, Training Accuracy= 0.94531
Iter 92160, Minibatch Loss= 0.192093, Training Accuracy= 0.94531
Iter 93440, Minibatch Loss= 0.108329, Training Accuracy= 0.98438
Iter 94720, Minibatch Loss= 0.102838, Training Accuracy= 0.97656
Iter 96000, Minibatch Loss= 0.065941, Training Accuracy= 0.98438
Iter 97280, Minibatch Loss= 0.138307, Training Accuracy= 0.96094
Iter 98560, Minibatch Loss= 0.106404, Training Accuracy= 0.95312
Iter 99840, Minibatch Loss= 0.184734, Training Accuracy= 0.93750Finished!
Testing Accuracy: 0.9765625

实例65：构建多层BiRNN对MNIST数据集分类

# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
#                                              dtype=tf.float32)outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,dtype=tf.float32)print(outputs[0].shape,len(outputs))
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 2.172426, Training Accuracy= 0.36719
Iter 2560, Minibatch Loss= 1.861094, Training Accuracy= 0.30469
Iter 3840, Minibatch Loss= 1.509905, Training Accuracy= 0.53125
Iter 5120, Minibatch Loss= 1.420893, Training Accuracy= 0.52344
Iter 6400, Minibatch Loss= 1.035572, Training Accuracy= 0.65625
Iter 7680, Minibatch Loss= 0.857329, Training Accuracy= 0.69531
Iter 8960, Minibatch Loss= 0.792173, Training Accuracy= 0.75000
Iter 10240, Minibatch Loss= 0.736671, Training Accuracy= 0.77344
Iter 11520, Minibatch Loss= 0.735905, Training Accuracy= 0.77344
Iter 12800, Minibatch Loss= 0.661815, Training Accuracy= 0.80469
Iter 14080, Minibatch Loss= 0.606804, Training Accuracy= 0.77344
Iter 15360, Minibatch Loss= 0.560650, Training Accuracy= 0.81250
Iter 16640, Minibatch Loss= 0.492422, Training Accuracy= 0.84375
Iter 17920, Minibatch Loss= 0.556491, Training Accuracy= 0.83594
Iter 19200, Minibatch Loss= 0.479821, Training Accuracy= 0.83594
Iter 20480, Minibatch Loss= 0.519516, Training Accuracy= 0.80469
Iter 21760, Minibatch Loss= 0.408189, Training Accuracy= 0.86719
Iter 23040, Minibatch Loss= 0.294251, Training Accuracy= 0.89844
Iter 24320, Minibatch Loss= 0.237520, Training Accuracy= 0.92188
Iter 25600, Minibatch Loss= 0.510871, Training Accuracy= 0.87500
Iter 26880, Minibatch Loss= 0.335866, Training Accuracy= 0.89844
Iter 28160, Minibatch Loss= 0.234492, Training Accuracy= 0.92188
Iter 29440, Minibatch Loss= 0.294053, Training Accuracy= 0.92188
Iter 30720, Minibatch Loss= 0.268390, Training Accuracy= 0.91406
Iter 32000, Minibatch Loss= 0.233152, Training Accuracy= 0.92188
Iter 33280, Minibatch Loss= 0.252517, Training Accuracy= 0.92969
Iter 34560, Minibatch Loss= 0.280433, Training Accuracy= 0.92188
Iter 35840, Minibatch Loss= 0.430729, Training Accuracy= 0.86719
Iter 37120, Minibatch Loss= 0.330445, Training Accuracy= 0.92969
Iter 38400, Minibatch Loss= 0.234638, Training Accuracy= 0.92969
Iter 39680, Minibatch Loss= 0.223448, Training Accuracy= 0.95312
Iter 40960, Minibatch Loss= 0.288984, Training Accuracy= 0.89844
Iter 42240, Minibatch Loss= 0.223206, Training Accuracy= 0.92969
Iter 43520, Minibatch Loss= 0.204564, Training Accuracy= 0.95312
Iter 44800, Minibatch Loss= 0.137583, Training Accuracy= 0.97656
Iter 46080, Minibatch Loss= 0.172719, Training Accuracy= 0.95312
Iter 47360, Minibatch Loss= 0.181917, Training Accuracy= 0.95312
Iter 48640, Minibatch Loss= 0.245667, Training Accuracy= 0.93750
Iter 49920, Minibatch Loss= 0.171087, Training Accuracy= 0.94531
Iter 51200, Minibatch Loss= 0.273686, Training Accuracy= 0.92969
Iter 52480, Minibatch Loss= 0.110942, Training Accuracy= 0.96094
Iter 53760, Minibatch Loss= 0.252516, Training Accuracy= 0.92969
Iter 55040, Minibatch Loss= 0.184445, Training Accuracy= 0.96094
Iter 56320, Minibatch Loss= 0.212692, Training Accuracy= 0.93750
Iter 57600, Minibatch Loss= 0.148188, Training Accuracy= 0.94531
Iter 58880, Minibatch Loss= 0.202650, Training Accuracy= 0.93750
Iter 60160, Minibatch Loss= 0.162564, Training Accuracy= 0.95312
Iter 61440, Minibatch Loss= 0.140252, Training Accuracy= 0.96875
Iter 62720, Minibatch Loss= 0.080859, Training Accuracy= 0.98438
Iter 64000, Minibatch Loss= 0.271262, Training Accuracy= 0.92188
Iter 65280, Minibatch Loss= 0.130727, Training Accuracy= 0.96875
Iter 66560, Minibatch Loss= 0.202950, Training Accuracy= 0.94531
Iter 67840, Minibatch Loss= 0.104672, Training Accuracy= 0.97656
Iter 69120, Minibatch Loss= 0.104267, Training Accuracy= 0.96875
Iter 70400, Minibatch Loss= 0.264764, Training Accuracy= 0.91406
Iter 71680, Minibatch Loss= 0.247204, Training Accuracy= 0.92969
Iter 72960, Minibatch Loss= 0.114761, Training Accuracy= 0.96875
Iter 74240, Minibatch Loss= 0.127167, Training Accuracy= 0.96875
Iter 75520, Minibatch Loss= 0.150572, Training Accuracy= 0.95312
Iter 76800, Minibatch Loss= 0.266346, Training Accuracy= 0.92969
Iter 78080, Minibatch Loss= 0.188211, Training Accuracy= 0.92188
Iter 79360, Minibatch Loss= 0.178170, Training Accuracy= 0.94531
Iter 80640, Minibatch Loss= 0.065349, Training Accuracy= 0.99219
Iter 81920, Minibatch Loss= 0.114197, Training Accuracy= 0.96094
Iter 83200, Minibatch Loss= 0.166545, Training Accuracy= 0.94531
Iter 84480, Minibatch Loss= 0.170978, Training Accuracy= 0.96094
Iter 85760, Minibatch Loss= 0.100027, Training Accuracy= 0.98438
Iter 87040, Minibatch Loss= 0.068403, Training Accuracy= 0.97656
Iter 88320, Minibatch Loss= 0.084172, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.109292, Training Accuracy= 0.96875
Iter 90880, Minibatch Loss= 0.142447, Training Accuracy= 0.96094
Iter 92160, Minibatch Loss= 0.153235, Training Accuracy= 0.94531
Iter 93440, Minibatch Loss= 0.244075, Training Accuracy= 0.93750
Iter 94720, Minibatch Loss= 0.181213, Training Accuracy= 0.96094
Iter 96000, Minibatch Loss= 0.112537, Training Accuracy= 0.97656
Iter 97280, Minibatch Loss= 0.081839, Training Accuracy= 0.97656
Iter 98560, Minibatch Loss= 0.052867, Training Accuracy= 0.99219
Iter 99840, Minibatch Loss= 0.153948, Training Accuracy= 0.96875Finished!
Testing Accuracy: 0.984375

list多层BiRNN

# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
#                                              dtype=tf.float32)#outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
#                                              dtype=tf.float32)stacked_rnn = []
stacked_bw_rnn = []
for i in range(3):stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))stacked_bw_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))outputs, _, _ = rnn.stack_bidirectional_rnn(stacked_rnn,stacked_bw_rnn, x1,dtype=tf.float32)    pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 1.821590, Training Accuracy= 0.35156
Iter 2560, Minibatch Loss= 1.534426, Training Accuracy= 0.45312
Iter 3840, Minibatch Loss= 1.064070, Training Accuracy= 0.64062
Iter 5120, Minibatch Loss= 0.864886, Training Accuracy= 0.69531
Iter 6400, Minibatch Loss= 0.761668, Training Accuracy= 0.71094
Iter 7680, Minibatch Loss= 0.740874, Training Accuracy= 0.77344
Iter 8960, Minibatch Loss= 0.588481, Training Accuracy= 0.81250
Iter 10240, Minibatch Loss= 0.462982, Training Accuracy= 0.82812
Iter 11520, Minibatch Loss= 0.370633, Training Accuracy= 0.88281
Iter 12800, Minibatch Loss= 0.338352, Training Accuracy= 0.89062
Iter 14080, Minibatch Loss= 0.331004, Training Accuracy= 0.92188
Iter 15360, Minibatch Loss= 0.311066, Training Accuracy= 0.86719
Iter 16640, Minibatch Loss= 0.343783, Training Accuracy= 0.87500
Iter 17920, Minibatch Loss= 0.372595, Training Accuracy= 0.88281
Iter 19200, Minibatch Loss= 0.325485, Training Accuracy= 0.92188
Iter 20480, Minibatch Loss= 0.264375, Training Accuracy= 0.89844
Iter 21760, Minibatch Loss= 0.139555, Training Accuracy= 0.98438
Iter 23040, Minibatch Loss= 0.148688, Training Accuracy= 0.96094
Iter 24320, Minibatch Loss= 0.258720, Training Accuracy= 0.93750
Iter 25600, Minibatch Loss= 0.226550, Training Accuracy= 0.92188
Iter 26880, Minibatch Loss= 0.138458, Training Accuracy= 0.96875
Iter 28160, Minibatch Loss= 0.213251, Training Accuracy= 0.92188
Iter 29440, Minibatch Loss= 0.254001, Training Accuracy= 0.92188
Iter 30720, Minibatch Loss= 0.167976, Training Accuracy= 0.96875
Iter 32000, Minibatch Loss= 0.135351, Training Accuracy= 0.95312
Iter 33280, Minibatch Loss= 0.160028, Training Accuracy= 0.94531
Iter 34560, Minibatch Loss= 0.077145, Training Accuracy= 0.98438
Iter 35840, Minibatch Loss= 0.191825, Training Accuracy= 0.94531
Iter 37120, Minibatch Loss= 0.165540, Training Accuracy= 0.95312
Iter 38400, Minibatch Loss= 0.196213, Training Accuracy= 0.94531
Iter 39680, Minibatch Loss= 0.287433, Training Accuracy= 0.93750
Iter 40960, Minibatch Loss= 0.191275, Training Accuracy= 0.95312
Iter 42240, Minibatch Loss= 0.151342, Training Accuracy= 0.96094
Iter 43520, Minibatch Loss= 0.125097, Training Accuracy= 0.96875
Iter 44800, Minibatch Loss= 0.102935, Training Accuracy= 0.98438
Iter 46080, Minibatch Loss= 0.266253, Training Accuracy= 0.92969
Iter 47360, Minibatch Loss= 0.142998, Training Accuracy= 0.95312
Iter 48640, Minibatch Loss= 0.184273, Training Accuracy= 0.93750
Iter 49920, Minibatch Loss= 0.119624, Training Accuracy= 0.95312
Iter 51200, Minibatch Loss= 0.142909, Training Accuracy= 0.96875
Iter 52480, Minibatch Loss= 0.116738, Training Accuracy= 0.96875
Iter 53760, Minibatch Loss= 0.060568, Training Accuracy= 0.97656
Iter 55040, Minibatch Loss= 0.058581, Training Accuracy= 0.98438
Iter 56320, Minibatch Loss= 0.124698, Training Accuracy= 0.96094
Iter 57600, Minibatch Loss= 0.100896, Training Accuracy= 0.98438
Iter 58880, Minibatch Loss= 0.110977, Training Accuracy= 0.95312
Iter 60160, Minibatch Loss= 0.106460, Training Accuracy= 0.97656
Iter 61440, Minibatch Loss= 0.110304, Training Accuracy= 0.96875
Iter 62720, Minibatch Loss= 0.232378, Training Accuracy= 0.94531
Iter 64000, Minibatch Loss= 0.203506, Training Accuracy= 0.93750
Iter 65280, Minibatch Loss= 0.056710, Training Accuracy= 0.97656
Iter 66560, Minibatch Loss= 0.072753, Training Accuracy= 0.97656
Iter 67840, Minibatch Loss= 0.079578, Training Accuracy= 0.97656
Iter 69120, Minibatch Loss= 0.107077, Training Accuracy= 0.96094
Iter 70400, Minibatch Loss= 0.096304, Training Accuracy= 0.97656
Iter 71680, Minibatch Loss= 0.144363, Training Accuracy= 0.96094
Iter 72960, Minibatch Loss= 0.091259, Training Accuracy= 0.96094
Iter 74240, Minibatch Loss= 0.048933, Training Accuracy= 0.97656
Iter 75520, Minibatch Loss= 0.160002, Training Accuracy= 0.92969
Iter 76800, Minibatch Loss= 0.058088, Training Accuracy= 0.99219
Iter 78080, Minibatch Loss= 0.127986, Training Accuracy= 0.96094
Iter 79360, Minibatch Loss= 0.038077, Training Accuracy= 0.99219
Iter 80640, Minibatch Loss= 0.111614, Training Accuracy= 0.96875
Iter 81920, Minibatch Loss= 0.042832, Training Accuracy= 0.99219
Iter 83200, Minibatch Loss= 0.099700, Training Accuracy= 0.95312
Iter 84480, Minibatch Loss= 0.070158, Training Accuracy= 0.97656
Iter 85760, Minibatch Loss= 0.044101, Training Accuracy= 0.98438
Iter 87040, Minibatch Loss= 0.148444, Training Accuracy= 0.94531
Iter 88320, Minibatch Loss= 0.053288, Training Accuracy= 0.99219
Iter 89600, Minibatch Loss= 0.086438, Training Accuracy= 0.96875
Iter 90880, Minibatch Loss= 0.084410, Training Accuracy= 0.97656
Iter 92160, Minibatch Loss= 0.070647, Training Accuracy= 0.96875
Iter 93440, Minibatch Loss= 0.157281, Training Accuracy= 0.94531
Iter 94720, Minibatch Loss= 0.154711, Training Accuracy= 0.96875
Iter 96000, Minibatch Loss= 0.067240, Training Accuracy= 0.96875
Iter 97280, Minibatch Loss= 0.067891, Training Accuracy= 0.99219
Iter 98560, Minibatch Loss= 0.040220, Training Accuracy= 0.99219
Iter 99840, Minibatch Loss= 0.113512, Training Accuracy= 0.96094Finished!
Testing Accuracy: 0.96875

MultiBiRNN

# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
#                                              dtype=tf.float32)#outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
#                                              dtype=tf.float32)stacked_rnn = []
stacked_bw_rnn = []
for i in range(3):stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))stacked_bw_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))#outputs, _, _ = rnn.stack_bidirectional_rnn(stacked_rnn,stacked_bw_rnn, x1,
#                                              dtype=tf.float32)    mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
mcell_bw = tf.contrib.rnn.MultiRNNCell(stacked_bw_rnn)outputs, _, _ = rnn.stack_bidirectional_rnn([mcell],[mcell_bw], x1,dtype=tf.float32)#outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn([mcell],[mcell_bw], x,
#                                              dtype=tf.float32)
#outputs = tf.transpose(outputs, [1, 0, 2])
#print(outputs[0].shape,outputs.shape)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 1.969089, Training Accuracy= 0.28906
Iter 2560, Minibatch Loss= 1.607973, Training Accuracy= 0.40625
Iter 3840, Minibatch Loss= 1.223222, Training Accuracy= 0.59375
Iter 5120, Minibatch Loss= 1.054859, Training Accuracy= 0.64844
Iter 6400, Minibatch Loss= 0.956632, Training Accuracy= 0.71875
Iter 7680, Minibatch Loss= 0.862249, Training Accuracy= 0.70312
Iter 8960, Minibatch Loss= 0.752958, Training Accuracy= 0.77344
Iter 10240, Minibatch Loss= 0.697205, Training Accuracy= 0.77344
Iter 11520, Minibatch Loss= 0.563148, Training Accuracy= 0.79688
Iter 12800, Minibatch Loss= 0.559524, Training Accuracy= 0.82031
Iter 14080, Minibatch Loss= 0.394891, Training Accuracy= 0.87500
Iter 15360, Minibatch Loss= 0.651662, Training Accuracy= 0.79688
Iter 16640, Minibatch Loss= 0.458168, Training Accuracy= 0.85938
Iter 17920, Minibatch Loss= 0.525461, Training Accuracy= 0.84375
Iter 19200, Minibatch Loss= 0.323415, Training Accuracy= 0.89844
Iter 20480, Minibatch Loss= 0.407945, Training Accuracy= 0.90625
Iter 21760, Minibatch Loss= 0.342875, Training Accuracy= 0.90625
Iter 23040, Minibatch Loss= 0.278589, Training Accuracy= 0.92188
Iter 24320, Minibatch Loss= 0.248584, Training Accuracy= 0.91406
Iter 25600, Minibatch Loss= 0.314651, Training Accuracy= 0.88281
Iter 26880, Minibatch Loss= 0.256853, Training Accuracy= 0.90625
Iter 28160, Minibatch Loss= 0.305443, Training Accuracy= 0.92969
Iter 29440, Minibatch Loss= 0.370154, Training Accuracy= 0.90625
Iter 30720, Minibatch Loss= 0.262370, Training Accuracy= 0.92188
Iter 32000, Minibatch Loss= 0.239071, Training Accuracy= 0.92969
Iter 33280, Minibatch Loss= 0.277529, Training Accuracy= 0.90625
Iter 34560, Minibatch Loss= 0.147705, Training Accuracy= 0.96094
Iter 35840, Minibatch Loss= 0.210542, Training Accuracy= 0.92969
Iter 37120, Minibatch Loss= 0.221123, Training Accuracy= 0.92188
Iter 38400, Minibatch Loss= 0.357465, Training Accuracy= 0.92188
Iter 39680, Minibatch Loss= 0.164977, Training Accuracy= 0.95312
Iter 40960, Minibatch Loss= 0.243880, Training Accuracy= 0.92969
Iter 42240, Minibatch Loss= 0.213714, Training Accuracy= 0.94531
Iter 43520, Minibatch Loss= 0.133602, Training Accuracy= 0.96875
Iter 44800, Minibatch Loss= 0.304978, Training Accuracy= 0.90625
Iter 46080, Minibatch Loss= 0.228339, Training Accuracy= 0.92188
Iter 47360, Minibatch Loss= 0.240235, Training Accuracy= 0.92188
Iter 48640, Minibatch Loss= 0.154399, Training Accuracy= 0.96875
Iter 49920, Minibatch Loss= 0.252016, Training Accuracy= 0.92188
Iter 51200, Minibatch Loss= 0.171817, Training Accuracy= 0.96875
Iter 52480, Minibatch Loss= 0.145279, Training Accuracy= 0.96875
Iter 53760, Minibatch Loss= 0.164326, Training Accuracy= 0.93750
Iter 55040, Minibatch Loss= 0.099755, Training Accuracy= 0.96094
Iter 56320, Minibatch Loss= 0.085352, Training Accuracy= 0.97656
Iter 57600, Minibatch Loss= 0.172950, Training Accuracy= 0.95312
Iter 58880, Minibatch Loss= 0.123920, Training Accuracy= 0.96875
Iter 60160, Minibatch Loss= 0.115939, Training Accuracy= 0.97656
Iter 61440, Minibatch Loss= 0.084130, Training Accuracy= 0.97656
Iter 62720, Minibatch Loss= 0.149050, Training Accuracy= 0.93750
Iter 64000, Minibatch Loss= 0.168845, Training Accuracy= 0.95312
Iter 65280, Minibatch Loss= 0.140895, Training Accuracy= 0.96094
Iter 66560, Minibatch Loss= 0.126092, Training Accuracy= 0.95312
Iter 67840, Minibatch Loss= 0.147673, Training Accuracy= 0.96094
Iter 69120, Minibatch Loss= 0.057092, Training Accuracy= 0.99219
Iter 70400, Minibatch Loss= 0.037331, Training Accuracy= 1.00000
Iter 71680, Minibatch Loss= 0.188352, Training Accuracy= 0.95312
Iter 72960, Minibatch Loss= 0.108095, Training Accuracy= 0.96875
Iter 74240, Minibatch Loss= 0.059791, Training Accuracy= 0.99219
Iter 75520, Minibatch Loss= 0.113426, Training Accuracy= 0.96875
Iter 76800, Minibatch Loss= 0.133168, Training Accuracy= 0.97656
Iter 78080, Minibatch Loss= 0.183791, Training Accuracy= 0.95312
Iter 79360, Minibatch Loss= 0.122682, Training Accuracy= 0.96875
Iter 80640, Minibatch Loss= 0.057580, Training Accuracy= 0.97656
Iter 81920, Minibatch Loss= 0.167665, Training Accuracy= 0.94531
Iter 83200, Minibatch Loss= 0.122606, Training Accuracy= 0.96094
Iter 84480, Minibatch Loss= 0.091936, Training Accuracy= 0.98438
Iter 85760, Minibatch Loss= 0.171889, Training Accuracy= 0.95312
Iter 87040, Minibatch Loss= 0.117054, Training Accuracy= 0.96875
Iter 88320, Minibatch Loss= 0.051410, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.071451, Training Accuracy= 0.97656
Iter 90880, Minibatch Loss= 0.048790, Training Accuracy= 0.99219
Iter 92160, Minibatch Loss= 0.123331, Training Accuracy= 0.96094
Iter 93440, Minibatch Loss= 0.044237, Training Accuracy= 0.96875
Iter 94720, Minibatch Loss= 0.115570, Training Accuracy= 0.96875
Iter 96000, Minibatch Loss= 0.103702, Training Accuracy= 0.97656
Iter 97280, Minibatch Loss= 0.088024, Training Accuracy= 0.99219
Iter 98560, Minibatch Loss= 0.071973, Training Accuracy= 0.98438
Iter 99840, Minibatch Loss= 0.065105, Training Accuracy= 0.99219Finished!
Testing Accuracy: 0.9765625

实例66：构建多层动态BiRNN对MNIST数据集分类

# -*- coding: utf-8 -*-
import tensorflow as tf
from tensorflow.contrib import rnn
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)#参数设置
learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Network Parameters
n_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)
lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)#outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x1,
#                                              dtype=tf.float32)#outputs, _, _ = rnn.stack_bidirectional_rnn([lstm_fw_cell],[lstm_bw_cell], x1,
#                                              dtype=tf.float32)stacked_rnn = []
stacked_bw_rnn = []
for i in range(3):stacked_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))stacked_bw_rnn.append(tf.contrib.rnn.LSTMCell(n_hidden))#outputs, _, _ = rnn.stack_bidirectional_rnn(stacked_rnn,stacked_bw_rnn, x1,
#                                              dtype=tf.float32)    mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)
mcell_bw = tf.contrib.rnn.MultiRNNCell(stacked_bw_rnn)#outputs, _, _ = rnn.stack_bidirectional_rnn([mcell],[mcell_bw], x1,
#                                              dtype=tf.float32)outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn([mcell],[mcell_bw], x,dtype=tf.float32)
outputs = tf.transpose(outputs, [1, 0, 2]) #batch_size, max_time, layers_output]`
print(outputs[0].shape,outputs.shape)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 1.925515, Training Accuracy= 0.28906
Iter 2560, Minibatch Loss= 1.478539, Training Accuracy= 0.47656
Iter 3840, Minibatch Loss= 1.367934, Training Accuracy= 0.59375
Iter 5120, Minibatch Loss= 1.002120, Training Accuracy= 0.67188
Iter 6400, Minibatch Loss= 0.918952, Training Accuracy= 0.64062
Iter 7680, Minibatch Loss= 0.822888, Training Accuracy= 0.70312
Iter 8960, Minibatch Loss= 0.806084, Training Accuracy= 0.72656
Iter 10240, Minibatch Loss= 0.769249, Training Accuracy= 0.75781
Iter 11520, Minibatch Loss= 0.657592, Training Accuracy= 0.75781
Iter 12800, Minibatch Loss= 0.659502, Training Accuracy= 0.78125
Iter 14080, Minibatch Loss= 0.473548, Training Accuracy= 0.83594
Iter 15360, Minibatch Loss= 0.301996, Training Accuracy= 0.92188
Iter 16640, Minibatch Loss= 0.417549, Training Accuracy= 0.85938
Iter 17920, Minibatch Loss= 0.380066, Training Accuracy= 0.88281
Iter 19200, Minibatch Loss= 0.340269, Training Accuracy= 0.91406
Iter 20480, Minibatch Loss= 0.351742, Training Accuracy= 0.88281
Iter 21760, Minibatch Loss= 0.278537, Training Accuracy= 0.91406
Iter 23040, Minibatch Loss= 0.345716, Training Accuracy= 0.89844
Iter 24320, Minibatch Loss= 0.338512, Training Accuracy= 0.92969
Iter 25600, Minibatch Loss= 0.301185, Training Accuracy= 0.92188
Iter 26880, Minibatch Loss= 0.335147, Training Accuracy= 0.90625
Iter 28160, Minibatch Loss= 0.329022, Training Accuracy= 0.89062
Iter 29440, Minibatch Loss= 0.278270, Training Accuracy= 0.90625
Iter 30720, Minibatch Loss= 0.141217, Training Accuracy= 0.96094
Iter 32000, Minibatch Loss= 0.316236, Training Accuracy= 0.91406
Iter 33280, Minibatch Loss= 0.170242, Training Accuracy= 0.96094
Iter 34560, Minibatch Loss= 0.169931, Training Accuracy= 0.96094
Iter 35840, Minibatch Loss= 0.261950, Training Accuracy= 0.91406
Iter 37120, Minibatch Loss= 0.275674, Training Accuracy= 0.92188
Iter 38400, Minibatch Loss= 0.269527, Training Accuracy= 0.92188
Iter 39680, Minibatch Loss= 0.288827, Training Accuracy= 0.92969
Iter 40960, Minibatch Loss= 0.123902, Training Accuracy= 0.97656
Iter 42240, Minibatch Loss= 0.222748, Training Accuracy= 0.93750
Iter 43520, Minibatch Loss= 0.275202, Training Accuracy= 0.91406
Iter 44800, Minibatch Loss= 0.114382, Training Accuracy= 0.97656
Iter 46080, Minibatch Loss= 0.232849, Training Accuracy= 0.93750
Iter 47360, Minibatch Loss= 0.183887, Training Accuracy= 0.96094
Iter 48640, Minibatch Loss= 0.143083, Training Accuracy= 0.94531
Iter 49920, Minibatch Loss= 0.119365, Training Accuracy= 0.95312
Iter 51200, Minibatch Loss= 0.165366, Training Accuracy= 0.93750
Iter 52480, Minibatch Loss= 0.142011, Training Accuracy= 0.96875
Iter 53760, Minibatch Loss= 0.124913, Training Accuracy= 0.96875
Iter 55040, Minibatch Loss= 0.271773, Training Accuracy= 0.91406
Iter 56320, Minibatch Loss= 0.151414, Training Accuracy= 0.94531
Iter 57600, Minibatch Loss= 0.312587, Training Accuracy= 0.89844
Iter 58880, Minibatch Loss= 0.128200, Training Accuracy= 0.96094
Iter 60160, Minibatch Loss= 0.082254, Training Accuracy= 0.96875
Iter 61440, Minibatch Loss= 0.143054, Training Accuracy= 0.94531
Iter 62720, Minibatch Loss= 0.189484, Training Accuracy= 0.94531
Iter 64000, Minibatch Loss= 0.145638, Training Accuracy= 0.95312
Iter 65280, Minibatch Loss= 0.095384, Training Accuracy= 0.96875
Iter 66560, Minibatch Loss= 0.107284, Training Accuracy= 0.96094
Iter 67840, Minibatch Loss= 0.072667, Training Accuracy= 0.98438
Iter 69120, Minibatch Loss= 0.078070, Training Accuracy= 0.98438
Iter 70400, Minibatch Loss= 0.145665, Training Accuracy= 0.96875
Iter 71680, Minibatch Loss= 0.131772, Training Accuracy= 0.96094
Iter 72960, Minibatch Loss= 0.054384, Training Accuracy= 0.97656
Iter 74240, Minibatch Loss= 0.080717, Training Accuracy= 0.99219
Iter 75520, Minibatch Loss= 0.127881, Training Accuracy= 0.94531
Iter 76800, Minibatch Loss= 0.050177, Training Accuracy= 0.98438
Iter 78080, Minibatch Loss= 0.204734, Training Accuracy= 0.92969
Iter 79360, Minibatch Loss= 0.134242, Training Accuracy= 0.95312
Iter 80640, Minibatch Loss= 0.102928, Training Accuracy= 0.96094
Iter 81920, Minibatch Loss= 0.151719, Training Accuracy= 0.96094
Iter 83200, Minibatch Loss= 0.063140, Training Accuracy= 0.98438
Iter 84480, Minibatch Loss= 0.044821, Training Accuracy= 0.99219
Iter 85760, Minibatch Loss= 0.042410, Training Accuracy= 0.99219
Iter 87040, Minibatch Loss= 0.043164, Training Accuracy= 1.00000
Iter 88320, Minibatch Loss= 0.095371, Training Accuracy= 0.97656
Iter 89600, Minibatch Loss= 0.107568, Training Accuracy= 0.96094
Iter 90880, Minibatch Loss= 0.097504, Training Accuracy= 0.96875
Iter 92160, Minibatch Loss= 0.065621, Training Accuracy= 0.98438
Iter 93440, Minibatch Loss= 0.089349, Training Accuracy= 0.97656
Iter 94720, Minibatch Loss= 0.203302, Training Accuracy= 0.93750
Iter 96000, Minibatch Loss= 0.076798, Training Accuracy= 0.98438
Iter 97280, Minibatch Loss= 0.087679, Training Accuracy= 0.97656
Iter 98560, Minibatch Loss= 0.081388, Training Accuracy= 0.96875
Iter 99840, Minibatch Loss= 0.062613, Training Accuracy= 0.98438Finished!
Testing Accuracy: 0.9765625

实例67：在GRUCell中实现LN

# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)from tensorflow.python.ops.rnn_cell_impl import  RNNCell
from tensorflow.python.ops.rnn_cell_impl import  LayerRNNCell
from tensorflow.python.ops.math_ops import sigmoid
from tensorflow.python.ops.math_ops import tanh
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import array_ops
from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear
from tensorflow.python.layers import base as base_layer
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import nn_opsprint(tf.__version__)tf.reset_default_graph()def ln(tensor, scope = None, epsilon = 1e-5):""" Layer normalizes a 2D tensor along its second axis """assert(len(tensor.get_shape()) == 2)m, v = tf.nn.moments(tensor, [1], keep_dims=True)if not isinstance(scope, str):scope = ''with tf.variable_scope(scope + 'layer_norm'):scale = tf.get_variable('scale',shape=[tensor.get_shape()[1]],initializer=tf.constant_initializer(1))shift = tf.get_variable('shift',shape=[tensor.get_shape()[1]],initializer=tf.constant_initializer(0))LN_initial = (tensor - m) / tf.sqrt(v + epsilon)return LN_initial * scale + shift_BIAS_VARIABLE_NAME = "bias"
_WEIGHTS_VARIABLE_NAME = "kernel"class LNGRUCell(LayerRNNCell):"""Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).Args:num_units: int, The number of units in the GRU cell.activation: Nonlinearity to use.  Default: `tanh`.reuse: (optional) Python boolean describing whether to reuse variablesin an existing scope.  If not `True`, and the existing scope already hasthe given variables, an error is raised.kernel_initializer: (optional) The initializer to use for the weight andprojection matrices.bias_initializer: (optional) The initializer to use for the bias.name: String, the name of the layer. Layers with the same name willshare weights, but to avoid mistakes we require reuse=True in suchcases."""def __init__(self,num_units,activation=None,reuse=None,kernel_initializer=None,bias_initializer=None,name=None):#super(GRUCell, self).__init__(_reuse=reuse, name=name)#原来super(LNGRUCell, self).__init__(_reuse=reuse, name=name)# Inputs must be 2-dimensional.self.input_spec = base_layer.InputSpec(ndim=2)self._num_units = num_unitsself._activation = activation or math_ops.tanhself._kernel_initializer = kernel_initializerself._bias_initializer = bias_initializer@propertydef state_size(self):return self._num_units@propertydef output_size(self):return self._num_unitsdef build(self, inputs_shape):if inputs_shape[1].value is None:raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"% inputs_shape)input_depth = inputs_shape[1].valueself._gate_kernel = self.add_variable("gates/%s" % _WEIGHTS_VARIABLE_NAME,shape=[input_depth + self._num_units, 2 * self._num_units],initializer=self._kernel_initializer)self._gate_bias = self.add_variable("gates/%s" % _BIAS_VARIABLE_NAME,shape=[2 * self._num_units],initializer=(self._bias_initializerif self._bias_initializer is not Noneelse init_ops.constant_initializer(1.0, dtype=self.dtype)))self._candidate_kernel = self.add_variable("candidate/%s" % _WEIGHTS_VARIABLE_NAME,shape=[input_depth + self._num_units, self._num_units],initializer=self._kernel_initializer)self._candidate_bias = self.add_variable("candidate/%s" % _BIAS_VARIABLE_NAME,shape=[self._num_units],initializer=(self._bias_initializerif self._bias_initializer is not Noneelse init_ops.zeros_initializer(dtype=self.dtype)))self.built = Truedef call(self, inputs, state):"""Gated recurrent unit (GRU) with nunits cells."""gate_inputs = math_ops.matmul(array_ops.concat([inputs, state], 1), self._gate_kernel)gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias)value = math_ops.sigmoid(gate_inputs)r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)r = ln(r, scope = 'r/')# 新添加u = ln(u, scope = 'u/')# 新添加r_state = r * statecandidate = math_ops.matmul(array_ops.concat([inputs, r_state], 1), self._candidate_kernel)candidate = nn_ops.bias_add(candidate, self._candidate_bias)c = self._activation(candidate)new_h = u * state + (1 - u) * creturn new_h, new_hn_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)tf.reset_default_graph()# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])x1 = tf.unstack(x, n_steps, 1)#1 BasicLSTMCell
#lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)#2 LSTMCell
#lstm_cell = tf.contrib.rnn.LSTMCell(n_hidden, forget_bias=1.0)
#outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)#3 gru
#gru = tf.contrib.rnn.GRUCell(n_hidden)
gru = LNGRUCell(n_hidden)
#outputs = tf.contrib.rnn.static_rnn(gru, x1, dtype=tf.float32)#4 创建动态RNN
outputs,_  = tf.nn.dynamic_rnn(gru,x,dtype=tf.float32)
outputs = tf.transpose(outputs, [1, 0, 2])pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)learning_rate = 0.001
training_iters = 100000
batch_size = 128
display_step = 10# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 128test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 12.231302, Training Accuracy= 0.08594
Iter 2560, Minibatch Loss= 11.580451, Training Accuracy= 0.08594
Iter 3840, Minibatch Loss= 12.903033, Training Accuracy= 0.06250
Iter 5120, Minibatch Loss= 10.890457, Training Accuracy= 0.12500
Iter 6400, Minibatch Loss= 9.708689, Training Accuracy= 0.12500
Iter 7680, Minibatch Loss= 11.344480, Training Accuracy= 0.07031
Iter 8960, Minibatch Loss= 21.113043, Training Accuracy= 0.11719
Iter 10240, Minibatch Loss= 16.354618, Training Accuracy= 0.12500
Iter 11520, Minibatch Loss= 221.128204, Training Accuracy= 0.15625
Iter 12800, Minibatch Loss= 51.676239, Training Accuracy= 0.14062
Iter 14080, Minibatch Loss= 51.621651, Training Accuracy= 0.16406
Iter 15360, Minibatch Loss= 32.979252, Training Accuracy= 0.13281
Iter 16640, Minibatch Loss= 11.387983, Training Accuracy= 0.16406
Iter 17920, Minibatch Loss= 106.375748, Training Accuracy= 0.13281
Iter 19200, Minibatch Loss= 162.647675, Training Accuracy= 0.07031
Iter 20480, Minibatch Loss= 70.351883, Training Accuracy= 0.03125
Iter 21760, Minibatch Loss= 55.518311, Training Accuracy= 0.12500
Iter 23040, Minibatch Loss= 31.811563, Training Accuracy= 0.10938
Iter 24320, Minibatch Loss= 42.223648, Training Accuracy= 0.11719
Iter 25600, Minibatch Loss= 278.500854, Training Accuracy= 0.08594
Iter 26880, Minibatch Loss= 237.631607, Training Accuracy= 0.10938
Iter 28160, Minibatch Loss= 105.564705, Training Accuracy= 0.14062
Iter 29440, Minibatch Loss= 56.833820, Training Accuracy= 0.08594
Iter 30720, Minibatch Loss= 22.568174, Training Accuracy= 0.10938
Iter 32000, Minibatch Loss= 11.444073, Training Accuracy= 0.10156
Iter 33280, Minibatch Loss= 5.311275, Training Accuracy= 0.11719
Iter 34560, Minibatch Loss= 3.783988, Training Accuracy= 0.10938
Iter 35840, Minibatch Loss= 5.724679, Training Accuracy= 0.10938
Iter 37120, Minibatch Loss= 13.915645, Training Accuracy= 0.06250
Iter 38400, Minibatch Loss= 7.799623, Training Accuracy= 0.06250
Iter 39680, Minibatch Loss= 4.735735, Training Accuracy= 0.10156
Iter 40960, Minibatch Loss= 3.492851, Training Accuracy= 0.14062
Iter 42240, Minibatch Loss= 2.750641, Training Accuracy= 0.12500
Iter 43520, Minibatch Loss= 3.131867, Training Accuracy= 0.03125
Iter 44800, Minibatch Loss= 12.454140, Training Accuracy= 0.14062
Iter 46080, Minibatch Loss= 34.983681, Training Accuracy= 0.11719
Iter 47360, Minibatch Loss= 30.050432, Training Accuracy= 0.11719
Iter 48640, Minibatch Loss= 89.305038, Training Accuracy= 0.14062
Iter 49920, Minibatch Loss= 717.047119, Training Accuracy= 0.04688
Iter 51200, Minibatch Loss= 593.221436, Training Accuracy= 0.07812
Iter 52480, Minibatch Loss= 552.920593, Training Accuracy= 0.07812
Iter 53760, Minibatch Loss= 298.492462, Training Accuracy= 0.10938
Iter 55040, Minibatch Loss= 293.587799, Training Accuracy= 0.12500
Iter 56320, Minibatch Loss= 90.123314, Training Accuracy= 0.08594
Iter 57600, Minibatch Loss= 131.756165, Training Accuracy= 0.11719
Iter 58880, Minibatch Loss= 90.683121, Training Accuracy= 0.10156
Iter 60160, Minibatch Loss= 56.682476, Training Accuracy= 0.09375
Iter 61440, Minibatch Loss= 64.690117, Training Accuracy= 0.07031
Iter 62720, Minibatch Loss= 73.732903, Training Accuracy= 0.12500
Iter 64000, Minibatch Loss= 39.817921, Training Accuracy= 0.18750
Iter 65280, Minibatch Loss= 26.479805, Training Accuracy= 0.03906
Iter 66560, Minibatch Loss= 28.650154, Training Accuracy= 0.03906
Iter 67840, Minibatch Loss= 32.062138, Training Accuracy= 0.07812
Iter 69120, Minibatch Loss= 24.480392, Training Accuracy= 0.10938
Iter 70400, Minibatch Loss= 18.083035, Training Accuracy= 0.14062
Iter 71680, Minibatch Loss= 33.831909, Training Accuracy= 0.08594
Iter 72960, Minibatch Loss= 31.264828, Training Accuracy= 0.17969
Iter 74240, Minibatch Loss= 24.374344, Training Accuracy= 0.12500
Iter 75520, Minibatch Loss= 32.843307, Training Accuracy= 0.09375
Iter 76800, Minibatch Loss= 20.493128, Training Accuracy= 0.05469
Iter 78080, Minibatch Loss= 21.611864, Training Accuracy= 0.16406
Iter 79360, Minibatch Loss= 25.270271, Training Accuracy= 0.17188
Iter 80640, Minibatch Loss= 25.224813, Training Accuracy= 0.14844
Iter 81920, Minibatch Loss= 27.151264, Training Accuracy= 0.08594
Iter 83200, Minibatch Loss= 24.466787, Training Accuracy= 0.06250
Iter 84480, Minibatch Loss= 26.366667, Training Accuracy= 0.21094
Iter 85760, Minibatch Loss= 19.625292, Training Accuracy= 0.18750
Iter 87040, Minibatch Loss= 18.934591, Training Accuracy= 0.17188
Iter 88320, Minibatch Loss= 32.218357, Training Accuracy= 0.10938
Iter 89600, Minibatch Loss= 46.415516, Training Accuracy= 0.12500
Iter 90880, Minibatch Loss= 30.490982, Training Accuracy= 0.11719
Iter 92160, Minibatch Loss= 30.738121, Training Accuracy= 0.16406
Iter 93440, Minibatch Loss= 25.382904, Training Accuracy= 0.10938
Iter 94720, Minibatch Loss= 21.190367, Training Accuracy= 0.11719
Iter 96000, Minibatch Loss= 32.771732, Training Accuracy= 0.10938
Iter 97280, Minibatch Loss= 19.819483, Training Accuracy= 0.11719
Iter 98560, Minibatch Loss= 13.052861, Training Accuracy= 0.15625
Iter 99840, Minibatch Loss= 21.311205, Training Accuracy= 0.15625Finished!
Testing Accuracy: 0.140625

Ln多GRu1

# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)from tensorflow.python.ops.rnn_cell_impl import  RNNCell
from tensorflow.python.ops.rnn_cell_impl import  LayerRNNCell
from tensorflow.python.ops.math_ops import sigmoid
from tensorflow.python.ops.math_ops import tanh
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import array_ops
from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear
from tensorflow.python.layers import base as base_layer
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import nn_opstf.reset_default_graph()def ln(tensor, scope = None, epsilon = 1e-5):""" Layer normalizes a 2D tensor along its second axis """assert(len(tensor.get_shape()) == 2)m, v = tf.nn.moments(tensor, [1], keep_dims=True)if not isinstance(scope, str):scope = ''with tf.variable_scope(scope + 'layer_norm'):scale = tf.get_variable('scale',shape=[tensor.get_shape()[1]],initializer=tf.constant_initializer(1))shift = tf.get_variable('shift',shape=[tensor.get_shape()[1]],initializer=tf.constant_initializer(0))LN_initial = (tensor - m) / tf.sqrt(v + epsilon)return LN_initial * scale + shiftclass LNGRUCell(LayerRNNCell):"""Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).Args:num_units: int, The number of units in the GRU cell.activation: Nonlinearity to use.  Default: `tanh`.reuse: (optional) Python boolean describing whether to reuse variablesin an existing scope.  If not `True`, and the existing scope already hasthe given variables, an error is raised.kernel_initializer: (optional) The initializer to use for the weight andprojection matrices.bias_initializer: (optional) The initializer to use for the bias.name: String, the name of the layer. Layers with the same name willshare weights, but to avoid mistakes we require reuse=True in suchcases."""def __init__(self,num_units,activation=None,reuse=None,kernel_initializer=None,bias_initializer=None,name=None):#super(GRUCell, self).__init__(_reuse=reuse, name=name)#原来super(LNGRUCell, self).__init__(_reuse=reuse, name=name)# Inputs must be 2-dimensional.self.input_spec = base_layer.InputSpec(ndim=2)self._num_units = num_unitsself._activation = activation or math_ops.tanhself._kernel_initializer = kernel_initializerself._bias_initializer = bias_initializer@propertydef state_size(self):return self._num_units@propertydef output_size(self):return self._num_unitsdef build(self, inputs_shape):if inputs_shape[1].value is None:raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"% inputs_shape)input_depth = inputs_shape[1].valueself._gate_kernel = self.add_variable("gates/%s" % _WEIGHTS_VARIABLE_NAME,shape=[input_depth + self._num_units, 2 * self._num_units],initializer=self._kernel_initializer)self._gate_bias = self.add_variable("gates/%s" % _BIAS_VARIABLE_NAME,shape=[2 * self._num_units],initializer=(self._bias_initializerif self._bias_initializer is not Noneelse init_ops.constant_initializer(1.0, dtype=self.dtype)))self._candidate_kernel = self.add_variable("candidate/%s" % _WEIGHTS_VARIABLE_NAME,shape=[input_depth + self._num_units, self._num_units],initializer=self._kernel_initializer)self._candidate_bias = self.add_variable("candidate/%s" % _BIAS_VARIABLE_NAME,shape=[self._num_units],initializer=(self._bias_initializerif self._bias_initializer is not Noneelse init_ops.zeros_initializer(dtype=self.dtype)))self.built = Truedef call(self, inputs, state):"""Gated recurrent unit (GRU) with nunits cells."""gate_inputs = math_ops.matmul(array_ops.concat([inputs, state], 1), self._gate_kernel)gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias)value = math_ops.sigmoid(gate_inputs)r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)r = ln(r, scope = 'r/')# 新添加u = ln(u, scope = 'u/')# 新添加r_state = r * statecandidate = math_ops.matmul(array_ops.concat([inputs, r_state], 1), self._candidate_kernel)candidate = nn_ops.bias_add(candidate, self._candidate_bias)c = self._activation(candidate)new_h = u * state + (1 - u) * creturn new_h, new_hn_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)
batch_size = 128tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])stacked_rnn = []
for i in range(3):stacked_rnn.append(LNGRUCell(n_hidden))
mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)outputs,states  = tf.nn.dynamic_rnn(mcell,x,dtype=tf.float32)#(?, 28, 256)
outputs = tf.transpose(outputs, [1, 0, 2])#(28, ?, 256) 28个时序，取最后一个时序outputs[-1]=(?,256)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))training_iters = 100000display_step = 10# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 100test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 11.682723, Training Accuracy= 0.08594
Iter 2560, Minibatch Loss= 12.917633, Training Accuracy= 0.13281
Iter 3840, Minibatch Loss= 11.147453, Training Accuracy= 0.14062
Iter 5120, Minibatch Loss= 11.501393, Training Accuracy= 0.10938
Iter 6400, Minibatch Loss= 11.329225, Training Accuracy= 0.14062
Iter 7680, Minibatch Loss= 12.902426, Training Accuracy= 0.12500
Iter 8960, Minibatch Loss= 14.787603, Training Accuracy= 0.12500
Iter 10240, Minibatch Loss= 14.239416, Training Accuracy= 0.11719
Iter 11520, Minibatch Loss= 11.831277, Training Accuracy= 0.14062
Iter 12800, Minibatch Loss= 13.174420, Training Accuracy= 0.12500
Iter 14080, Minibatch Loss= 10.532808, Training Accuracy= 0.07812
Iter 15360, Minibatch Loss= 21.689716, Training Accuracy= 0.11719
Iter 16640, Minibatch Loss= 18.296116, Training Accuracy= 0.10938
Iter 17920, Minibatch Loss= 12.612600, Training Accuracy= 0.14844
Iter 19200, Minibatch Loss= 8.998665, Training Accuracy= 0.09375
Iter 20480, Minibatch Loss= 7.482079, Training Accuracy= 0.13281
Iter 21760, Minibatch Loss= 5.310290, Training Accuracy= 0.20312
Iter 23040, Minibatch Loss= 11.217176, Training Accuracy= 0.10156
Iter 24320, Minibatch Loss= 10.468469, Training Accuracy= 0.11719
Iter 25600, Minibatch Loss= 9.238225, Training Accuracy= 0.09375
Iter 26880, Minibatch Loss= 9.928499, Training Accuracy= 0.10156
Iter 28160, Minibatch Loss= 11.853710, Training Accuracy= 0.05469
Iter 29440, Minibatch Loss= 11.066288, Training Accuracy= 0.03906
Iter 30720, Minibatch Loss= 8.290665, Training Accuracy= 0.14844
Iter 32000, Minibatch Loss= 7.704072, Training Accuracy= 0.14844
Iter 33280, Minibatch Loss= 7.526510, Training Accuracy= 0.13281
Iter 34560, Minibatch Loss= 10.274215, Training Accuracy= 0.08594
Iter 35840, Minibatch Loss= 9.874534, Training Accuracy= 0.07812
Iter 37120, Minibatch Loss= 7.862291, Training Accuracy= 0.07812
Iter 38400, Minibatch Loss= 7.622520, Training Accuracy= 0.10156
Iter 39680, Minibatch Loss= 5.845595, Training Accuracy= 0.13281
Iter 40960, Minibatch Loss= 4.240401, Training Accuracy= 0.14062
Iter 42240, Minibatch Loss= 3.776126, Training Accuracy= 0.11719
Iter 43520, Minibatch Loss= 4.534883, Training Accuracy= 0.08594
Iter 44800, Minibatch Loss= 4.593569, Training Accuracy= 0.08594
Iter 46080, Minibatch Loss= 4.776147, Training Accuracy= 0.06250
Iter 47360, Minibatch Loss= 4.990581, Training Accuracy= 0.07812
Iter 48640, Minibatch Loss= 4.928415, Training Accuracy= 0.08594
Iter 49920, Minibatch Loss= 4.748688, Training Accuracy= 0.09375
Iter 51200, Minibatch Loss= 2.824999, Training Accuracy= 0.14062
Iter 52480, Minibatch Loss= 2.998337, Training Accuracy= 0.15625
Iter 53760, Minibatch Loss= 3.980899, Training Accuracy= 0.12500
Iter 55040, Minibatch Loss= 3.022208, Training Accuracy= 0.17969
Iter 56320, Minibatch Loss= 3.015280, Training Accuracy= 0.13281
Iter 57600, Minibatch Loss= 2.676783, Training Accuracy= 0.13281
Iter 58880, Minibatch Loss= 6.211295, Training Accuracy= 0.10938
Iter 60160, Minibatch Loss= 4.801533, Training Accuracy= 0.14062
Iter 61440, Minibatch Loss= 7.826266, Training Accuracy= 0.08594
Iter 62720, Minibatch Loss= 4.575137, Training Accuracy= 0.08594
Iter 64000, Minibatch Loss= 4.202699, Training Accuracy= 0.11719
Iter 65280, Minibatch Loss= 5.797447, Training Accuracy= 0.04688
Iter 66560, Minibatch Loss= 3.881121, Training Accuracy= 0.12500
Iter 67840, Minibatch Loss= 3.441604, Training Accuracy= 0.11719
Iter 69120, Minibatch Loss= 3.097083, Training Accuracy= 0.10938
Iter 70400, Minibatch Loss= 3.482257, Training Accuracy= 0.09375
Iter 71680, Minibatch Loss= 3.060152, Training Accuracy= 0.09375
Iter 72960, Minibatch Loss= 2.767920, Training Accuracy= 0.14062
Iter 74240, Minibatch Loss= 2.690722, Training Accuracy= 0.10156
Iter 75520, Minibatch Loss= 2.394173, Training Accuracy= 0.08594
Iter 76800, Minibatch Loss= 2.383599, Training Accuracy= 0.11719
Iter 78080, Minibatch Loss= 1.973354, Training Accuracy= 0.23438
Iter 79360, Minibatch Loss= 2.104026, Training Accuracy= 0.23438
Iter 80640, Minibatch Loss= 2.073925, Training Accuracy= 0.21094
Iter 81920, Minibatch Loss= 1.942348, Training Accuracy= 0.25781
Iter 83200, Minibatch Loss= 2.008878, Training Accuracy= 0.26562
Iter 84480, Minibatch Loss= 1.999240, Training Accuracy= 0.28125
Iter 85760, Minibatch Loss= 1.956447, Training Accuracy= 0.23438
Iter 87040, Minibatch Loss= 1.900973, Training Accuracy= 0.25781
Iter 88320, Minibatch Loss= 2.097849, Training Accuracy= 0.25000
Iter 89600, Minibatch Loss= 2.013985, Training Accuracy= 0.28906
Iter 90880, Minibatch Loss= 1.828421, Training Accuracy= 0.28125
Iter 92160, Minibatch Loss= 1.944018, Training Accuracy= 0.29688
Iter 93440, Minibatch Loss= 1.839489, Training Accuracy= 0.33594
Iter 94720, Minibatch Loss= 1.945905, Training Accuracy= 0.28125
Iter 96000, Minibatch Loss= 2.672914, Training Accuracy= 0.10156
Iter 97280, Minibatch Loss= 2.854921, Training Accuracy= 0.03906
Iter 98560, Minibatch Loss= 2.538456, Training Accuracy= 0.04688
Iter 99840, Minibatch Loss= 2.212051, Training Accuracy= 0.19531Finished!
Testing Accuracy: 0.24

# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
# 导入 MINST 数据集
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/data/", one_hot=True)from tensorflow.python.ops.rnn_cell_impl import  RNNCell
from tensorflow.python.ops.rnn_cell_impl import  LayerRNNCell
from tensorflow.python.ops.math_ops import sigmoid
from tensorflow.python.ops.math_ops import tanh
from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import array_ops
from tensorflow.contrib.rnn.python.ops.core_rnn_cell import _linear
from tensorflow.python.layers import base as base_layer
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import init_ops
from tensorflow.python.ops import nn_opstf.reset_default_graph()
print(tf.__version__)
def ln(tensor, scope = None, epsilon = 1e-5):""" Layer normalizes a 2D tensor along its second axis """assert(len(tensor.get_shape()) == 2)m, v = tf.nn.moments(tensor, [1], keep_dims=True)if not isinstance(scope, str):scope = ''with tf.variable_scope(scope + 'layer_norm'):scale = tf.get_variable('scale',shape=[tensor.get_shape()[1]],initializer=tf.constant_initializer(1))shift = tf.get_variable('shift',shape=[tensor.get_shape()[1]],initializer=tf.constant_initializer(0))LN_initial = (tensor - m) / tf.sqrt(v + epsilon)return LN_initial * scale + shift#class LNGRUCell(RNNCell):
#    """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078)."""
#
#    def __init__(self, num_units, input_size=None, activation=tanh):
#        if input_size is not None:
#            print("%s: The input_size parameter is deprecated." % self)
#        self._num_units = num_units
#        self._activation = activation
#
#    @property
#    def state_size(self):
#        return self._num_units
#
#    @property
#    def output_size(self):
#        return self._num_units
#
#    def __call__(self, inputs, state):
#        """Gated recurrent unit (GRU) with nunits cells."""
#        with vs.variable_scope("Gates"):  # Reset gate and update gate.,reuse=True
#            # We start with bias of 1.0 to not reset and not update.
#            value =_linear([inputs, state], 2 * self._num_units, True, 1.0)
#            r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)
#            r = ln(r, scope = 'r/')
#            u = ln(u, scope = 'u/')
#            r, u = sigmoid(r), sigmoid(u)
#        with vs.variable_scope("Candidate"):
##            with vs.variable_scope("Layer_Parameters"):
#            Cand = _linear([inputs,  r *state], self._num_units, True)
#            c_pre = ln(Cand,  scope = 'new_h/')
#            c = self._activation(c_pre)
#        new_h = u * state + (1 - u) * c
#        return new_h, new_hclass LNGRUCell(LayerRNNCell):"""Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).Args:num_units: int, The number of units in the GRU cell.activation: Nonlinearity to use.  Default: `tanh`.reuse: (optional) Python boolean describing whether to reuse variablesin an existing scope.  If not `True`, and the existing scope already hasthe given variables, an error is raised.kernel_initializer: (optional) The initializer to use for the weight andprojection matrices.bias_initializer: (optional) The initializer to use for the bias.name: String, the name of the layer. Layers with the same name willshare weights, but to avoid mistakes we require reuse=True in suchcases."""def __init__(self,num_units,activation=None,reuse=None,kernel_initializer=None,bias_initializer=None,name=None):#super(GRUCell, self).__init__(_reuse=reuse, name=name)#原来super(LNGRUCell, self).__init__(_reuse=reuse, name=name)# Inputs must be 2-dimensional.self.input_spec = base_layer.InputSpec(ndim=2)self._num_units = num_unitsself._activation = activation or math_ops.tanhself._kernel_initializer = kernel_initializerself._bias_initializer = bias_initializer@propertydef state_size(self):return self._num_units@propertydef output_size(self):return self._num_unitsdef build(self, inputs_shape):if inputs_shape[1].value is None:raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"% inputs_shape)input_depth = inputs_shape[1].valueself._gate_kernel = self.add_variable("gates/%s" % _WEIGHTS_VARIABLE_NAME,shape=[input_depth + self._num_units, 2 * self._num_units],initializer=self._kernel_initializer)self._gate_bias = self.add_variable("gates/%s" % _BIAS_VARIABLE_NAME,shape=[2 * self._num_units],initializer=(self._bias_initializerif self._bias_initializer is not Noneelse init_ops.constant_initializer(1.0, dtype=self.dtype)))self._candidate_kernel = self.add_variable("candidate/%s" % _WEIGHTS_VARIABLE_NAME,shape=[input_depth + self._num_units, self._num_units],initializer=self._kernel_initializer)self._candidate_bias = self.add_variable("candidate/%s" % _BIAS_VARIABLE_NAME,shape=[self._num_units],initializer=(self._bias_initializerif self._bias_initializer is not Noneelse init_ops.zeros_initializer(dtype=self.dtype)))self.built = Truedef call(self, inputs, state):"""Gated recurrent unit (GRU) with nunits cells."""gate_inputs = math_ops.matmul(array_ops.concat([inputs, state], 1), self._gate_kernel)gate_inputs = nn_ops.bias_add(gate_inputs, self._gate_bias)value = math_ops.sigmoid(gate_inputs)r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1)r = ln(r, scope = 'r/')# 新添加u = ln(u, scope = 'u/')# 新添加r_state = r * statecandidate = math_ops.matmul(array_ops.concat([inputs, r_state], 1), self._candidate_kernel)candidate = nn_ops.bias_add(candidate, self._candidate_bias)c = self._activation(candidate)new_h = u * state + (1 - u) * creturn new_h, new_hn_input = 28 # MNIST data 输入 (img shape: 28*28)
n_steps = 28 # timesteps
n_hidden = 128 # hidden layer num of features
n_classes = 10  # MNIST 列别 (0-9 ，一共10类)
batch_size = 128tf.reset_default_graph()
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
y = tf.placeholder("float", [None, n_classes])stacked_rnn = []
for i in range(3):stacked_rnn.append(LNGRUCell(n_hidden))#stacked_rnn.append(tf.contrib.rnn.GRUCell(n_hidden))mcell = tf.contrib.rnn.MultiRNNCell(stacked_rnn)x1 = tf.unstack(x, n_steps, 1)
outputs, states = tf.contrib.rnn.static_rnn(mcell, x1, dtype=tf.float32)
pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)#outputs,states  = tf.nn.dynamic_rnn(mcell,x,dtype=tf.float32)#(?, 28, 256)
#outputs = tf.transpose(outputs, [1, 0, 2])#(28, ?, 256) 28个时序，取最后一个时序outputs[-1]=(?,256)
#pred = tf.contrib.layers.fully_connected(outputs[-1],n_classes,activation_fn = None)
#learning_rate = 0.001
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))training_iters = 100000display_step = 10# 启动session
with tf.Session() as sess:sess.run(tf.global_variables_initializer())step = 1# Keep training until reach max iterationswhile step * batch_size < training_iters:batch_x, batch_y = mnist.train.next_batch(batch_size)# Reshape data to get 28 seq of 28 elementsbatch_x = batch_x.reshape((batch_size, n_steps, n_input))# Run optimization op (backprop)sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})if step % display_step == 0:# 计算批次数据的准确率acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})# Calculate batch lossloss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})print ("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \"{:.6f}".format(loss) + ", Training Accuracy= " + \"{:.5f}".format(acc))step += 1print (" Finished!")# 计算准确率 for 128 mnist test imagestest_len = 100test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))test_label = mnist.test.labels[:test_len]print ("Testing Accuracy:", \sess.run(accuracy, feed_dict={x: test_data, y: test_label}))

Iter 1280, Minibatch Loss= 12.811991, Training Accuracy= 0.09375
Iter 2560, Minibatch Loss= 12.545067, Training Accuracy= 0.07031
Iter 3840, Minibatch Loss= 11.355438, Training Accuracy= 0.07031
Iter 5120, Minibatch Loss= 10.871563, Training Accuracy= 0.10156
Iter 6400, Minibatch Loss= 11.371557, Training Accuracy= 0.11719
Iter 7680, Minibatch Loss= 11.038004, Training Accuracy= 0.10938
Iter 8960, Minibatch Loss= 11.118206, Training Accuracy= 0.07812
Iter 10240, Minibatch Loss= 10.915785, Training Accuracy= 0.10156
Iter 11520, Minibatch Loss= 10.551557, Training Accuracy= 0.17969
Iter 12800, Minibatch Loss= 17.443115, Training Accuracy= 0.07031
Iter 14080, Minibatch Loss= 9.930510, Training Accuracy= 0.16406
Iter 15360, Minibatch Loss= 10.727852, Training Accuracy= 0.08594
Iter 16640, Minibatch Loss= 14.488325, Training Accuracy= 0.10938
Iter 17920, Minibatch Loss= 15.069497, Training Accuracy= 0.10938
Iter 19200, Minibatch Loss= 9.160069, Training Accuracy= 0.10938
Iter 20480, Minibatch Loss= 11.217492, Training Accuracy= 0.06250
Iter 21760, Minibatch Loss= 9.194048, Training Accuracy= 0.07812
Iter 23040, Minibatch Loss= 6.942473, Training Accuracy= 0.06250
Iter 24320, Minibatch Loss= 7.054324, Training Accuracy= 0.10938
Iter 25600, Minibatch Loss= 7.900481, Training Accuracy= 0.06250
Iter 26880, Minibatch Loss= 7.393861, Training Accuracy= 0.14844
Iter 28160, Minibatch Loss= 6.575096, Training Accuracy= 0.12500
Iter 29440, Minibatch Loss= 6.346336, Training Accuracy= 0.08594
Iter 30720, Minibatch Loss= 6.810472, Training Accuracy= 0.11719
Iter 32000, Minibatch Loss= 6.466897, Training Accuracy= 0.10938
Iter 33280, Minibatch Loss= 5.635717, Training Accuracy= 0.11719
Iter 34560, Minibatch Loss= 4.871469, Training Accuracy= 0.07812
Iter 35840, Minibatch Loss= 4.517097, Training Accuracy= 0.12500
Iter 37120, Minibatch Loss= 4.759137, Training Accuracy= 0.13281
Iter 38400, Minibatch Loss= 4.027944, Training Accuracy= 0.13281
Iter 39680, Minibatch Loss= 4.569145, Training Accuracy= 0.10156
Iter 40960, Minibatch Loss= 2.790155, Training Accuracy= 0.13281
Iter 42240, Minibatch Loss= 3.320386, Training Accuracy= 0.14844
Iter 43520, Minibatch Loss= 3.730759, Training Accuracy= 0.11719
Iter 44800, Minibatch Loss= 6.588100, Training Accuracy= 0.10938
Iter 46080, Minibatch Loss= 3.605313, Training Accuracy= 0.06250
Iter 47360, Minibatch Loss= 3.313602, Training Accuracy= 0.08594
Iter 48640, Minibatch Loss= 2.854012, Training Accuracy= 0.12500
Iter 49920, Minibatch Loss= 3.053122, Training Accuracy= 0.09375
Iter 51200, Minibatch Loss= 3.263433, Training Accuracy= 0.07812
Iter 52480, Minibatch Loss= 3.164253, Training Accuracy= 0.10156
Iter 53760, Minibatch Loss= 2.568920, Training Accuracy= 0.08594
Iter 55040, Minibatch Loss= 2.622833, Training Accuracy= 0.08594
Iter 56320, Minibatch Loss= 2.574237, Training Accuracy= 0.10156
Iter 57600, Minibatch Loss= 2.320722, Training Accuracy= 0.14062
Iter 58880, Minibatch Loss= 2.416647, Training Accuracy= 0.14062
Iter 60160, Minibatch Loss= 2.378407, Training Accuracy= 0.12500
Iter 61440, Minibatch Loss= 2.236217, Training Accuracy= 0.21094
Iter 62720, Minibatch Loss= 2.288638, Training Accuracy= 0.15625
Iter 64000, Minibatch Loss= 2.273193, Training Accuracy= 0.16406
Iter 65280, Minibatch Loss= 2.243811, Training Accuracy= 0.16406
Iter 66560, Minibatch Loss= 2.228886, Training Accuracy= 0.21875
Iter 67840, Minibatch Loss= 2.233191, Training Accuracy= 0.19531
Iter 69120, Minibatch Loss= 2.175747, Training Accuracy= 0.21875
Iter 70400, Minibatch Loss= 2.162902, Training Accuracy= 0.23438
Iter 71680, Minibatch Loss= 2.325285, Training Accuracy= 0.14062
Iter 72960, Minibatch Loss= 2.281687, Training Accuracy= 0.19531
Iter 74240, Minibatch Loss= 2.340919, Training Accuracy= 0.14062
Iter 75520, Minibatch Loss= 2.432267, Training Accuracy= 0.12500
Iter 76800, Minibatch Loss= 2.398518, Training Accuracy= 0.13281
Iter 78080, Minibatch Loss= 2.573318, Training Accuracy= 0.10938
Iter 79360, Minibatch Loss= 2.431727, Training Accuracy= 0.15625
Iter 80640, Minibatch Loss= 2.496175, Training Accuracy= 0.10156
Iter 81920, Minibatch Loss= 2.485367, Training Accuracy= 0.10156
Iter 83200, Minibatch Loss= 2.326829, Training Accuracy= 0.09375
Iter 84480, Minibatch Loss= 2.413650, Training Accuracy= 0.14062
Iter 85760, Minibatch Loss= 2.327744, Training Accuracy= 0.13281
Iter 87040, Minibatch Loss= 2.327587, Training Accuracy= 0.13281
Iter 88320, Minibatch Loss= 2.335200, Training Accuracy= 0.11719
Iter 89600, Minibatch Loss= 2.370470, Training Accuracy= 0.15625
Iter 90880, Minibatch Loss= 2.429362, Training Accuracy= 0.09375
Iter 92160, Minibatch Loss= 2.286218, Training Accuracy= 0.12500
Iter 93440, Minibatch Loss= 2.334179, Training Accuracy= 0.14844
Iter 94720, Minibatch Loss= 2.313130, Training Accuracy= 0.15625
Iter 96000, Minibatch Loss= 2.388861, Training Accuracy= 0.10938
Iter 97280, Minibatch Loss= 2.468186, Training Accuracy= 0.16406
Iter 98560, Minibatch Loss= 2.380651, Training Accuracy= 0.12500
Iter 99840, Minibatch Loss= 2.311615, Training Accuracy= 0.08594Finished!
Testing Accuracy: 0.14

实例68：利用BiRNN实现语音识别

# -*- coding: utf-8 -*-
import numpy as np
import time
import tensorflow as tf
from tensorflow.python.ops import ctc_ops
from collections import Counter
## 自定义
yuyinutils = __import__("9-24__yuyinutils")
sparse_tuple_to_texts_ch = yuyinutils.sparse_tuple_to_texts_ch
ndarray_to_text_ch = yuyinutils.ndarray_to_text_ch
get_audio_and_transcriptch = yuyinutils.get_audio_and_transcriptch
pad_sequences = yuyinutils.pad_sequences
sparse_tuple_from = yuyinutils.sparse_tuple_from
get_wavs_lables = yuyinutils.get_wavs_lablestf.reset_default_graph()b_stddev = 0.046875
h_stddev = 0.046875n_hidden = 1024
n_hidden_1 = 1024
n_hidden_2 =1024
n_hidden_5 = 1024
n_cell_dim = 1024
n_hidden_3 = 2 * 1024keep_dropout_rate=0.95
relu_clip = 20def BiRNN_model( batch_x, seq_length, n_input, n_context,n_character ,keep_dropout):# batch_x_shape: [batch_size, n_steps, n_input + 2*n_input*n_context]batch_x_shape = tf.shape(batch_x)# 将输入转成时间序列优先batch_x = tf.transpose(batch_x, [1, 0, 2])# 再转成2维传入第一层batch_x = tf.reshape(batch_x,[-1, n_input + 2 * n_input * n_context])  # (n_steps*batch_size, n_input + 2*n_input*n_context)# 使用clipped RELU activation and dropout.# 1st layerwith tf.name_scope('fc1'):b1 = variable_on_cpu('b1', [n_hidden_1], tf.random_normal_initializer(stddev=b_stddev))h1 = variable_on_cpu('h1', [n_input + 2 * n_input * n_context, n_hidden_1],tf.random_normal_initializer(stddev=h_stddev))layer_1 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(batch_x, h1), b1)), relu_clip)layer_1 = tf.nn.dropout(layer_1, keep_dropout)# 2nd layerwith tf.name_scope('fc2'):b2 = variable_on_cpu('b2', [n_hidden_2], tf.random_normal_initializer(stddev=b_stddev))h2 = variable_on_cpu('h2', [n_hidden_1, n_hidden_2], tf.random_normal_initializer(stddev=h_stddev))layer_2 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_1, h2), b2)), relu_clip)layer_2 = tf.nn.dropout(layer_2, keep_dropout)# 3rd layerwith tf.name_scope('fc3'):b3 = variable_on_cpu('b3', [n_hidden_3], tf.random_normal_initializer(stddev=b_stddev))h3 = variable_on_cpu('h3', [n_hidden_2, n_hidden_3], tf.random_normal_initializer(stddev=h_stddev))layer_3 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_2, h3), b3)), relu_clip)layer_3 = tf.nn.dropout(layer_3, keep_dropout)# 双向rnnwith tf.name_scope('lstm'):# Forward direction cell:lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim, forget_bias=1.0, state_is_tuple=True)lstm_fw_cell = tf.contrib.rnn.DropoutWrapper(lstm_fw_cell,input_keep_prob=keep_dropout)# Backward direction cell:lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim, forget_bias=1.0, state_is_tuple=True)lstm_bw_cell = tf.contrib.rnn.DropoutWrapper(lstm_bw_cell,input_keep_prob=keep_dropout)# `layer_3`  `[n_steps, batch_size, 2*n_cell_dim]`layer_3 = tf.reshape(layer_3, [-1, batch_x_shape[0], n_hidden_3])outputs, output_states = tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,cell_bw=lstm_bw_cell,inputs=layer_3,dtype=tf.float32,time_major=True,sequence_length=seq_length)# 连接正反向结果[n_steps, batch_size, 2*n_cell_dim]outputs = tf.concat(outputs, 2)# to a single tensor of shape [n_steps*batch_size, 2*n_cell_dim]        outputs = tf.reshape(outputs, [-1, 2 * n_cell_dim])with tf.name_scope('fc5'):b5 = variable_on_cpu('b5', [n_hidden_5], tf.random_normal_initializer(stddev=b_stddev))h5 = variable_on_cpu('h5', [(2 * n_cell_dim), n_hidden_5], tf.random_normal_initializer(stddev=h_stddev))layer_5 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(outputs, h5), b5)), relu_clip)layer_5 = tf.nn.dropout(layer_5, keep_dropout)with tf.name_scope('fc6'):# 全连接层用于softmax分类b6 = variable_on_cpu('b6', [n_character], tf.random_normal_initializer(stddev=b_stddev))h6 = variable_on_cpu('h6', [n_hidden_5, n_character], tf.random_normal_initializer(stddev=h_stddev))layer_6 = tf.add(tf.matmul(layer_5, h6), b6)# 将2维[n_steps*batch_size, n_character]转成3维 time-major [n_steps, batch_size, n_character].layer_6 = tf.reshape(layer_6, [-1, batch_x_shape[0], n_character])# Output shape: [n_steps, batch_size, n_character]return layer_6"""
used to create a variable in CPU memory.
"""
def variable_on_cpu(name, shape, initializer):# Use the /cpu:0 device for scoped operationswith tf.device('/cpu:0'):# Create or get apropos variablevar = tf.get_variable(name=name, shape=shape, initializer=initializer)return varwav_path='H:/tensorflow_projects/chap9/data_thchs30/wav/train'
label_file='H:/tensorflow_projects/chap9/data_thchs30/doc/trans/train.word.txt'wav_files, labels = get_wavs_lables(wav_path,label_file)
print(wav_files[0], labels[0])
# wav/train/A11/A11_0.WAV -> 绿 是 阳春 烟 景 大块 文章 的 底色 四月 的 林 峦 更是 绿 得 鲜活 秀媚 诗意 盎然  print("wav:",len(wav_files),"label",len(labels))# 字表
all_words = []
for label in labels:  #print(label)    all_words += [word for word in label]
counter = Counter(all_words)
words = sorted(counter)
words_size= len(words)
word_num_map = dict(zip(words, range(words_size))) print('字表大小:', words_size) n_input = 26#计算美尔倒谱系数的个数
n_context = 9#对于每个时间点，要包含上下文样本的个数
batch_size =8
def next_batch(labels, start_idx = 0,batch_size=1,wav_files = wav_files):filesize = len(labels)end_idx = min(filesize, start_idx + batch_size)idx_list = range(start_idx, end_idx)txt_labels = [labels[i] for i in idx_list]wav_files = [wav_files[i] for i in idx_list](source, audio_len, target, transcript_len) = get_audio_and_transcriptch(None,wav_files,n_input,n_context,word_num_map,txt_labels)start_idx += batch_size# Verify that the start_idx is not larger than total available sample sizeif start_idx >= filesize:start_idx = -1# Pad input to max_time_step of this batchsource, source_lengths = pad_sequences(source)#如果多个文件将长度统一，支持按最大截断或补0sparse_labels = sparse_tuple_from(target)return start_idx,source, source_lengths, sparse_labelsnext_idx,source,source_len,sparse_lab = next_batch(labels,0,batch_size)
print(len(sparse_lab))
print(np.shape(source))
#print(sparse_lab)
t = sparse_tuple_to_texts_ch(sparse_lab,words)
print(t[0])
#source已经将变为前9（不够补空）+本身+后9，每个26，第一个顺序是第10个的数据。# shape = [batch_size, max_stepsize, n_input + (2 * n_input * n_context)]
# the batch_size and max_stepsize每步都是变长的。
input_tensor = tf.placeholder(tf.float32, [None, None, n_input + (2 * n_input * n_context)], name='input')#语音log filter bank or MFCC features
# Use sparse_placeholder; will generate a SparseTensor, required by ctc_loss op.
targets = tf.sparse_placeholder(tf.int32, name='targets')#文本
# 1d array of size [batch_size]
seq_length = tf.placeholder(tf.int32, [None], name='seq_length')#序列长
keep_dropout= tf.placeholder(tf.float32)# logits is the non-normalized output/activations from the last layer.
# logits will be input for the loss function.
# nn_model is from the import statement in the load_model function
logits = BiRNN_model( input_tensor, tf.to_int64(seq_length), n_input, n_context,words_size +1,keep_dropout)#调用ctc loss
avg_loss = tf.reduce_mean(ctc_ops.ctc_loss(targets, logits, seq_length))#[optimizer]
learning_rate = 0.001
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(avg_loss)with tf.name_scope("decode"):    decoded, log_prob = ctc_ops.ctc_beam_search_decoder( logits, seq_length, merge_repeated=False)with tf.name_scope("accuracy"):distance = tf.edit_distance( tf.cast(decoded[0], tf.int32), targets)# 计算label error rate (accuracy)ler = tf.reduce_mean(distance, name='label_error_rate')epochs = 100
savedir = "log/yuyinchalltest/"
saver = tf.train.Saver(max_to_keep=1) # 生成saver
# create the session
sess = tf.Session()
# 没有模型的话，就重新初始化
sess.run(tf.global_variables_initializer())kpt = tf.train.latest_checkpoint(savedir)
print("kpt:",kpt)
startepo= 0
if kpt!=None:saver.restore(sess, kpt) ind = kpt.find("-")startepo = int(kpt[ind+1:])print(startepo)# 准备运行训练步骤
section = '\n{0:=^40}\n'
print(section.format('Run training epoch'))train_start = time.time()
for epoch in range(epochs):#样本集迭代次数epoch_start = time.time()if epoch<startepo:continueprint("epoch start:",epoch,"total epochs= ",epochs)
#######################run batch####n_batches_per_epoch = int(np.ceil(len(labels) / batch_size))print("total loop ",n_batches_per_epoch,"in one epoch，",batch_size,"items in one loop") train_cost = 0train_ler = 0next_idx =0for batch in range(n_batches_per_epoch):#一次batch_size，取多少次#取数据next_idx,source,source_lengths,sparse_labels = \next_batch(labels,next_idx ,batch_size)feed = {input_tensor: source, targets: sparse_labels,seq_length: source_lengths,keep_dropout:keep_dropout_rate}#计算 avg_loss optimizer ;batch_cost, _ = sess.run([avg_loss, optimizer],  feed_dict=feed )train_cost += batch_cost if (batch +1)%20 == 0:print('loop:',batch, 'Train cost: ', train_cost/(batch+1))feed2 = {input_tensor: source, targets: sparse_labels,seq_length: source_lengths,keep_dropout:1.0}d,train_ler = sess.run([decoded[0],ler], feed_dict=feed2)dense_decoded = tf.sparse_tensor_to_dense( d, default_value=-1).eval(session=sess)dense_labels = sparse_tuple_to_texts_ch(sparse_labels,words)counter =0print('Label err rate: ', train_ler)for orig, decoded_arr in zip(dense_labels, dense_decoded):# convert to stringsdecoded_str = ndarray_to_text_ch(decoded_arr,words)print(' file {}'.format( counter))print('Original: {}'.format(orig))print('Decoded:  {}'.format(decoded_str))counter=counter+1breakepoch_duration = time.time() - epoch_startlog = 'Epoch {}/{}, train_cost: {:.3f}, train_ler: {:.3f}, time: {:.2f} sec'print(log.format(epoch ,epochs, train_cost,train_ler,epoch_duration))saver.save(sess, savedir+"yuyinch.cpkt", global_step=epoch)train_duration = time.time() - train_start
print('Training complete, total duration: {:.2f} min'.format(train_duration / 60))sess.close()   

H:/tensorflow_projects/chap9/data_thchs30/wav/train\A11\A11_0.WAV 绿 是 阳春 烟 景 大块 文章 的 底色 四月 的 林 峦 更是 绿 得 鲜活 秀媚 诗意 盎然
wav: 8911 label 8911
字表大小: 2666
3
(8, 584, 494)
绿 是 阳春 烟 景 大块 文章 的 底色 四月 的 林 峦 更是 绿 得 鲜活 秀媚 诗意 盎然
WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:197: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From C:\Users\adward\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:53: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:* https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md* https://github.com/tensorflow/addons
If you depend on functionality not listed there, please file an issue.WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:73: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.
WARNING:tensorflow:From <ipython-input-1-68bf84832ef0>:89: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API
WARNING:tensorflow:From C:\Users\adward\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.
Instructions for updating:
Please use `keras.layers.RNN(cell)`, which is equivalent to this API
WARNING:tensorflow:From C:\Users\adward\Anaconda3\lib\site-packages\tensorflow\python\ops\rnn.py:626: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
kpt: None===========Run training epoch===========epoch start: 0 total epochs=  100
total loop  1114 in one epoch， 8 items in one loop
loop: 19 Train cost:  858.0828048706055
Label err rate:  0.98041475file 0
Original: 小 鸳鸯 成长 很快 到了 深秋 在 北方 出生 的 小 鸳鸯 便 能 跟随 鸳鸯 大群 一起 南下 越冬 了
Decoded:
loop: 39 Train cost:  575.0958625793457
Label err rate:  0.9796773file 0
Original: 马 晓 年 作画 的 特征 与 神韵 其 画风 区别 于 文人 画 和 年画 可谓 雅俗共赏
Decoded:
loop: 59 Train cost:  475.83005854288734
Label err rate:  0.97848856file 0
Original: 另外 加工 修理 和 修配 业务 不 属于 营业税 的 应 税 劳务 不 缴纳 营业税
Decoded:
loop: 79 Train cost:  422.05160999298096
Label err rate:  0.9786789file 0
Original: 这 碗 离 娘 饭 姑娘 再有 离 娘 痛楚 也 要 每样 都 吃 一点 才 算 循 规 遵 俗 的
Decoded:
loop: 99 Train cost:  389.91693817138673
Label err rate:  0.97975487file 0
Original: 国防大学 政委 王茂 润 中将 与 延安 八一 敬老院 八 十五岁 的 老红军 柴 福耀 亲切 交谈
Decoded:
loop: 119 Train cost:  368.8437096913656
Label err rate:  0.973868file 0
Original: 仅 绘画 而论 齐白石 是 巍巍 昆仑 可 这位 附庸风雅 的 门外汉 连 一块 石头 都 不是
Decoded:   的
loop: 139 Train cost:  353.2190541948591
Label err rate:  0.95776415file 0
Original: 这次 全国 青年 排球 联赛 共 设 天津 舟山 武汉 三个 赛区 每个 赛区 的 前 两 名将 参加 复赛
Decoded:   的
loop: 159 Train cost:  341.92204132080076
Label err rate:  0.9404359file 0
Original: 人代会 开幕 那天 奥 申 委 执行主席 伍 绍 祖 在 人民大会堂 东 大厅 一 露面 就 被 团团 围住
Decoded:   的 龚龚
loop: 179 Train cost:  333.7818296644423
Label err rate:  0.9725291file 0
Original: 主要原因 是 世界 红 白藤 主产 国 印尼 禁止 红 白藤 原料 出口 导致 国际 红 白藤 原料 价格 上涨
Decoded:   龚龚
loop: 199 Train cost:  326.5086625671387
Label err rate:  0.93530095file 0
Original: 赛 距 为 职业 运动员 最长 二十四 千米 业余 运动员 最长 二十一 千米 青年 运动员 最长 十五 千米
Decoded:   的 龚龚
loop: 219 Train cost:  321.0388072620739
Label err rate:  0.980114file 0
Original: 据说 母亲 生下 没 一年 功夫 染 上了 一 种 怪病 不吃 不喝 乱 哭 乱 闹 急 得 外婆 满山遍野 去找 草药 来 驱鬼
Decoded:
loop: 239 Train cost:  316.15502440134685
Label err rate:  0.9510975file 0
Original: 所谓 软 是指 软件 包括 专利 使用权 设计 图纸 技术 诀窍 纯 技术 引进 不 带 硬件
Decoded:   的
loop: 259 Train cost:  311.4005855266864
Label err rate:  0.9620612file 0
Original: 要知道 秦俑 二 号 坑 的 军 阵 性质 首先 要 了解一下 学者 们 关于 秦俑 一号 坑 军 阵 性质 的 探讨
Decoded:   龚龚
loop: 279 Train cost:  307.48678610665456
Label err rate:  0.9382373file 0
Original: 使用 这样的 牙刷 会 不同 程度 地 刺伤 牙龈 造成 牙龈出血 并 导致 牙齿 过度 磨损
Decoded:   的 龚龚
loop: 299 Train cost:  304.18842356363933
Label err rate:  0.9661161file 0
Original: 新华社 基 多 二月 六 日电 最近 二 十天 来 厄瓜多尔 全国 疟疾 迅速 蔓延 已有 多 人 死亡
Decoded:   的
loop: 319 Train cost:  301.10915036201476

实例69：利用RNN训练语言模型

# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
from tensorflow.contrib import rnn
import random
import time
from collections import Counterstart_time = time.time()
def elapsed(sec):if sec<60:return str(sec) + " sec"elif sec<(60*60):return str(sec/60) + " min"else:return str(sec/(60*60)) + " hr"# Target log path
tf.reset_default_graph()
training_file = 'wordstest.txt'#中文多文件
def readalltxt(txt_files):labels = []for txt_file in txt_files:target = get_ch_lable(txt_file)labels.append(target)  return labels#中文字
def get_ch_lable(txt_file):  labels= ""with open(txt_file, 'rb') as f:for label in f: #labels =label.decode('utf-8')labels =labels+label.decode('gb2312')return  labels#优先转文件里的字符到向量
def get_ch_lable_v(txt_file,word_num_map,txt_label=None):words_size = len(word_num_map)   to_num = lambda word: word_num_map.get(word, words_size) if txt_file!= None:txt_label = get_ch_lable(txt_file)labels_vector = list(map(to_num, txt_label)) return labels_vector  training_data =get_ch_lable(training_file)print("Loaded training data...")print(len(training_data))
counter = Counter(training_data)
words = sorted(counter)
words_size= len(words)
word_num_map = dict(zip(words, range(words_size))) print('字表大小:', words_size)
wordlabel = get_ch_lable_v(training_file,word_num_map)#参数设置
learning_rate = 0.001
training_iters = 10000
display_step = 1000
n_input = 4n_hidden1 = 256
n_hidden2 = 512
n_hidden3 = 512
# tf Graph input
x = tf.placeholder("float", [None, n_input,1])
wordy = tf.placeholder("float", [None, words_size])x1 = tf.reshape(x, [-1, n_input])
x2 = tf.split(x1,n_input,1)
# 2-layer LSTM, each layer has n_hidden units.
rnn_cell = rnn.MultiRNNCell([rnn.LSTMCell(n_hidden1),rnn.LSTMCell(n_hidden2),rnn.LSTMCell(n_hidden3)])# generate prediction
outputs, states = rnn.static_rnn(rnn_cell, x2, dtype=tf.float32)#  last output
pred = tf.contrib.layers.fully_connected(outputs[-1],words_size,activation_fn = None)# Loss optimizer
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=wordy))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)# Model evaluation
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(wordy,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))savedir = "log/rnnword/"
saver = tf.train.Saver(max_to_keep=1) # 生成saver# 启动session
with tf.Session() as session:session.run(tf.global_variables_initializer())step = 0offset = random.randint(0,n_input+1)end_offset = n_input + 1acc_total = 0loss_total = 0kpt = tf.train.latest_checkpoint(savedir)print("kpt:",kpt)startepo= 0if kpt!=None:saver.restore(session, kpt) ind = kpt.find("-")startepo = int(kpt[ind+1:])print(startepo)step = startepowhile step < training_iters:# 随机取一个位置偏移if offset > (len(training_data)-end_offset):offset = random.randint(0, n_input+1)inwords = [ [wordlabel[ i]] for i in range(offset, offset+n_input) ]#按照指定的位置偏移获取后面的4个文字向量当作输入inwords = np.reshape(np.array(inwords), [-1, n_input, 1])out_onehot= np.zeros([words_size], dtype=float)out_onehot[wordlabel[offset+n_input]] = 1.0out_onehot = np.reshape(out_onehot,[1,-1])#所有的字都变成onehot_, acc, lossval, onehot_pred = session.run([optimizer, accuracy, loss, pred],feed_dict={x: inwords, wordy: out_onehot})loss_total += lossvalacc_total += accif (step+1) % display_step == 0:print("Iter= " + str(step+1) + ", Average Loss= " + \"{:.6f}".format(loss_total/display_step) + ", Average Accuracy= " + \"{:.2f}%".format(100*acc_total/display_step))acc_total = 0loss_total = 0in2 = [words [wordlabel[i]] for i in range(offset, offset + n_input)]out2 = words [wordlabel[offset + n_input]]out_pred=words[int(tf.argmax(onehot_pred, 1).eval())]print("%s - [%s] vs [%s]" % (in2,out2,out_pred))            saver.save(session, savedir+"rnnwordtest.cpkt", global_step=step)step += 1offset += (n_input+1)#中间隔了一个，作为预测print("Finished!")saver.save(session, savedir+"rnnwordtest.cpkt", global_step=step)print("Elapsed time: ", elapsed(time.time() - start_time))while True:prompt = "请输入%s个字: " % n_inputsentence = input(prompt)inputword = sentence.strip()if len(inputword) != n_input:print("您输入的字符长度为：",len(inputword),"请输入4个字")continuetry:inputword = get_ch_lable_v(None,word_num_map,inputword)for i in range(32):keys = np.reshape(np.array(inputword), [-1, n_input, 1])onehot_pred = session.run(pred, feed_dict={x: keys})onehot_pred_index = int(tf.argmax(onehot_pred, 1).eval())sentence = "%s%s" % (sentence,words[onehot_pred_index])inputword = inputword[1:]inputword.append(onehot_pred_index)print(sentence)except:print("该字我还没学会")

实例70：word2vect

# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
import random
import collections
from collections import Counter
import jiebafrom sklearn.manifold import TSNE
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rcParams['font.sans-serif']=['SimHei']#用来正常显示中文标签
mpl.rcParams['font.family'] = 'STSong'
mpl.rcParams['font.size'] = 20training_file = '人体阴阳与电能.txt'#中文字
def get_ch_lable(txt_file):  labels= ""with open(txt_file, 'rb') as f:for label in f: #labels =label.decode('utf-8')labels =labels+label.decode('gb2312')return  labels#分词
def fenci(training_data):seg_list = jieba.cut(training_data)  # 默认是精确模式  training_ci = " ".join(seg_list)training_ci = training_ci.split()#以空格将字符串分开training_ci = np.array(training_ci)training_ci = np.reshape(training_ci, [-1, ])return training_cidef build_dataset(words, n_words):"""Process raw inputs into a dataset."""count = [['UNK', -1]]count.extend(collections.Counter(words).most_common(n_words - 1))dictionary = dict()for word, _ in count:dictionary[word] = len(dictionary)data = list()unk_count = 0for word in words:if word in dictionary:index = dictionary[word]else:index = 0  # dictionary['UNK']unk_count += 1data.append(index)count[0][1] = unk_countreversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))return data, count, dictionary, reversed_dictionarytraining_data =get_ch_lable(training_file)
print("总字数",len(training_data))
training_ci =fenci(training_data)
#print(training_ci)
print("总词数",len(training_ci))
training_label, count, dictionary, words = build_dataset(training_ci, 350)words_size = len(dictionary)
print("字典词数",words_size)
#print(training_label)#将文本转为词向量
#print(words)#每个编号对应的词
#print(dictionary)#每个词对应的编号
#print(count)#每个词对应的个数
####################################################
print('Sample data', training_label[:10], [words[i] for i in training_label[:10]])
data_index = 0
def generate_batch(data,batch_size, num_skips, skip_window):global data_indexassert batch_size % num_skips == 0assert num_skips <= 2 * skip_windowbatch = np.ndarray(shape=(batch_size), dtype=np.int32)labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)span = 2 * skip_window + 1  # [ skip_window target skip_window ]buffer = collections.deque(maxlen=span)if data_index + span > len(data):data_index = 0buffer.extend(data[data_index:data_index + span])data_index += spanfor i in range(batch_size // num_skips):target = skip_window  # target label at the center of the buffertargets_to_avoid = [skip_window]for j in range(num_skips):while target in targets_to_avoid:target = random.randint(0, span - 1)targets_to_avoid.append(target)batch[i * num_skips + j] = buffer[skip_window]labels[i * num_skips + j, 0] = buffer[target]if data_index == len(data):#print(data_index,len(data),span,len(data[:span]))#buffer[:] = data[:span]buffer = data[:span]data_index = spanelse:buffer.append(data[data_index])data_index += 1# Backtrack a little bit to avoid skipping words in the end of a batchdata_index = (data_index + len(data) - span) % len(data)return batch, labelsbatch, labels = generate_batch(training_label,batch_size=8, num_skips=2, skip_window=1)for i in range(8):# 取第一个字，后一个是标签，再取其前一个字当标签，print(batch[i], words[batch[i]], '->', labels[i, 0], words[labels[i, 0]])batch_size = 128
embedding_size = 128  # Dimension of the embedding vector.
skip_window = 1       # How many words to consider left and right.
num_skips = 2         # How many times to reuse an input to generate a label.valid_size = 16     # Random set of words to evaluate similarity on.
valid_window =np.int32( words_size/2 ) # Only pick dev samples in the head of the distribution.
print("valid_window",valid_window)
valid_examples = np.random.choice(valid_window, valid_size, replace=False)#0-words_size/2,中的数取16个。不能重复。
num_sampled = 64    # Number of negative examples to sample.tf.reset_default_graph()train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)# Ops and variables pinned to the CPU because of missing GPU implementation
with tf.device('/cpu:0'):# Look up embeddings for inputs.embeddings = tf.Variable(tf.random_uniform([words_size, embedding_size], -1.0, 1.0))#94个，每个128个向量embed = tf.nn.embedding_lookup(embeddings, train_inputs)# Construct the variables for the NCE lossnce_weights = tf.Variable( tf.truncated_normal([words_size, embedding_size],stddev=1.0 / tf.sqrt(np.float32(embedding_size))))nce_biases = tf.Variable(tf.zeros([words_size]))# Compute the average NCE loss for the batch.
# tf.nce_loss automatically draws a new sample of the negative labels each
# time we evaluate the loss.
loss = tf.reduce_mean(
tf.nn.nce_loss(weights=nce_weights, biases=nce_biases,labels=train_labels, inputs=embed,num_sampled=num_sampled, num_classes=words_size))# Construct the SGD optimizer using a learning rate of 1.0.
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)# Compute the cosine similarity between minibatch examples and all embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)
print("________________________",similarity.shape)#Begin training.
num_steps = 100001
with tf.Session() as sess:sess.run( tf.global_variables_initializer() )print('Initialized')average_loss = 0for step in range(num_steps):batch_inputs, batch_labels = generate_batch(training_label, batch_size, num_skips, skip_window)feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}# We perform one update step by evaluating the optimizer op (including it# in the list of returned values for session.run()_, loss_val = sess.run([optimizer, loss], feed_dict=feed_dict)average_loss += loss_val#通过打印测试可以看到  embed的值在逐渐的被调节
#        emv = sess.run(embed,feed_dict = {train_inputs: [37,18]})
#        print("emv-------------------",emv[0])if step % 2000 == 0:if step > 0:average_loss /= 2000# The average loss is an estimate of the loss over the last 2000 batches.print('Average loss at step ', step, ': ', average_loss)average_loss = 0# Note that this is expensive (~20% slowdown if computed every 500 steps)if step % 10000 == 0:sim = similarity.eval(session=sess)#print(valid_size)for i in range(valid_size):valid_word = words[valid_examples[i]]#print("valid_word",valid_word)#16top_k = 8  # number of nearest neighborsnearest = (-sim[i, :]).argsort()[1:top_k + 1]  #argsort函数返回的是数组值从小到大的索引值#print("nearest",nearest,top_k)log_str = 'Nearest to %s:' % valid_wordfor k in range(top_k):close_word = words[nearest[k]]log_str = '%s,%s' % (log_str, close_word)print(log_str)final_embeddings = normalized_embeddings.eval()def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'plt.figure(figsize=(18, 18))  # in inchesfor i, label in enumerate(labels):x, y = low_dim_embs[i, :]plt.scatter(x, y)plt.annotate(label,xy=(x, y),xytext=(5, 2), textcoords='offset points',ha='right',va='bottom')plt.savefig(filename)try:# pylint: disable=g-import-not-at-toptsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)plot_only = 80#输出100个词low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])labels = [words[i] for i in range(plot_only)]#print(labels)plot_with_labels(low_dim_embs, labels)except ImportError:print('Please install sklearn, matplotlib, and scipy to show embeddings.')

实例71：word2vect自定义候选采样

# -*- coding: utf-8 -*-
import numpy as np
import tensorflow as tf
import random
import collections
from collections import Counter
import jiebafrom sklearn.manifold import TSNE
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rcParams['font.sans-serif']=['SimHei']#用来正常显示中文标签
mpl.rcParams['font.family'] = 'STSong'
mpl.rcParams['font.size'] = 20training_file = '人体阴阳与电能.txt'#中文字
def get_ch_lable(txt_file):  labels= ""with open(txt_file, 'rb') as f:for label in f: #labels =label.decode('utf-8')labels =labels+label.decode('gb2312')return  labels#分词
def fenci(training_data):seg_list = jieba.cut(training_data)  # 默认是精确模式  training_ci = " ".join(seg_list)training_ci = training_ci.split()#以空格将字符串分开training_ci = np.array(training_ci)training_ci = np.reshape(training_ci, [-1, ])return training_cidef build_dataset(words, n_words):"""Process raw inputs into a dataset."""count = [['UNK', -1]]count.extend(collections.Counter(words).most_common(n_words - 1))dictionary = dict()vocab_freqs = []for word, nvocab in count:dictionary[word] = len(dictionary)vocab_freqs.append(nvocab)data = list()unk_count = 0for word in words:if word in dictionary:index = dictionary[word]else:index = 0  # dictionary['UNK']unk_count += 1data.append(index)count[0][1] = unk_countreversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))return data, count, dictionary, reversed_dictionary,vocab_freqstraining_data =get_ch_lable(training_file)
print("总字数",len(training_data))
training_ci =fenci(training_data)
#print(training_ci)
print("总词数",len(training_ci))
training_label, count, dictionary, words,vocab_freqs = build_dataset(training_ci, 350)words_size = len(dictionary)
print("字典词数",words_size)
#print(training_label)#将文本转为词向量
#print(words)#每个编号对应的词
#print(dictionary)#每个词对应的编号
#print(count)#每个词对应的个数
####################################################
print('Sample data', training_label[:10], [words[i] for i in training_label[:10]])
data_index = 0
def generate_batch(data,batch_size, num_skips, skip_window):global data_indexassert batch_size % num_skips == 0assert num_skips <= 2 * skip_windowbatch = np.ndarray(shape=(batch_size), dtype=np.int32)labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)span = 2 * skip_window + 1  # [ skip_window target skip_window ]buffer = collections.deque(maxlen=span)if data_index + span > len(data):data_index = 0buffer.extend(data[data_index:data_index + span])data_index += spanfor i in range(batch_size // num_skips):target = skip_window  # target label at the center of the buffertargets_to_avoid = [skip_window]for j in range(num_skips):while target in targets_to_avoid:target = random.randint(0, span - 1)targets_to_avoid.append(target)batch[i * num_skips + j] = buffer[skip_window]labels[i * num_skips + j, 0] = buffer[target]if data_index == len(data):#print(data_index,len(data),span,len(data[:span]))#buffer[:] = data[:span]buffer = data[:span]data_index = spanelse:buffer.append(data[data_index])data_index += 1# Backtrack a little bit to avoid skipping words in the end of a batchdata_index = (data_index + len(data) - span) % len(data)return batch, labelsbatch, labels = generate_batch(training_label,batch_size=8, num_skips=2, skip_window=1)for i in range(8):# 取第一个字，后一个是标签，再取其前一个字当标签，print(batch[i], words[batch[i]], '->', labels[i, 0], words[labels[i, 0]])batch_size = 128
embedding_size = 128  # Dimension of the embedding vector.
skip_window = 1       # How many words to consider left and right.
num_skips = 2         # How many times to reuse an input to generate a label.valid_size = 16     # Random set of words to evaluate similarity on.
valid_window =np.int32( words_size/2 ) # Only pick dev samples in the head of the distribution.
print("valid_window",valid_window)
valid_examples = np.random.choice(valid_window, valid_size, replace=False)#0-words_size/2,中的数取16个。不能重复。
num_sampled = 64    # Number of negative examples to sample.tf.reset_default_graph()train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)# Ops and variables pinned to the CPU because of missing GPU implementation
with tf.device('/cpu:0'):# Look up embeddings for inputs.embeddings = tf.Variable(tf.random_uniform([words_size, embedding_size], -1.0, 1.0))#94个，每个128个向量embed = tf.nn.embedding_lookup(embeddings, train_inputs)# Construct the variables for the NCE lossnce_weights = tf.Variable( tf.truncated_normal([words_size, embedding_size],stddev=1.0 / tf.sqrt(np.float32(embedding_size))))nce_biases = tf.Variable(tf.zeros([words_size]))vocab_freqs[0] = 90sampled = tf.nn.fixed_unigram_candidate_sampler(true_classes=tf.cast(train_labels,tf.int64),num_true=1,num_sampled=num_sampled,unique=True,range_max=words_size,unigrams=vocab_freqs)loss = tf.reduce_mean(
tf.nn.sampled_softmax_loss(weights=nce_weights, biases=nce_biases,labels=train_labels, inputs=embed,num_sampled=num_sampled, num_classes=words_size,sampled_values=sampled))# Construct the SGD optimizer using a learning rate of 1.0.
optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)# Compute the cosine similarity between minibatch examples and all embeddings.
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)
print("________________________",similarity.shape)#Begin training.
num_steps = 100001
with tf.Session() as sess:sess.run( tf.global_variables_initializer() )print('Initialized')average_loss = 0for step in range(num_steps):batch_inputs, batch_labels = generate_batch(training_label, batch_size, num_skips, skip_window)feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}# We perform one update step by evaluating the optimizer op (including it# in the list of returned values for session.run()_, loss_val = sess.run([optimizer, loss], feed_dict=feed_dict)average_loss += loss_val#通过打印测试可以看到  embed的值在逐渐的被调节
#        emv = sess.run(embed,feed_dict = {train_inputs: [37,18]})
#        print("emv-------------------",emv[0])if step % 2000 == 0:if step > 0:average_loss /= 2000# The average loss is an estimate of the loss over the last 2000 batches.print('Average loss at step ', step, ': ', average_loss)average_loss = 0# Note that this is expensive (~20% slowdown if computed every 500 steps)if step % 10000 == 0:sim = similarity.eval(session=sess)#print(valid_size)for i in range(valid_size):valid_word = words[valid_examples[i]]#print("valid_word",valid_word)#16top_k = 8  # number of nearest neighborsnearest = (-sim[i, :]).argsort()[1:top_k + 1]  #argsort函数返回的是数组值从小到大的索引值#print("nearest",nearest,top_k)log_str = 'Nearest to %s:' % valid_wordfor k in range(top_k):close_word = words[nearest[k]]log_str = '%s,%s' % (log_str, close_word)print(log_str)final_embeddings = normalized_embeddings.eval()def plot_with_labels(low_dim_embs, labels, filename='tsne.png'):assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'plt.figure(figsize=(18, 18))  # in inchesfor i, label in enumerate(labels):x, y = low_dim_embs[i, :]plt.scatter(x, y)plt.annotate(label,xy=(x, y),xytext=(5, 2), textcoords='offset points',ha='right',va='bottom')plt.savefig(filename)try:# pylint: disable=g-import-not-at-toptsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)plot_only = 80#输出100个词low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])labels = [words[i] for i in range(plot_only)]#print(labels)plot_with_labels(low_dim_embs, labels)except ImportError:print('Please install sklearn, matplotlib, and scipy to show embeddings.')

[TF进阶] 循环神经网络相关推荐

1. 动手学深度学习-过拟合、欠拟合及其解决方案；梯度消失、梯度爆炸；循环神经网络进阶

   一.过拟合.欠拟合及其解决方案 前序知识点 模型选择 正则化:添加参数的惩罚项,防止过拟合.有L2正则化与L1正则化. 奥卡姆剃刀原则:有多个假设模型时,应该选择假设条件最少的. 模型的泛化能力:指模 ...

2. AI之循环神经网络进阶

   AI之循环神经网络进阶 GRU 从零实现GNU 载入数据集 初始化参数 GRU模型 训练模型 简洁实现 LSTM 初始化参数 建立LSTM模型 训练模型 简洁实现 深度循环神经网络 双向循环神经网络 ...

3. Task03：过拟合、欠拟合及其解决方案；梯度消失、梯度爆炸；循环神经网络进阶

   写这个名字有一点标题党的意思哈,为了能够被搜索引擎检索到.下面进入正题,主要是以描述性语言加代码的形式说出我目前的理解,可能在理解过程中会有偏差,请见谅哈. 过拟合和欠拟合: 过拟合,顾名思义,就是模 ...

4. 《动手学深度学习》task3_3 循环神经网络进阶

   目录 GRU GRU 重置门和更新门 候选隐藏状态 隐藏状态 GRU的实现 载入数据集 初始化参数 GRU模型 训练模型 简洁实现 LSTM 长短期记忆 输入门.遗忘门和输出门 候选记忆细胞 记忆细胞 ...

5. CS224N笔记(四) Lecture 7：循环神经网络RNN的进阶——LSTM与GRU

   本文将介绍两种比RNN更好地应对梯度消失问题的模型结构--LSTM和GRU,文章以CS224N的课件和材料为基础,重点分析他们的结构特点和梯度计算,在梯度消失的解决策略上进行了深入探究,并进一步分析它 ...

6. 《14天动手学深度学习》——循环神经网络进阶

   GRU RNN存在的问题:梯度较容易出现衰减或爆炸(BPTT) ⻔控循环神经⽹络:捕捉时间序列中时间步距离较⼤的依赖关系 RNN: Ht=ϕ(XtWxh+Ht−1Whh+bh)Ht=ϕ(XtWxh+H ...

7. RNN 扫盲：循环神经网络解读及其 PyTorch 应用实现

   点击上方"小白学视觉",选择加"星标"或"置顶" 重磅干货,第一时间送达 来自 | 知乎 作者 | Lucas 地址 | https://z ...

8. 干货 | 循环神经网络LSTM的数学过程

   击上方"小白学视觉",选择加"星标"或"置顶" 重磅干货,第一时间送达本文转自 | OpenCV学堂 引言 长短时记忆(long short ...

9. 人人都能看懂的循环神经网络RNN

   循环神经网络 基础篇   我们假设您有一个管家,他很擅长做苹果派.汉堡以及炸鸡这三样食物.管家制作食物的种类取决于天气,若是晴天,他会做苹果派:若是雨天,他会做汉堡.这样制作食物的规则很容易用神经网络 ...

10. 循环神经网络基础介绍

    在应用循环神经网络的过程中,还是会有些地方疑惑,所以还是要回归下问题的本质. 学而不思则惘,思而不学则怠.. 1. 循环神经网路简介 首先循环神经网络的主要用途是处理和预测序列数据.在之前的全链接神经 ...

最新文章

1. MapReduce简述、工作流程
2. swift5.x基础知识入门
3. html如何与php,html页面怎么跟php文件连接
4. 前端学习（3224）：字符串形式
5. linux系统编译qt代码需要编译qt,linux下如何静态编译QT库，以及如何静态编译自己的程序...
6. Flash 杂志《summer tree》 第七期发布
7. HEVC将会取代H.264的原因
8. HTTP传输协议原理
9. 自适应直方图均衡化matlab 程序,直方图均衡化程序 matlab
10. QOS-CBWFQ\QOS-LLQ\QOS-PQ\QOS-CQ\QOS-WFQ
11. java脚本引擎Rhino 入门
12. ENVI 5.3 计算NDVI的几种方法及结果的差异分析
13. 云原生发展趋势-2022
14. createfile调用失败_Java NIO Files.createFile()以NoSuchFileException失败
15. 大前端CPU优化技术--NEON自动向量化
16. inet manual 和 static 区别
17. mars3d与echart图表结合使用
18. 深圳仙湖弘法寺佛家感言
19. 计算机基础和综合实验,计算机基础与综合编程实验报告.doc
20. github网页版使用教程

热门文章

1. oracle+in条件优化,Oracle语句优化30个规则详解
2. unity 控制人物模型移动
3. tcpclient java_[Java] 网络-01 TCPClient / TCPServer
4. VS2013 VS2015应用程序无法正常启动0xc000007b。
5. Oracle数据访问组件ODAC(Oracle Data Access Components)安装和使用
6. 黑客为什么不攻击网贷平台？
7. 需要升的不是舱，是京东的价值观！
8. AgentWeb嵌套h5 上传图片文件
9. spring Aop 实现原理
10. oracle中文转全拼音,汉字转拼音的Oracle函数