pure seq2seq

1.pure seq2seq

enc_inputs = tf.placeholder(tf.int32,shape=[None, enc_sentence_length],name='input_sentences')            #enc_sentence_length:为encoder端的time_setp数sequence_lengths = tf.placeholder(tf.int32,shape=[None],name='sentences_length')           #为一个batch中的每一个sentence长度dec_inputs = tf.placeholder(tf.int32,shape=[None, dec_sentence_length+1],name='output_sentences')           #dec_sentence_length+1为decoder端的time_step数# batch major => time major
enc_inputs_t = tf.transpose(enc_inputs, perm=[1,0])
dec_inputs_t = tf.transpose(dec_inputs, perm=[1,0])#先把time_step的维度调至第一维with tf.device('/cpu:0'):enc_Wemb = tf.get_variable('enc_word_emb',initializer=tf.random_uniform([enc_vocab_size+1, enc_emb_size]))#encoder端的embed矩阵dec_Wemb = tf.get_variable('dec_word_emb',initializer=tf.random_uniform([dec_vocab_size+2, dec_emb_size]))#decoder端的embed矩阵dec_out_W = tf.get_variable('dec_out_W',initializer=tf.random_uniform([hidden_size, dec_vocab_size+2]))dec_out_b = tf.get_variable('dec_out_b',initializer=tf.random_uniform([dec_vocab_size+2]))with tf.variable_scope('encoder'):enc_emb_inputs = tf.nn.embedding_lookup(enc_Wemb, enc_inputs_t)# enc_emb_inputs:#     list(enc_sent_len) of tensor[batch_size x embedding_size]# Because `static_rnn` takes list inputsenc_emb_inputs = tf.unstack(enc_emb_inputs)enc_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)# enc_sent_len x batch_size x embedding_sizeenc_outputs, enc_last_state = tf.contrib.rnn.static_rnn(cell=enc_cell,inputs=enc_emb_inputs,sequence_length=sequence_lengths,dtype=tf.float32)dec_outputs = []
dec_predictions = []
with tf.variable_scope('decoder') as scope:dec_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)for i in range(dec_sentence_length+1):if i == 0:input_ = tf.nn.embedding_lookup(dec_Wemb, dec_inputs_t[i])state = enc_last_stateelse:scope.reuse_variables()input_ = tf.nn.embedding_lookup(dec_Wemb, dec_prediction)# dec_output: batch_size x dec_vocab_size+2# state:      batch_size x hidden_sizedec_output, state = dec_cell(input_, state)dec_output = tf.nn.xw_plus_b(dec_output, dec_out_W, dec_out_b)# dec_prediction: batch_size x 1dec_prediction = tf.argmax(dec_output, axis=1)dec_outputs.append(dec_output)dec_predictions.append(dec_prediction)# predictions: [batch_size x dec_sentence_lengths+1]
predictions = tf.transpose(tf.stack(dec_predictions), [1,0])# labels & logits: [dec_sentence_length+1 x batch_size x dec_vocab_size+2]
labels = tf.one_hot(dec_inputs_t, dec_vocab_size+2)
logits = tf.stack(dec_outputs)loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits))# training_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(loss)
training_op = tf.train.RMSPropOptimizer(learning_rate=0.0001).minimize(loss)

2.projection和embed wrapping的纯seq2seq

enc_inputs = tf.placeholder(tf.int32,shape=[None, enc_sentence_length],name='input_sentences')sequence_lengths = tf.placeholder(tf.int32,shape=[None],name='sentences_length')dec_inputs = tf.placeholder(tf.int32,shape=[None, dec_sentence_length+1],name='output_sentences')# batch major => time major
enc_inputs_t = tf.transpose(enc_inputs, [1,0])
dec_inputs_t = tf.transpose(dec_inputs, [1,0])with tf.variable_scope('encoder'):enc_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)enc_cell = EmbeddingWrapper(enc_cell, enc_vocab_size+1, enc_emb_size)# enc_sent_len x batch_size x embedding_sizeenc_outputs, enc_last_state = tf.contrib.rnn.static_rnn(cell=enc_cell,inputs=tf.unstack(enc_inputs_t),sequence_length=sequence_lengths,dtype=tf.float32)dec_outputs = []
dec_predictions = []
with tf.variable_scope('decoder') as scope:dec_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)dec_cell = EmbeddingWrapper(dec_cell, dec_vocab_size+2, dec_emb_size)dec_cell = OutputProjectionWrapper(dec_cell, dec_vocab_size+2)for i in range(dec_sentence_length+1):if i == 0:input_ = dec_inputs_t[i]state = enc_last_stateelse:scope.reuse_variables()input_ = dec_prediction# dec_output: batch_size x dec_vocab_size+2# state:      batch_size x hidden_sizedec_output, state = dec_cell(input_, state)# dec_prediction: batch_size x 1dec_prediction = tf.argmax(dec_output, axis=1)dec_outputs.append(dec_output)dec_predictions.append(dec_prediction)# predictions: [batch_size x dec_sentence_lengths+1]
predictions = tf.transpose(tf.stack(dec_predictions), [1,0])# labels & logits: [dec_sentence_length+1 x batch_size x dec_vocab_size+2]
labels = tf.one_hot(dec_inputs_t, dec_vocab_size+2)
logits = tf.stack(dec_outputs)loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits))# training_op = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(loss)
training_op = tf.train.RMSPropOptimizer(learning_rate=0.0001).minimize(loss)

pure seq2seq相关推荐

《Deep Learning With Python second edition》英文版读书笔记：第十一章DL for text: NLP、Transformer、Seq2Seq
文章目录第十一章:Deep learning for text 11.1 Natural language processing: The bird's eye view 11.2 Preparin ...
attention seq2seq transformer bert 学习总结 _20201107
https://blog.csdn.net/weixin_44388679/article/details/102575223 Seq2Seq原理详解一文读懂BERT(原理篇) 2018年的10月1 ...
transformer bert seq2seq 深度学习编码和解码的逻辑-重点
参考文献: 详解从 Seq2Seq模型.RNN结构.Encoder-Decoder模型到 Attention模型 [NLP]Attention Model(注意力模型)学习总结(https://ww ...
基于PyTorch的Seq2Seq翻译模型详细注释介绍（一）
版权声明:本文为博主原创文章,遵循 CC 4.0 BY-SA 版权协议,转载请附上原文出处链接和本声明. 本文链接:https://blog.csdn.net/qysh123/article/deta ...
Pytorch学习记录-torchtext和Pytorch的实例（使用神经网络训练Seq2Seq代码）
Pytorch学习记录-torchtext和Pytorch的实例1 0. PyTorch Seq2Seq项目介绍 1. 使用神经网络训练Seq2Seq 1.1 简介,对论文中公式的解读 1.2 数据预 ...
[实现] 利用 Seq2Seq 预测句子后续字词（Pytorch）2
最近有个任务:利用 RNN 进行句子补全,即给定一个不完整的句子,预测其后续的字词. 本文使用了 Seq2Seq 模型,输入为 5 个中文字词,输出为 1 个中文字词. 目录关于RNN 语料预处理 ...
[实现] 利用 Seq2Seq 预测句子后续字词（Pytorch）
最近有个任务:利用 RNN 进行句子补全,即给定一个不完整的句子,预测其后续的字词. 本文使用了 Seq2Seq 模型,输入为5个中文字词,输出为一个中文字词. 目录关于RNN 语料预处理搭建数据 ...
seq2seq（1）- EncoderDecoder架构
零 seq2seq是从序列到序列的学习过程,最重要的是输入序列和输出序列是可变长的,这种方式就非常灵活了,典型的机器翻译就是这样一个过程. 一最基本的seq2seq网络架构如下所示: 可以看到,en ...
Pytorch系列教程-使用Seq2Seq网络和注意力机制进行机器翻译
NLP FROM SCRATCH: TRANSLATION WITH A SEQUENCE TO SEQUENCE NETWORK AND ATTENTION Author: Sean Roberts ...
PyTorch: 序列到序列模型(Seq2Seq)实现机器翻译实战
版权声明:博客文章都是作者辛苦整理的,转载请注明出处,谢谢!http://blog.csdn.net/m0_37306360/article/details/79318644 简介在这个项目中,我们 ...

pure seq2seq

pure seq2seq相关推荐

最新文章

热门文章