0. 前言
1. 训练语料库
2. 数据预处理
3. 词汇转向量
4. 训练
5. 聊天机器人 - 验证效果

http://naturali.io/deeplearning/chatbot/introduction/2016/04/28/chatbot-part1.html
http://blog.topspeedsnail.com/archives/10735/comment-page-1#comment-1161
http://blog.csdn.net/malefactor/article/details/51901115

wget https://raw.githubusercontent.com/rustch3n/dgk_lost_conv/master/dgk_shooter_min.conv.zip
解压
unzip dgk_shooter_min.conv.zip

https://github.com/rustch3n/dgk_lost_conv

1. 分词(tokenized)
2. 英文单词取词根(stemmed)
3. 英文单词变形的归类(lemmatized)(例如单复数归类)等
4. 此外，例如人名、地名、组织名、URL链接、系统路径等专有名词，我们也可以统一用类型标识符来替代 

convs = []  # conversation set
with open(conv_path, encoding="utf8") as f:one_conv = []  # a complete conversationfor line in f:line = line.strip('\n').replace('/', '')if line == '':continueif line[0] == 'E':if one_conv:convs.append(one_conv)one_conv = []elif line[0] == 'M':one_conv.append(line.split(' ')[1])

# Grasping calligraphy answer answer
ask = []  # ask
response = []  # answers
for conv in convs:if len(conv) == 1:continueif len(conv) % 2 != 0:conv = conv[:-1]for i in range(len(conv)):if i % 2 == 0:ask.append(conv[i])else:response.append(conv[i])

def gen_vocabulary_file(input_file, output_file):vocabulary = {}with open(input_file) as f:counter = 0for line in f:counter += 1tokens = [word for word in line.strip()]for word in tokens:if word in vocabulary:vocabulary[word] += 1else:vocabulary[word] = 1vocabulary_list = START_VOCABULART + sorted(vocabulary, key=vocabulary.get, reverse=True)# For taking 10000 custom character kanjiif len(vocabulary_list) > 10000:vocabulary_list = vocabulary_list[:10000]print(input_file + " phrase table size:", len(vocabulary_list))with open(output_file, "w") as ff:for word in vocabulary_list:ff.write(word + "\n")

def convert_conversation_to_vector(input_file, vocabulary_file, output_file):tmp_vocab = []with open(vocabulary_file, "r") as f:tmp_vocab.extend(f.readlines())tmp_vocab = [line.strip() for line in tmp_vocab]vocab = dict([(x, y) for (y, x) in enumerate(tmp_vocab)])for item in vocab:print item.encode('utf-8')

土 968
"土"字在训练集词汇表中的位置是968，我们就给该字设置一个编码968

# -*- coding: utf-8 -*-import tensorflow as tf  # 0.12
from tensorflow.models.rnn.translate import seq2seq_model
import os
import numpy as np
import mathPAD_ID = 0
GO_ID = 1
EOS_ID = 2
UNK_ID = 3# ask/answer conversation vector file
train_ask_vec_file = 'train_ask.vec'
train_answer_vec_file = 'train_answer.vec'
test_ask_vec_file = 'test_ask.vec'
test_answer_vec_file = 'test_answer.vec'# word table 6000
vocabulary_ask_size = 6000
vocabulary_answer_size = 6000buckets = [(5, 10), (10, 15), (20, 25), (40, 50)]
layer_size = 256
num_layers = 3
batch_size = 64# read *dencode.vec和*decode.vec data into memory
def read_data(source_path, target_path, max_size=None):data_set = [[] for _ in buckets]with tf.gfile.GFile(source_path, mode="r") as source_file:with tf.gfile.GFile(target_path, mode="r") as target_file:source, target = source_file.readline(), target_file.readline()counter = 0while source and target and (not max_size or counter < max_size):counter += 1source_ids = [int(x) for x in source.split()]target_ids = [int(x) for x in target.split()]target_ids.append(EOS_ID)for bucket_id, (source_size, target_size) in enumerate(buckets):if len(source_ids) < source_size and len(target_ids) < target_size:data_set[bucket_id].append([source_ids, target_ids])breaksource, target = source_file.readline(), target_file.readline()return data_setif __name__ == '__main__':model = seq2seq_model.Seq2SeqModel(source_vocab_size=vocabulary_ask_size,target_vocab_size=vocabulary_answer_size,buckets=buckets, size=layer_size, num_layers=num_layers, max_gradient_norm=5.0,batch_size=batch_size, learning_rate=0.5, learning_rate_decay_factor=0.97,forward_only=False)config = tf.ConfigProto()config.gpu_options.allocator_type = 'BFC'  # forbidden out of memorywith tf.Session(config=config) as sess:# 恢复前一次训练ckpt = tf.train.get_checkpoint_state('.')if ckpt != None:print(ckpt.model_checkpoint_path)model.saver.restore(sess, ckpt.model_checkpoint_path)else:sess.run(tf.global_variables_initializer())train_set = read_data(train_ask_vec_file, train_answer_vec_file)test_set = read_data(test_ask_vec_file, test_answer_vec_file)train_bucket_sizes = [len(train_set[b]) for b in range(len(buckets))]train_total_size = float(sum(train_bucket_sizes))train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in range(len(train_bucket_sizes))]loss = 0.0total_step = 0previous_losses = []# continue train，save modle after a decade of timewhile True:random_number_01 = np.random.random_sample()bucket_id = min([i for i in range(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01])encoder_inputs, decoder_inputs, target_weights = model.get_batch(train_set, bucket_id)_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, False)loss += step_loss / 500total_step += 1print(total_step)if total_step % 500 == 0:print(model.global_step.eval(), model.learning_rate.eval(), loss)# if model has't not improve，decrese the learning rateif len(previous_losses) > 2 and loss > max(previous_losses[-3:]):sess.run(model.learning_rate_decay_op)previous_losses.append(loss)# save modelcheckpoint_path = "chatbot_seq2seq.ckpt"model.saver.save(sess, checkpoint_path, global_step=model.global_step)loss = 0.0# evaluation the model by test datasetfor bucket_id in range(len(buckets)):if len(test_set[bucket_id]) == 0:continueencoder_inputs, decoder_inputs, target_weights = model.get_batch(test_set, bucket_id)_, eval_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, True)eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf')print(bucket_id, eval_ppx)

https://www.tensorflow.org/tutorials/seq2seq
http://suriyadeepan.github.io/2016-06-28-easy-seq2seq/

# -*- coding: utf-8 -*-import tensorflow as tf  # 0.12
from tensorflow.models.rnn.translate import seq2seq_model
import os
import sys
import locale
import numpy as npPAD_ID = 0
GO_ID = 1
EOS_ID = 2
UNK_ID = 3train_ask_vocabulary_file = "train_ask_vocabulary.vec"
train_answer_vocabulary_file = "train_answer_vocabulary.vec"def read_vocabulary(input_file):tmp_vocab = []with open(input_file, "r") as f:tmp_vocab.extend(f.readlines())tmp_vocab = [line.strip() for line in tmp_vocab]vocab = dict([(x, y) for (y, x) in enumerate(tmp_vocab)])return vocab, tmp_vocabif __name__ == '__main__':vocab_en, _, = read_vocabulary(train_ask_vocabulary_file)_, vocab_de, = read_vocabulary(train_answer_vocabulary_file)# word table 6000vocabulary_ask_size = 6000vocabulary_answer_size = 6000buckets = [(5, 10), (10, 15), (20, 25), (40, 50)]layer_size = 256num_layers = 3batch_size = 1model = seq2seq_model.Seq2SeqModel(source_vocab_size=vocabulary_ask_size,target_vocab_size=vocabulary_answer_size,buckets=buckets, size=layer_size, num_layers=num_layers, max_gradient_norm=5.0,batch_size=batch_size, learning_rate=0.5, learning_rate_decay_factor=0.99,forward_only=True)model.batch_size = 1with tf.Session() as sess:# restore last trainckpt = tf.train.get_checkpoint_state('.')if ckpt != None:print(ckpt.model_checkpoint_path)model.saver.restore(sess, ckpt.model_checkpoint_path)else:print("model not found")while True:input_string = raw_input('me > ').decode(sys.stdin.encoding or locale.getpreferredencoding(True)).strip()# 退出if input_string == 'quit':exit()# convert the user's input to vectorinput_string_vec = []for words in input_string.strip():input_string_vec.append(vocab_en.get(words, UNK_ID))bucket_id = min([b for b in range(len(buckets)) if buckets[b][0] > len(input_string_vec)])encoder_inputs, decoder_inputs, target_weights = model.get_batch({bucket_id: [(input_string_vec, [])]},bucket_id)_, _, output_logits = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, True)outputs = [int(np.argmax(logit, axis=1)) for logit in output_logits]if EOS_ID in outputs:outputs = outputs[:outputs.index(EOS_ID)]response = "".join([tf.compat.as_str(vocab_de[output]) for output in outputs])print('AI > ' + response)