Tensorflow和Gensim里word2vec训练

Tensorflow里word2vec训练

# -*- coding:utf-8 -*-
import tensorflow as tf
import numpy as np
import math
import collections
import pickle as pkl
from pprint import pprint
#from pymongo import MongoClient
import re
import jieba
import os.path as path
import osclass word2vec():def __init__(self,vocab_list=None,embedding_size=200,win_len=3, # 单边窗口长num_sampled=1000,learning_rate=1.0,logdir='/tmp/simple_word2vec',model_path= None):# 获得模型的基本参数self.batch_size     = None # 一批中数据个数, 目前是根据情况来的if model_path!=None:self.load_model(model_path)else:# model parametersassert type(vocab_list)==listself.vocab_list     = vocab_listself.vocab_size     = vocab_list.__len__()self.embedding_size = embedding_sizeself.win_len        = win_lenself.num_sampled    = num_sampledself.learning_rate  = learning_rateself.logdir         = logdirself.word2id = {}   # word => id 的映射for i in range(self.vocab_size):self.word2id[self.vocab_list[i]] = i# train timesself.train_words_num = 0 # 训练的单词对数self.train_sents_num = 0 # 训练的句子数self.train_times_num = 0 # 训练的次数（一次可以有多个句子）# train loss recordsself.train_loss_records = collections.deque(maxlen=10) # 保存最近10次的误差self.train_loss_k10 = 0self.build_graph()self.init_op()if model_path!=None:tf_model_path = os.path.join(model_path,'tf_vars')self.saver.restore(self.sess,tf_model_path)def init_op(self):self.sess = tf.Session(graph=self.graph)self.sess.run(self.init)self.summary_writer = tf.train.SummaryWriter(self.logdir, self.sess.graph)def build_graph(self):self.graph = tf.Graph()with self.graph.as_default():self.train_inputs = tf.placeholder(tf.int32, shape=[self.batch_size])self.train_labels = tf.placeholder(tf.int32, shape=[self.batch_size, 1])self.embedding_dict = tf.Variable(tf.random_uniform([self.vocab_size,self.embedding_size],-1.0,1.0))self.nce_weight = tf.Variable(tf.truncated_normal([self.vocab_size, self.embedding_size],stddev=1.0/math.sqrt(self.embedding_size)))self.nce_biases = tf.Variable(tf.zeros([self.vocab_size]))# 将输入序列向量化embed = tf.nn.embedding_lookup(self.embedding_dict, self.train_inputs) # batch_size# 得到NCE损失self.loss = tf.reduce_mean(tf.nn.nce_loss(weights = self.nce_weight,biases = self.nce_biases,labels = self.train_labels,inputs = embed,num_sampled = self.num_sampled,num_classes = self.vocab_size))# tensorboard 相关tf.scalar_summary('loss',self.loss)  # 让tensorflow记录参数# 根据 nce loss 来更新梯度和embeddingself.train_op = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(self.loss)  # 训练操作# 计算与指定若干单词的相似度self.test_word_id = tf.placeholder(tf.int32,shape=[None])vec_l2_model = tf.sqrt(  # 求各词向量的L2模tf.reduce_sum(tf.square(self.embedding_dict),1,keep_dims=True))avg_l2_model = tf.reduce_mean(vec_l2_model)tf.scalar_summary('avg_vec_model',avg_l2_model)self.normed_embedding = self.embedding_dict / vec_l2_model# self.embedding_dict = norm_vec # 对embedding向量正则化test_embed = tf.nn.embedding_lookup(self.normed_embedding, self.test_word_id)self.similarity = tf.matmul(test_embed, self.normed_embedding, transpose_b=True)# 变量初始化self.init = tf.global_variables_initializer()self.merged_summary_op = tf.merge_all_summaries()self.saver = tf.train.Saver()def train_by_sentence(self, input_sentence=[]):#  input_sentence: [sub_sent1, sub_sent2, ...]# 每个sub_sent是一个单词序列，例如['这次','大选','让']sent_num = input_sentence.__len__()batch_inputs = []batch_labels = []for sent in input_sentence:for i in range(sent.__len__()):start = max(0,i-self.win_len)end = min(sent.__len__(),i+self.win_len+1)for index in range(start,end):if index == i:continueelse:input_id = self.word2id.get(sent[i])label_id = self.word2id.get(sent[index])if not (input_id and label_id):continuebatch_inputs.append(input_id)batch_labels.append(label_id)if len(batch_inputs)==0:returnbatch_inputs = np.array(batch_inputs,dtype=np.int32)batch_labels = np.array(batch_labels,dtype=np.int32)batch_labels = np.reshape(batch_labels,[batch_labels.__len__(),1])feed_dict = {self.train_inputs: batch_inputs,self.train_labels: batch_labels}_, loss_val, summary_str = self.sess.run([self.train_op,self.loss,self.merged_summary_op], feed_dict=feed_dict)# train lossself.train_loss_records.append(loss_val)# self.train_loss_k10 = sum(self.train_loss_records)/self.train_loss_records.__len__()self.train_loss_k10 = np.mean(self.train_loss_records)if self.train_sents_num % 1000 == 0 :self.summary_writer.add_summary(summary_str,self.train_sents_num)print("{a} sentences dealed, loss: {b}".format(a=self.train_sents_num,b=self.train_loss_k10))# train timesself.train_words_num += batch_inputs.__len__()self.train_sents_num += input_sentence.__len__()self.train_times_num += 1def cal_similarity(self,test_word_id_list,top_k=10):sim_matrix = self.sess.run(self.similarity, feed_dict={self.test_word_id:test_word_id_list})sim_mean = np.mean(sim_matrix)sim_var = np.mean(np.square(sim_matrix-sim_mean))test_words = []near_words = []for i in range(test_word_id_list.__len__()):test_words.append(self.vocab_list[test_word_id_list[i]])nearst_id = (-sim_matrix[i,:]).argsort()[1:top_k+1]nearst_word = [self.vocab_list[x] for x in nearst_id]near_words.append(nearst_word)return test_words,near_words,sim_mean,sim_vardef save_model(self, save_path):if os.path.isfile(save_path):raise RuntimeError('the save path should be a dir')if not os.path.exists(save_path):os.mkdir(save_path)# 记录模型各参数model = {}var_names = ['vocab_size',      # int       model parameters'vocab_list',      # list'learning_rate',   # int'word2id',         # dict'embedding_size',  # int'logdir',          # str'win_len',         # int'num_sampled',     # int'train_words_num', # int       train info'train_sents_num', # int'train_times_num', # int'train_loss_records',  # int   train loss'train_loss_k10',  # int]for var in var_names:model[var] = eval('self.'+var)param_path = os.path.join(save_path,'params.pkl')if os.path.exists(param_path):os.remove(param_path)with open(param_path,'wb') as f:pkl.dump(model,f)# 记录tf模型tf_path = os.path.join(save_path,'tf_vars')if os.path.exists(tf_path):os.remove(tf_path)self.saver.save(self.sess,tf_path)def load_model(self, model_path):if not os.path.exists(model_path):raise RuntimeError('file not exists')param_path = os.path.join(model_path,'params.pkl')with open(param_path,'rb') as f:model = pkl.load(f)self.vocab_list = model['vocab_list']self.vocab_size = model['vocab_size']self.logdir = model['logdir']self.word2id = model['word2id']self.embedding_size = model['embedding_size']self.learning_rate = model['learning_rate']self.win_len = model['win_len']self.num_sampled = model['num_sampled']self.train_words_num = model['train_words_num']self.train_sents_num = model['train_sents_num']self.train_times_num = model['train_times_num']self.train_loss_records = model['train_loss_records']self.train_loss_k10 = model['train_loss_k10']if __name__=='__main__':# step 1 读取停用词stop_words = []with open('stop_words.txt',encoding= 'utf-8') as f:line = f.readline()while line:stop_words.append(line[:-1])line = f.readline()stop_words = set(stop_words)print('停用词读取完毕，共{n}个单词'.format(n=len(stop_words)))# step2 读取文本，预处理，分词，得到词典raw_word_list = []sentence_list = []with open('2800.txt',encoding='gbk') as f:line = f.readline()while line:while '\n' in line:line = line.replace('\n','')while ' ' in line:line = line.replace(' ','')if len(line)>0: # 如果句子非空raw_words = list(jieba.cut(line,cut_all=False))dealed_words = []for word in raw_words:if word not in stop_words and word not in ['qingkan520','www','com','http']:raw_word_list.append(word)dealed_words.append(word)sentence_list.append(dealed_words)line = f.readline()word_count = collections.Counter(raw_word_list)print('文本中总共有{n1}个单词,不重复单词数{n2},选取前30000个单词进入词典'.format(n1=len(raw_word_list),n2=len(word_count)))word_count = word_count.most_common(30000)word_list = [x[0] for x in word_count]# 创建模型，训练w2v = word2vec(vocab_list=word_list,    # 词典集embedding_size=200,win_len=2,learning_rate=1,num_sampled=100,         # 负采样个数logdir='/tmp/280')       # tensorboard记录地址num_steps = 10000for i in range(num_steps):#print (i%len(sentence_list))sent = sentence_list[i%len(sentence_list)]w2v.train_by_sentence([sent])w2v.save_model('model')w2v.load_model('model') test_word = ['天地','级别']test_id = [word_list.index(x) for x in test_word]test_words,near_words,sim_mean,sim_var = w2v.cal_similarity(test_id)print (test_words,near_words,sim_mean,sim_var)

语料库

《斗破苍穹》第一章 陨落的天才http://www.qingkan520.com/第一章陨落的天才(本章免费)“斗之力，三段！”望着测验魔石碑上面闪亮得甚至有些刺眼的五个大字，少年面无表情，唇角有着一抹自嘲，紧握的手掌，因为大力，而导致略微尖锐的指甲深深的刺进了掌心之中，带来一阵阵钻心的疼痛…“萧炎，斗之力，三段！级别：低级！”测验魔石碑之旁，一位中年男子，看了一眼碑上所显示出来的信息，语气漠然的将之公布了出来…中年男子话刚刚脱口，便是不出意外的在人头汹涌的广场上带起了一阵嘲讽的『骚』动。“三段？嘿嘿，果然不出我所料，这个“天才”这一年又是在原地踏步！”“哎，这废物真是把家族的脸都给丢光了。”“要不是族长是他的父亲，这种废物，早就被驱赶出家族，任其自生自灭了，哪还有机会待在家族中白吃白喝。”“唉，昔年那名闻乌坦城的天才少年，如今怎么落魄成这般模样了啊？”“谁知道呢，或许做了什么亏心事，惹得神灵降怒了吧…”周围传来的不屑嘲笑以及惋惜轻叹，落在那如木桩待在原地的少年耳中，恍如一根根利刺狠狠的扎在心脏一般，让得少年呼吸微微急促。少年缓缓抬起头来，『露』出一张有些清秀的稚嫩脸庞，漆黑的眸子木然的在周围那些嘲讽的同龄人身上扫过，少年嘴角的自嘲，似乎变得更加苦涩了。“这些人，都如此刻薄势力吗？或许是因为三年前他们曾经在自己面前『露』出过最谦卑的笑容，所以，如今想要讨还回去吧…”苦涩的一笑，萧炎落寞的转身，安静的回到了队伍的最后一排，孤单的身影，与周围的世界，有些格格不入。“下一个，萧媚！”听着测验人的喊声，一名少女快速的人群中跑出，少女刚刚出场，附近的议论声便是小了许多，一双双略微火热的目光，牢牢的锁定着少女的脸颊…少女年龄不过十四左右，虽然并算不上绝『色』，不过那张稚气未脱的小脸，却是蕴含着淡淡的妩媚，清纯与妩媚，矛盾的集合，让得她成功的成为了全场瞩目的焦点…少女快步上前，小手轻车熟路的触『摸』着漆黑的魔石碑，然后缓缓闭上眼睛…在少女闭眼片刻之后，漆黑的魔石碑之上再次亮起了光芒…“斗之气：七段！”“萧媚，斗之气：七段！级别:高级！”“耶！”听着测验员所喊出的成绩，少女脸颊扬起了得意的笑容…“啧啧，七段斗之气，真了不起，按这进度，恐怕顶多只需要三年时间，她就能称为一名真正的斗者了吧…”“不愧是家族中种子级别的人物啊…”听着人群中传来的一阵阵羡慕声，少女脸颊上的笑容更是多了几分，虚荣心，这是很多女孩都无法抗拒的诱『惑』…与平日里的几个姐妹互相笑谈着，萧媚的视线，忽然的透过周围的人群，停在了人群外的那一道孤单身影上…皱眉思虑了瞬间，萧媚还是打消了过去的念头，现在的两人，已经不在同一个阶层之上，以萧炎最近几年的表现，成年后，顶多只能作为家族中的下层人员，而天赋优秀的她，则将会成为家族重点培养的强者，前途可以说是不可限量。“唉…”莫名的轻叹了一口气，萧媚脑中忽然浮现出三年前那意气风发的少年，四岁练气，十岁拥有九段斗之气，十一岁突破十段斗之气，成功凝聚斗之气旋，一跃成为家族百年之内最年轻的斗者！当初的少年，自信而且潜力无可估量，不知让得多少少女对其春心『荡』漾，当然，这也包括以前的萧媚。然而天才的道路，貌似总是曲折的，三年之前，这名声望达到巅峰的天才少年，却是突兀的接受到了有生以来最残酷的打击，不仅辛辛苦苦修炼十数载方才凝聚的斗之气旋，一夜之间，化为乌有，而且体内的斗之气，也是随着时间的流逝，变得诡异的越来越少。斗之气消失的直接结果，便是导致其实力不断的后退。从天才的神坛，一夜跌落到了连普通人都不如的地步，这种打击，让得少年从此失魂落魄，天才之名，也是逐渐的被不屑与嘲讽所替代。站的越高，摔得越狠，这次的跌落，或许就再也没有爬起的机会。“下一个，萧薰儿！”、、、、、、、、、、

Gensim里word2vec训练

维基百科语料

process.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# 修改后的代码如下：
import logging
import os.path
import sys
from gensim.corpora import WikiCorpus
if __name__ == '__main__':program = os.path.basename(sys.argv[0])logger = logging.getLogger(program)logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')logging.root.setLevel(level=logging.INFO)logger.info("running %s" % ' '.join(sys.argv))# check and process input argumentsif len(sys.argv) < 3:print (globals()['__doc__'] % locals())sys.exit(1)inp, outp = sys.argv[1:3]space = b' 'i = 0output = open(outp, 'w',encoding='utf-8')wiki = WikiCorpus(inp, lemmatize=False, dictionary={})for text in wiki.get_texts():s=space.join(text)s=s.decode('utf8') + "\n"output.write(s)i = i + 1if (i % 10000 == 0):logger.info("Saved " + str(i) + " articles")output.close()logger.info("Finished Saved " + str(i) + " articles")
#python process.py zhwiki-latest-pages-articles.xml.bz2 wiki.zh.text

word2vec_model.py

import logging
import os.path
import sys
import multiprocessing
from gensim.corpora import WikiCorpus
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence
if __name__ == '__main__':program = os.path.basename(sys.argv[0])logger = logging.getLogger(program)logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')logging.root.setLevel(level=logging.INFO)logger.info("running %s" % ' '.join(sys.argv))# check and process input argumentsif len(sys.argv) < 4:print (globals()['__doc__'] % locals())sys.exit(1)inp, outp1, outp2 = sys.argv[1:4]model = Word2Vec(LineSentence(inp), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count())model.save(outp1)model.model.wv.save_word2vec_format(outp2, binary=False)
#python word2vec_model.py zh.jian.wiki.seg.txt wiki.zh.text.model wiki.zh.text.vector
#opencc -i wiki_texts.txt -o test.txt -c t2s.json

testModel.py

from gensim.models import Word2Vec
en_wiki_word2vec_model = Word2Vec.load('wiki.zh.text.model')
testwords = ['苹果','数学','学术','白痴','篮球']
for i in range(5):res = en_wiki_word2vec_model.most_similar(testwords[i])print (testwords[i])print (res)

Testjieba.py

import jieba
import jieba.analyse
import jieba.posseg as pseg
import codecs,sys
def cut_words(sentence):#print sentencereturn " ".join(jieba.cut(sentence)).encode('utf-8')
f=codecs.open('wiki.zh.jian.text','r',encoding="utf8")
target = codecs.open("zh.jian.wiki.seg-1.3g.txt", 'w',encoding="utf8")
print ('open files')
line_num=1
line = f.readline()
while line:print('---- processing ', line_num, ' article----------------')line_seg = " ".join(jieba.cut(line))target.writelines(line_seg)line_num = line_num + 1line = f.readline()
f.close()
target.close()
exit()
while line:curr = []for oneline in line:#print(oneline)curr.append(oneline)after_cut = map(cut_words, curr)target.writelines(after_cut)print ('saved',line_num,'articles')exit()line = f.readline1()
f.close()
target.close()# python Testjieba.py

test.py

import codecs,sys
f=codecs.open('zh.jian.wiki.seg-1.3gg.txt','r',encoding="utf8")
line=f.readline()
print(line)

Tensorflow和Gensim里word2vec训练相关推荐

利用gensim里word2vec训练实例——分析三国里人物关系
前言万物皆可Embedding 入坑cs224N后看完第二周和相关论文.觉得word2vec非常有意思,将一段具有上下文关系的短文(实体)词语学习嵌入到语义空间成为一个向量,然后判断两个词语(实体) ...
gensim Word2Vec 训练和使用
训练模型利用gensim.models.Word2Vec(sentences)建立词向量模型该构造函数执行了三个步骤:建立一个空的模型对象,遍历一次语料库建立词典,第二次遍历语料库建立神经网络模型 ...
gensim 中文语料训练 word2vec
gensim 的word2vec api参见: https://radimrehurek.com/gensim/models/word2vec.html 本文说一下中文语料的使用,很简单. 1 wor ...
自然语言处理之使用gensim.Word2Vec训练词向量进行词义消歧
自然语言处理之使用gensim.Word2Vec训练词向量进行词义消歧 NLP中进行词义消歧的一个非常方便且简单的方法就是训练词向量,通过词向量计算余弦值,来推断某个词在句子中的含义.python中的 ...
word2vec训练中文词向量
词向量作为文本的基本结构--词的模型.良好的词向量可以达到语义相近的词在词向量空间里聚集在一起,这对后续的文本分类,文本聚类等等操作提供了便利,这里简单介绍词向量的训练,主要是记录学习模型和词向量的保 ...
Word2vec原理浅析及gensim中word2vec使用
本文转载于以下博客链接:Word2vec原理浅析:https://blog.csdn.net/u010700066/article/details/83070102: gensim中word2vec使 ...
gensim的word2vec如何得出词向量（python）
首先需要具备gensim包,然后需要一个语料库用来训练,这里用到的是skip-gram或CBOW方法,具体细节可以去查查相关资料,这两种方法大致上就是把意思相近的词映射到词空间中相近的位置. 语料库t ...
使用word2vec训练中文词向量
https://www.jianshu.com/p/87798bccee48 一.文本处理流程通常我们文本处理流程如下: 1 对文本数据进行预处理:数据预处理,包括简繁体转换,去除xml符号,将单词 ...
windows下使用word2vec训练维基百科中文语料全攻略！（三
全文共1552个字,7张图,预计阅读时间8分钟. 训练一个聊天机器人的很重要的一步是词向量训练,无论是生成式聊天机器人还是检索式聊天机器人,都需要将文字转化为词向量,时下最火的词向量训练模型是word ...

Tensorflow和Gensim里word2vec训练

Tensorflow里word2vec训练

Gensim里word2vec训练

Tensorflow和Gensim里word2vec训练相关推荐

最新文章

热门文章