代码一：

import os
from six.moves.urllib.request import urlretrieve
import zipfile
import collections# http://mattmahoney.net/dc/textdata.html
dataset_link = 'http://mattmahoney.net/dc/'
zip_file = 'text8.zip'# 查看下载进度
def cbk(a,b,c):'''回调函数@a:已经下载的数据块@b:数据块的大小@c:远程文件的大小'''per = 100.0*a*b/cif per > 100:per = 100print('%.2f%%' % per)def data_download(zip_file):'''下载数据集'''if not os.path.exists(zip_file):# urlretrieve()方法直接将远程数据下载到本地zip_file, _ = urlretrieve(dataset_link + zip_file, zip_file, cbk)print('File downloaded successfully!')return Nonedef extracting(extracted_folder, zip_file):'''解压缩'''if not os.path.isdir(extracted_folder):with zipfile.ZipFile(zip_file) as zf:# 功能：解压zip文档中的所有文件到当前目录。zf.extractall(extracted_folder)def text_processing(ft8_text):# 标点处理ft8_text = ft8_text.lower()ft8_text = ft8_text.replace('.', ' <period> ')ft8_text = ft8_text.replace(',', ' <comma> ')ft8_text = ft8_text.replace('"', ' <quotation> ')ft8_text = ft8_text.replace(';', ' <semicolon> ')ft8_text = ft8_text.replace('!', ' <exclamation> ')ft8_text = ft8_text.replace('?', ' <question> ')ft8_text = ft8_text.replace('(', ' <paren_l> ')ft8_text = ft8_text.replace(')', ' <paren_r> ')ft8_text = ft8_text.replace('--', ' <hyphen> ')ft8_text = ft8_text.replace(':', ' <colon> ')ft8_text_tokens = ft8_text.split()return ft8_text_tokensdef remove_lowerfreword(ft_tokens):'''去除与单词相关的噪音：输入数据集中词频小于7的单词'''word_cnt = collections.Counter(ft_tokens)  # 统计列表元素出现次数,一个无序的容器类型，以字典的键值对形式存储,其中元素作为key，其计数作为valueshortlisted_words = [w for w in ft_tokens if word_cnt[w]>7]print(shortlisted_words[:15])  # 列出数据集中词频最高的15个单词print('Total number of shortlisted_words', len(shortlisted_words))  # 16616688print('Unique number of shortlisted_words', len(set(shortlisted_words)))  #53721return shortlisted_wordsdef dict_creation(shortlisted_words):'''创建词汇表：单词-词频'''counts = collections.Counter(shortlisted_words)vocabulary = sorted(counts, key=counts.get, reverse=True)rev_dictionary = {ii:word for ii,word in enumerate(vocabulary)}  # 整数:单词dictionary = {word:ii for ii, word in rev_dictionary.items()}    # 单词:整数return dictionary, rev_dictionary

部分库解读：
1. six是用来兼容python2和3的库。
six.moves 是用来处理那些在2和3里面函数的位置有变化的，直接用six.moves就可以屏蔽掉这些变化2. zipfile.ZipFile(zip_file) 打开压缩文件zip_fileZipFile.extractall([path[, members[, pwd]]])  解压zip文档中的所有文件到当前目录。参数：path        指定解析文件保存的文件夹member      指定要解压的文件名称或对应的ZipInfo对象pwd         解压密码

代码二：

import collections
import time
import numpy as np
import random
import tensorflow as tf
from text_processing import *
from sklearn.manifold import TSNEdef subsampling(words_cnt):# 采用子采样处理文本中的停止词thresh = 0.00005word_counts = collections.Counter(words_cnt)total_count = len(words_cnt)freqs = {word: count/total_count for word, count in word_counts.items()}p_drop = {word: 1 - np.sqrt(thresh/freqs[word]) for word in word_counts}train_words = [word for word in words_cnt if p_drop[word] < random.random()]return train_wordsdef skipG_target_set_generation(batch_, batch_index, word_window):# 以所需格式创建skip-gram模型的输入：即中心词周围的词random_num = np.random.randint(1, word_window+1)   # 在word_window范围内随机选取周围词的数量words_start = batch_index - random_num if (batch_index-random_num) > 0 else 0words_stop = batch_index + random_numwindow_target = set(batch_[words_start:batch_index] + batch_[batch_index+1:words_stop+1])return list(window_target)def skipG_batch_creation(short_words,batch_length,word_window):# 创建中心词及其周围单词的组合形式batch_cnt = len(short_words)//batch_lengthprint('batch_cnt=',batch_cnt)short_words = short_words[:batch_cnt*batch_length]for word_index in range(0, len(short_words), batch_length):input_words,label_words = [],[]word_batch = short_words[word_index:word_index+batch_length]for index_ in range(len(word_batch)):  # 遍历每个batch中的每个中词batch_input = word_batch[index_]batch_label = skipG_target_set_generation(word_batch, index_, word_window)   # 获取周围单词label_words.extend(batch_label)input_words.extend([batch_input]*len(batch_label))   # skip_gram的输入形式，周围单词都得对应上中心词yield input_words, label_words# extracted_folder = 'dataset'
# full_text = extracting(extracted_folder, zip_file)with open('dataset/text8') as ft_:full_text = ft_.read()ft_tokens = text_processing(full_text)   # 单词列表
shortlisted_words = remove_lowerfreword(ft_tokens)
dictionary, rev_dictionary = dict_creation(shortlisted_words)
words_cnt = [dictionary[word] for word in shortlisted_words]   # 通过词典获取每个单词对应的整数
train_words = subsampling(words_cnt)
print('train_words=',len(train_words))# 1.
tf_graph = tf.Graph()
with tf_graph.as_default():input_ = tf.placeholder(tf.int32, [None], name='input_')label_ = tf.placeholder(tf.int32, [None, None], name='label_')# 2. 得到embedding
with tf_graph.as_default():word_embed = tf.Variable(tf.random_uniform((len(rev_dictionary), 300),-1,1))embedding = tf.nn.embedding_lookup(word_embed, input_)  # 将单词转换为向量# 3.定义优化算法
vocabulary_size = len(rev_dictionary)
with tf_graph.as_default():sf_weights = tf.Variable(tf.truncated_normal((vocabulary_size,300),stddev=0.1))sf_bias = tf.Variable(tf.zeros(vocabulary_size))# 通过负采样计算lossloss_fn = tf.nn.sampled_softmax_loss(weights=sf_weights,biases=sf_bias,labels=label_,inputs=embedding,num_sampled=100,num_classes=vocabulary_size)cost_fn = tf.reduce_mean(loss_fn)optim = tf.train.AdamOptimizer().minimize(cost_fn)# 4. 验证集：在语料库中选择常见和不常见词的组合，并基于词向量之间的余弦相似性返回最接近它们之间的单词
with tf_graph.as_default():validation_cnt = 16validation_dict = 100validation_words = np.array(random.sample(range(validation_dict), validation_cnt//2))  # 从list(range(validation_dict))中随机获取8个元素，作为一个片断返回validation_words = np.append(validation_words, random.sample(range(1000, 1000+validation_dict), validation_cnt//2))validation_data = tf.constant(validation_words, dtype=tf.int32)normalization_embed = word_embed / (tf.sqrt(tf.reduce_sum(tf.square(word_embed),1,keep_dims=True)))validation_embed = tf.nn.embedding_lookup(normalization_embed, validation_data)word_similarity = tf.matmul(validation_embed,tf.transpose(normalization_embed))epochs = 2
batch_length = 1000
word_window = 10# 定义模型存储检查点model_checkpoint
with tf_graph.as_default():saver = tf.train.Saver()with tf.Session(graph=tf_graph) as sess:iteration = 1loss = 0sess.run(tf.global_variables_initializer())print("Begin training-----------")for e in range(1, epochs+1):batches = skipG_batch_creation(train_words, batch_length, word_window)start = time.time()for x, y in batches:train_loss, _ = sess.run([cost_fn, optim],feed_dict={input_:x, label_:np.array(y)[:,None]})loss += train_lossif iteration % 100 ==0:end = time.time()print('Epoch {}/{}'.format(e,epochs),', Iteration：{}'.format(iteration),', Avg.Training loss：{:.4f}'.format(loss/100),', Processing：{:.4f} sec/batch'.format((end-start)/100))loss = 0start = time.time()if iteration % 2000 ==0:similarity_ = word_similarity.eval()  # 返回结果值for i in range(validation_cnt):validated_words = rev_dictionary[validation_words[i]]top_k = 8nearest = (-similarity_[i,:]).argsort()[1:top_k+1]  # argsort将similarity_中的元素从小到大排列，提取其对应的index(索引)log = 'Nearest to %s:' % validated_wordsfor k in range(top_k):close_word = rev_dictionary[nearest[k]]log = '%s %s,' % (log, close_word)print(log)iteration += 1  # 每遍历一个batch，iteration值加1save_path = saver.save(sess, "model_checkpoint/skipGram_text8.ckpt")embed_mat = sess.run(normalization_embed)with tf_graph.as_default():saver = tf.train.Saver()with tf.Session(graph=tf_graph) as sess:saver.restore(sess, tf.train.latest_checkpoint('model_checkpoint'))embed_mat = sess.run(word_embed)# 使用t分布随机邻嵌入（t-SNE）来实现可视化
word_graph = 250
tsne = TSNE()
word_embedding_tsne = tsne.fit_transform(embed_mat[:word_graph,:])

可视化结果：

word2vec原理（五）：skip-gram和CBOW模型代码实现相关推荐

skip gram和cbow的优缺点
在cbow方法中,是用周围词预测中心词,从而利用中心词的预测结果情况,使用GradientDesent方法,不断的去调整周围词的向量.当训练完成之后,每个词都会作为中心词,把周围词的词向量进行了调整, ...
word2vec原理（一）：词向量、CBOW与Skip-Gram模型基础
word2vec原理(一): CBOW与Skip-Gram模型基础 word2vec原理(二):基于Hierarchical Softmax的模型 word2vec原理(三): 基于Negative ...
Word2Vec原理篇 | CBOW与Skip-Gram模型基础
作者:刘建平Pinard 博客地址:https://www.cnblogs.com/pinard 原文链接,点击文末阅读全文直达:https://www.cnblogs.com/pinard/p/71 ...
word2vec原理_深入理解Word2Vec底层原理
本文主要三个部分: (1)DNN如何计算字词的语义特征表示 (2)word2vec如何计算字词的语义特征表示 (3)霍夫曼对于word2vec的意义词嵌入,就是将文本的单词嵌入数学空间中. 不同于o ...
word2vec原理
首先声明,本文源自于刘建平老师的博客,原文写的非常好,在此结合鄙人的项目经验和大家分享. word2vec原理(一) CBOW与Skip-Gram模型基础一:词向量基础 word2vec是谷歌于20 ...
深度学习 | Word2vec原理及应用
聊聊Word2vec 1 前言 2 什么是Word2vec? 2.1 定义 2.1.1 分词的原理介绍 2.1.2 文本向量化的方式 2.2 数学原理 2.2.1 CBOW(Continuous Ba ...
word2vec原理及实现
word2vec的两种计算方式 CBOW模型原理图如下代码见github:word2vec_CBOW.py skip-gram 原理图如下: 代码见github:word2vec_skip-gra ...
《自然语言处理学习之路》02 词向量模型Word2Vec，CBOW，Skip Gram
本文主要是学习参考莫烦老师的教学,对老师课程的学习,记忆笔记. 原文链接文章目录书山有路勤为径,学海无涯苦作舟. 零.吃水不忘挖井人一.计算机如何实现对于词语的理解 1.1 万物数字化 1.2 ...
word2vec原理之CBOW与Skip-Gram模型基础
word2vec是google在2013年推出的一个NLP工具,它的特点是将所有的词向量化,这样词与词之间就可以定量的去度量他们之间的关系,挖掘词之间的联系.虽然源码是开源的,但是谷歌的代码库国内无法 ...

word2vec原理（五）：skip-gram和CBOW模型代码实现

代码一：

代码二：

word2vec原理（五）：skip-gram和CBOW模型代码实现相关推荐

最新文章

热门文章