import logging
import sys
import osfrom gensim.corpora import WikiCorpusprogram = os.path.basename( sys.argv[0] )
logger = logging.getLogger(program)logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')
logging.root.setLevel(level=logging.INFO)
logger.info( "running %s" % ' '.join(sys.argv) )inp = 'zhwiki-latest-pages-articles.xml.bz2'
outp = 'wiki.zh.txt'
space = ' '
i = 0
output = open(outp, 'w', encoding='utf-8')
#gensim里的维基百科处理类WikiCorpu
wiki = WikiCorpus(inp, lemmatize=False, dictionary=[] )#通过get_texts将维基里的每篇文章转换为1行text文本，并且去掉了标点符号等内容
for text in wiki.get_texts():output.write( space.join(text) + '\n' )i += 1if ( i % 10000 == 0):logger.info('Saved ' + str(i) + ' articles.')output.close()
logger.info('Finished Saved ' + str(i) + ' articles.')

opencc -i wiki.zh.txt -o wiki.zh.simp.txt -c t2s.json
#注意可以添加上opencc的绝对路径，t2s.json也要加绝对路径

import codecs,sys
i = 0
f = codecs.open('wiki.zh.txt','r',encoding="utf-8")
for eachline in f:i += 1if (i < 3):print(eachline)else:break

import codecs,sys
i = 0
f = codecs.open('wiki.zh.simp.txt','r',encoding="utf-8")
for eachline in f:i += 1if (i < 3):print(eachline)else:break

import jieba
import jieba.analyse
import codecsdef prepareData(sourceFile, targetFile):f =codecs.open(sourceFile, 'r', encoding='utf-8')target = codecs.open(targetFile, 'w', encoding='utf-8')print( 'open source file: '+ sourceFile )print( 'open target file: '+ targetFile )lineNum = 0for eachline in f:lineNum += 1if lineNum % 1000 == 0:print('---processing ', sourceFile, lineNum,' article---')seg_list = jieba.cut(eachline, cut_all=False)line_seg = ' '.join(seg_list)target.write(line_seg+ '\n')print('---Well Done!!!---' * 4)f.close()target.close()sourceFile = 'wiki.zh.simp.txt'
targetFile = 'wiki.zh.simp.seg.txt'
prepareData(sourceFile, targetFile)

import codecs,sys
i = 0
f = codecs.open('wiki.zh.simp.seg.txt','r',encoding="utf-8")
for eachline in f:i += 1if (i < 3):print(eachline)else:break

import os
import sys
import logging
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence
import multiprocessingprogram = os.path.basename(sys.argv[0])
logger = logging.getLogger(program)
logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s',level=logging.INFO)
logger.info("running %s" % ' '.join(sys.argv))# inp为输入语料, outp1 为输出模型, outp2为原始c版本word2vec的vector格式的模型
inp = 'wiki.zh.simp.seg.txt'
outp1 = 'wiki.zh.text.model'
outp2 = 'wiki.zh.text.vector'#训练skip-gram 模型
model = Word2Vec( LineSentence(inp), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count() )
model.save(outp1)
model.wv.save_word2vec_format(outp2, binary=False)

import warnings
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')# 忽略警告
import sysimport gensim
model = gensim.models.Word2Vec.load('wiki.zh.text.model')
#与足球最相似的
word = model.most_similar("足球")
for each in word:print(each[0], each[1])print('*' * 20)word = model.most_similar(positive=['皇上','国王'],negative=['皇后'])
for t in word:print (t[0],t[1])print(model.doesnt_match('太后 妃子 贵人 贵妃 才人'.split(' ')))
print('*' * 20)print(model.similarity('书籍','书本'))
print('*' * 20)
print(model.similarity('逛街','书本'))