python WikiExtractor.py -b 500M -o zhwiki zhwiki-20190401-pages-articles-multistream.xml.bz2

import logging,jieba,os,redef get_stopwords():logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',level=logging.INFO)#加载停用词表stopword_set = set()with open("../stop_words/stopwords.txt",'r',encoding="utf-8") as stopwords:for stopword in stopwords:stopword_set.add(stopword.strip("n"))return stopword_set'''
使用正则表达式解析文本
'''
def parse_zhwiki(read_file_path,save_file_path):#过滤掉<doc>regex_str = "[^<doc.*>$]|[^</doc>$]"file = open(read_file_path,"r",encoding="utf-8")#写文件output = open(save_file_path,"w+",encoding="utf-8")content_line = file.readline()#获取停用词表stopwords = get_stopwords()#定义一个字符串变量，表示一篇文章的分词结果article_contents = ""while content_line:match_obj = re.match(regex_str,content_line)content_line = content_line.strip("n")if len(content_line) > 0:if match_obj:#使用jieba进行分词words = jieba.cut(content_line,cut_all=False)for word in words:if word not in stopwords:article_contents += word+" "else:if len(article_contents) > 0:output.write(article_contents+"n")article_contents = ""content_line = file.readline()output.close()

'''
合并分词后的文件
'''
def merge_corpus():output = open("../dataset/zhwiki/BB/wiki_corpus","w",encoding="utf-8")input = "../dataset/zhwiki/BB"for i in range(3):file_path = os.path.join(input,str("wiki_corpus0%s"%str(i)))file = open(file_path,"r",encoding="utf-8")line = file.readline()while line:output.writelines(line)line = file.readline()file.close()output.close()

import logging
from gensim.models import word2vecdef main():logging.basicConfig(format="%(asctime)s:%(levelname)s:%(message)s",level=logging.INFO)sentences = word2vec.LineSentence("../dataset/zhwiki/BB/wiki_corpus")# size：单词向量的维度。model = word2vec.Word2Vec(sentences,size=250)#保存模型model.save("../model/wiki_corpus.bin")# model.save("../model/wiki_corpus.model")# model.wv.save_word2vec_format("./sogou_word2vec/min_count-1/sogou.wor2vec.txt")if __name__ == "__main__":main()

 logging.basicConfig(format="%(asctime)s:%(levelname)s:%(message)s",level=logging.INFO)model = models.Word2Vec.load("../wiki_chinese/model/wiki_corpus.bin")#输入一个词找出相似的前10个词one_corpus = ["人工智能"]# ''' 词库中与one_corpus最相似的10个词'''result = model.most_similar(one_corpus[0],topn=10)print(result)

  # 两个词的相似度# #输入两个词计算相似度two_corpus = ["腾讯","阿里巴巴"]res = model.similarity(two_corpus[0],two_corpus[1])print("similarity:%.4f"%res)

# KeyError: "word '报了' not in vocabulary"
# 错误：“单词'报了'不在词汇中”

[{"intent":"天气", "words":"杭州天气怎么样"},
{"intent": "年龄", "words": "你今年几岁了"}]新数据：北京天气怎么样
和天气句子最相似，最后会分类问为：天气

from jieba_tokenizer.tokenizer import JiebaTokenizer
from gensim import modelslogger = logging.getLogger(__name__)sys.path.append("./")class AvgWord2vec(object):"""用word2vec+平均词向量的方式生成句子向量"""def __init__(self,w2v_model_path="../wiki_chinese/model/wiki_corpus.bin"):# self._load_w2v_model(w2v_model_path)self.w2v_model = models.Word2Vec.load(w2v_model_path)self.tokenizer = JiebaTokenizer()def _load_w2v_model(self, path):"""加载w2v模型"""w2v_path = glob.glob("**{}".format(path), recursive=True)if len(w2v_path) == 0:logger.error("can not find w2v model")else:logger.info("Loading word2vec model file at {}".format(w2v_path[0]))self.w2v_model = models.Word2Vec.load(w2v_path[0], binary=True)returndef transfrom_sentence_to_vec(self, sentence):"""把句子转换成句子向量"""cutted_sentence = self.tokenizer.cut_sentence(sentence)# words_list = cutted_sentence.split(" ")# print(cutted_sentence)vec = self.seg_text_to_vector(cutted_sentence)# sentencereturn vecdef seg_text_to_vector(self, sentence):splited_text = sentence.split(" ")# size：单词向量的维度。与训练时保持一致vector = np.zeros(250)num_of_word = 0for word in splited_text:try:a = self.w2v_model[word]for q in a:if np.isnan(q):continueexcept:# print(j+"is not in vocabulary")continue# 不分词语的权重# vector += a# 词语带权重words_weight_dict = self.word_weight_dict(sentence)weight = words_weight_dict.get(word)if weight is None:weight = 1.0vector += (a * weight)num_of_word += 1if (num_of_word == 0) is True:return np.zeros(250)else:vector = vector / num_of_wordreturn vectordef word_weight_dict(self, sentence):"""根据输入sentence返回一个词权重查询词典"""tf_idf_list = jieba.analyse.extract_tags(sentence, topK=None, withWeight=True)weight_dict = {}for word_weight in tf_idf_list:weight_dict[word_weight[0]] = word_weight[1]return weight_dictif __name__ == '__main__':sentence = "这是一句测试用的句子，通过这个文件，可以把一句话转换成一个句子向量"s2v = AvgWord2vec()vec = s2v.transfrom_sentence_to_vec(sentence)print(vec)exit(0)

from example.avg_w2v import AvgWord2vec
from scipy.spatial.distance import cosine as cos# 夹角越小越好
best_score = 0.3def words_base(words_list):'''原数据的训练保存'''words_infos = []for words_info in words_list:sentence = words_info.get("words")s2v = AvgWord2vec()words_vec = s2v.transfrom_sentence_to_vec(sentence)words_dict = {"intent": words_info.get("intent"), "words_vec": words_vec}words_infos.append(words_dict)return words_infosdef words_score(sentence, words_infos):'''新数据与老数据对比，分类'''s2v = AvgWord2vec()words_vec = s2v.transfrom_sentence_to_vec(sentence)for words_info in words_infos:score = cos(words_vec, words_info.get("words_vec"))print(score)# 夹角越小越相似if score < best_score:return words_info.get("intent")else:return "匹配失败"if __name__ == '__main__':words_list = [{"intent":"天气", "words":"杭州天气怎么样"},{"intent": "年龄", "words": "你今年几岁了"}]words_infos = words_base(words_list)# print(words_infos)sentence = "北京天气怎么样"result = words_score(sentence, words_infos)print(result)