import jiebafrom jieba import analyseimport numpyimport gensimimport codecsimport pandas as pdimport jieba.posseg as pogfrom gensim.models import Word2Vecfrom gensim.models.word2vec import LineSentence#获取训练语料def data_handle(data):    n = data.shape[0]    data_str = ''    for i in numpy.arange(n):        data_str += str(data.ix[i, 'comment'])    return data_strdef fenci(data_str,stop_property,stopfile):    # 停用词    stop_word = [word.strip() for word in open(stopfile, encoding='utf-8').readlines()]    # 分词    word_cut = pog.cut(data_str)

    with open('weibo.txt','w',encoding='utf-8') as f:        for word, flag in word_cut:            if flag not in stop_property:                if word not in stop_word:                    f.write(word+'\n')

# 原始的训练语料转化成一个sentence的迭代器,每一次迭代返回的sentence是一个word（utf8格式）的列表def vctor_word():    wiki_news = open('weibo.txt', 'r',encoding='utf-8')    sentences=LineSentence(wiki_news)    model=Word2Vec(sentences,sg=0,size=100,window=5,min_count=5,workers=9)    model.save('weibo.word2vec')

# 实现给出任意字符串，获取字符串中某字符的位置以及出现的总次数def get_char_pos(string, char):    chPos = []    try:        chPos = list(((pos, char) for pos, val in enumerate(string) if (val == char)))    except:        pass    return chPos

# 利用训练好的词向量获取关键词的词向量

def cut_data(data,stopfile):    data.fillna(0,inplace=True)    stop_word = [word.strip() for word in open(stopfile, encoding='utf-8').readlines()]    charater=['a', 'nr', 'ns', 'nt', 'ng', 'vn', 'vi', 'l', 'n', 'v']    m=data.shape[0]    with open('seg_word.txt', 'w', encoding='utf-8') as f:

       for i in range(m):            str_cut = ''            str=data.ix[i,'comment']            if str!=0:                segs=jieba.posseg.cut(str)                for word,flag in segs:                    if flag in charater:                        if word not in stop_word:                             str_cut+=word+'/'                f.write(str_cut )            else:               str_cut=''            f.write('\n ')

def get_vector(data,model):#str    wordvec_size = 100    word_vec_all = numpy.zeros(wordvec_size)    space_pos = get_char_pos(data, '/')    first_word = data[0:space_pos[0][0]]    print('first_word', first_word)    if first_word in model:        print('yes')        word_vec_all = word_vec_all + model[first_word]

    for i in range(len(space_pos) - 2):        word = data[space_pos[i][0]:space_pos[i + 1][0]]        print('word',word)        if word in model:            print('yes')            word_vec_all = word_vec_all + model[first_word]

    print('word_vec_all',word_vec_all)    return word_vec_all

def word2vec(file_name, model,str):

   DataFile = codecs.open(file_name, "r", encoding='utf-8')   DataSet = DataFile.readlines()[:-1]

   score_list=[]

   str_vector=get_vector(str,model)   for data in DataSet:  #       if data.strip()!='':           word_vec_all=get_vector(data,model)           score=simlarityCalu(word_vec_all, str_vector)       else:           score=0       score_list.append(score)   print('score_list',score_list)   return score_list

# 词向量相似度计算代码：余弦def simlarityCalu(vector1, vector2):    vector1Mod = numpy.sqrt(vector1.dot(vector1))    vector2Mod = numpy.sqrt(vector2.dot(vector2))    if vector2Mod != 0 and vector1Mod != 0:        simlarity = (vector1.dot(vector2)) / (vector1Mod * vector2Mod)    else:        simlarity = 0    return simlarity

if __name__ == '__main__':

    stop_property = ['b', 'c', 'd', 'e', 'f', 'm', 'o', 'p', 'q', 'r', 't', 'u', 'x', 'y', 'z', 'uj', 'nrt', 'eng',                     'zg', 'ul']    stop_file='stop.txt'

    # 读取数据    data = pd.read_excel('C:/E/weibo.xlsx')    data.rename(columns={'粉丝ID': 'fans_id', '粉丝': 'fans_name', '微博账户id': 'weibo_user_id', '微博名': 'weibo_name',                         '微博id': 'weibo_id', '评论id': 'comment_id', '评论': 'comment'}, inplace=True)

    # 获取评论字符串    comment_str=data_handle(data)

    #获取语料    fenci(comment_str, stop_property, stop_file)    #训练模型    vctor_word()    #获取关键词    cut_data(data, stop_file)

    p1_keywords = 'seg_word.txt'    str1 = '农农/陈利农/宝贝'    # model = gensim.models.Word2Vec.load('weibo.word2vec')    model = gensim.models.Word2Vec.load('zhiwiki_news.word2vec')    p1_vec = word2vec(p1_keywords, model,str1)

    str2='舒蔻 尤妮佳 买'