import os
import codecs
import pandas as pd
import jieba
import jieba.analyse

'''定义变量
文件路径/文件内容/关键字（5个）'''
filepaths = []
contents =[]
tag1 = []
tag2 = []
tag3 = []
tag4 = []
tag5 = []#遍历文件，同时得到关键字
for root, dirs, files in os.walk(r'path'):for name in files:filepath = root + '\\' +name  #根目录加文件名构成文件路径f = codecs.open(filepath,'r','utf-8')  #根据文件路径以只读的形式打开文件content = f.read().strip()  #将文件内容传入content变量f.close() #关闭文件tags = jieba.analyse.extract_tags(content,topK=5) #根据文件内容获取前5个关键字(出现次数最多)filepaths.append(filepath) #得到文件路径的集合contents.append(content) #得到文件内容的集合
        tag1.append(tags[0]) tag2.append(tags[1])tag3.append(tags[2])tag4.append(tags[3])tag5.append(tags[4])tagDF = pd.DataFrame({'文件路径':filepaths,'文件内容':contents,'关键词1':tag1,'关键词2':tag2,'关键词3':tag3,'关键词4':tag4,'关键词5':tag5})

import re
zh = re.compile(u'[\u4e00-\u9fa5]+')
import jieba
segments = []
filepath = []
#导入停用词
stopwords = pd.read_csv(r'path',encoding='utf-8',index_col=False)for index, row in corpos.iterrows(): filePath = row['filePath']  fileContent = row['fileContent'] segs = jieba.cut(fileContent)  for seg in segs:if zh.search(seg):  #只匹配中文分词if (seg not in stopwords.stopword.values) and (len(seg.strip())>1): #取非停用词和长度>1的词segments.append(seg)filepath.append(filePath)segmeng_DF = pd.DataFrame({'segment': segments,'filePath': filepath})

import numpy as np
segcount = segmeng_DF.groupby(by=['filePath','segment'])['segment'].agg({'词频':np.size}).reset_index().sort_values(by=['词频'],ascending=False)
segcount = segcount[segcount.词频 > 1] #只取词频大于1的分词

TF =segcount.pivot_table(index='filePath',columns='segment',values='词频',fill_value=0)
TF.columns #列名是各篇文章的分词集合

def hanlder(x):return (np.log2(len(corpos) / (np.sum(x>0)+1)))IDF = TF.apply(hanlder)  #结果是各分词的权重

TF_IDF = pd.DataFrame(TF*IDF)TF_IDF.columns #列名是各篇文章的分词集合
TF_IDF.index #索引是文件路径

tag1 = []
tag2 = []
tag3 = []
tag4 = []
tag5 = []for filePath in TF_IDF.index:tagis = TF_IDF.loc[filePath].sort_values(ascending=False)[:5].indextag1.append(tagis[0]) tag2.append(tagis[1])tag3.append(tagis[2])tag4.append(tagis[3])tag5.append(tagis[4])