#!/usr/bin/env python
# -*- coding:utf-8 -*-
import jieba
import os
import re
import time
from jpype import *'''
title：利用结巴分词进行文本语料的批量处理1 首先对文本进行遍历查找2 创建原始文本的保存结构3 对原文本进行结巴分词和停用词处理4 对预处理结果进行标准化格式，并保存原文件结构路径
author：白宁超
myblog：http://www.cnblogs.com/baiboy/
time：2017年4月28日10:03:09
''''''
创建文件目录
path:根目录下创建子目录
'''
def mkdir(path):# 判断路径是否存在isExists=os.path.exists(path)# 判断结果if not isExists:os.makedirs(path)print(path+' 创建成功')return Trueelse:passprint('-->请稍后，文本正在预处理中...')'''
结巴分词工具进行中文分词处理：
read_folder_path：待处理的原始语料根路径
write_folder_path 中文分词经数据清洗后的语料
'''
def CHSegment(read_folder_path,write_folder_path):stopwords ={}.fromkeys([line.strip() for line in open('../Database/stopwords/CH_stopWords.txt','r',encoding='utf-8')]) # 停用词表# 获取待处理根目录下的所有类别folder_list = os.listdir(read_folder_path)# 类间循环# print(folder_list)for folder in folder_list:#某类下的路径new_folder_path = os.path.join(read_folder_path, folder)# 创建一致的保存文件路径mkdir(write_folder_path+folder)#某类下的保存路径save_folder_path = os.path.join(write_folder_path, folder)#某类下的全部文件集# 类内循环files = os.listdir(new_folder_path)j = 1for file in files:if j > len(files):break# 读取原始语料raw = open(os.path.join(new_folder_path, file),'r',encoding='utf-8').read()# 只保留汉字# raw1 = re.sub("[A-Za-z0-9\[\`\~\!\@\#\$\^\&\*\(\)\=\|\{\}\'\:\;\'\,\[\]\.\<\>\/\?\~\！\@\#\\\&\*\%]", "", raw)# jieba分词wordslist = jieba.cut(raw, cut_all=False) # 精确模式# 停用词处理cutwordlist=''for word in wordslist:if word not in stopwords and word=="\n":cutwordlist+="\n" # 保持原有文本换行格式elif len(word)>1 :cutwordlist+=word+"/" #去除空格#保存清洗后的数据with open(os.path.join(save_folder_path,file),'w',encoding='utf-8') as f:f.write(cutwordlist)j += 1'''
结巴分词工具进行中文分词处理：
read_folder_path：待处理的原始语料根路径
write_folder_path 中文分词经数据清洗后的语料
'''
def HanLPSeg(read_folder_path,write_folder_path):startJVM(getDefaultJVMPath(), "-Djava.class.path=C:\hanlp\hanlp-1.3.2.jar;C:\hanlp", "-Xms1g", "-Xmx1g") # 启动JVM，Linux需替换分号;为冒号:stopwords ={}.fromkeys([line.strip() for line in open('../Database/stopwords/CH_stopWords.txt','r',encoding='utf-8')]) # 停用词表# 获取待处理根目录下的所有类别folder_list = os.listdir(read_folder_path)# 类间循环# print(folder_list)for folder in folder_list:#某类下的路径new_folder_path = os.path.join(read_folder_path, folder)# 创建一致的保存文件路径mkdir(write_folder_path+folder)#某类下的保存路径save_folder_path = os.path.join(write_folder_path, folder)#某类下的全部文件集# 类内循环files = os.listdir(new_folder_path)j = 1for file in files:if j > len(files):break# 读取原始语料raw = open(os.path.join(new_folder_path, file),'r',encoding='utf-8').read()# HanLP分词HanLP = JClass('com.hankcs.hanlp.HanLP')wordslist = HanLP.segment(raw)#保存清洗后的数据wordslist1=str(wordslist).split(",")# print(wordslist1[1:len(wordslist1)-1])flagresult=""# 去除标签for v in wordslist1[1:len(wordslist1)-1]:if "/" in v:slope=v.index("/")letter=v[1:slope]if len(letter)>0 and '\n\u3000\u3000' in letter:flagresult+="\n"else:flagresult+=letter +"/" #去除空格# print(flagresult)with open(os.path.join(save_folder_path,file),'w',encoding='utf-8') as f:f.write(flagresult.replace(' /',''))j += 1shutdownJVM()if __name__ == '__main__' :print('开始进行文本分词操作：\n')t1 = time.time()dealpath="../Database/SogouC/FileTest/"savepath="../Database/SogouCCut/FileTest/"# 待分词的语料类别集根目录read_folder_path = '../Database/SogouC/FileNews/'write_folder_path = '../Database/SogouCCut/'#jieba中文分词CHSegment(read_folder_path,write_folder_path) #300个txtq其中结巴分词使用3.31秒HanLPSeg(read_folder_path,write_folder_path) #300个txt其中hanlp分词使用1.83秒t2 = time.time()print('完成中文文本切分: '+str(t2-t1)+"秒。")