<ansj.version>5.0.4</ansj.version>
<tree_split.version>1.5</tree_split.version>
<nlp-lang.version>1.7.7</nlp-lang.version><!-- ansj_seg -->
<dependency><groupId>org.ansj</groupId><artifactId>ansj_seg</artifactId><version>${ansj.version}</version>
</dependency>
<!-- tree_split -->
<dependency><groupId>org.ansj</groupId><artifactId>tree_split</artifactId><version>${tree_split.version}</version>
</dependency>
<!-- nlp-lang -->
<dependency><groupId>org.nlpcn</groupId><artifactId>nlp-lang</artifactId><version>${nlp-lang.version}</version>
</dependency>

package com.zhen.segment;import java.io.InputStream;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;import org.ansj.domain.Result;
import org.ansj.domain.Term;
import org.ansj.library.UserDefineLibrary;
import org.ansj.splitWord.analysis.BaseAnalysis;
import org.ansj.splitWord.analysis.NlpAnalysis;
import org.ansj.splitWord.analysis.ToAnalysis;
import org.nlpcn.commons.lang.tire.domain.Forest;
import org.nlpcn.commons.lang.tire.domain.Value;
import org.nlpcn.commons.lang.tire.library.Library;/*** @author FengZhen* @date Jan 30, 2019* ansj分词*/
public class SegmentTest {public static void main(String[] args) {
//        dynamicWord();
//        localDic();
//        moreUserDic();}/*** 多用户词典(新增、删除)*/public static void moreUserDic() {// 多用户词典String str = "神探夏洛克这部电影作者.是一个dota迷";System.out.println(ToAnalysis.parse(str));// 两个词汇 神探夏洛克 douta迷Forest dic1 = new Forest();Library.insertWord(dic1, new Value("神探夏洛克", "define", "1000"));Forest dic2 = new Forest();Library.insertWord(dic2, new Value("dota迷", "define", "1000"));System.out.println(ToAnalysis.parse(str, dic1, dic2));System.out.println("-------删除dic1中的词");Library.removeWord(dic1, "神探夏洛克");System.out.println(ToAnalysis.parse(str, dic1, dic2));}/*** 动态增删词库*/public static void dynamicWord(){// 增加新词,中间按照'\t'隔开UserDefineLibrary.insertWord("ansj中文分词", "userDefine", 1000);Result result =  ToAnalysis.parse("我觉得Ansj中文分词是一个不错的系统!我是王婆!");System.out.println("增加新词例子:" + result);// 删除词语,只能删除.用户自定义的词典.UserDefineLibrary.removeWord("ansj中文分词");result = ToAnalysis.parse("我觉得ansj中文分词是一个不错的系统!我是王婆!");System.out.println("删除用户自定义词典例子:" + result);//将用户自定义词典清空UserDefineLibrary.clear();}/*** 加载词典文件*/public static void localDic(){try {//读的是根目录下的Forest rootForest = Library.makeForest("library/userLibrary.dic");System.out.println(rootForest.toMap());//加载字典文件,取的是resource下的InputStream inputStream = SegmentTest.class.getResourceAsStream("/library/userLibrary.dic");Forest resoutceForest=Library.makeForest(inputStream);String str = "我觉得ansj中文分词是一个不错的系统!我是王婆!";Result result=ToAnalysis.parse(str, resoutceForest);//传入forestList<Term> termList=result.getTerms();for(Term term:termList){System.out.println(term.getName()+":"+term.getNatureStr());}} catch (Exception e) {e.printStackTrace();}}/*** 基本分词* 基本就是保证了最基本的分词.词语颗粒度最非常小的..所涉及到的词大约是10万左右.* 基本分词速度非常快.在macAir上.能到每秒300w字每秒.同时准确率也很高.但是对于新词他的功能十分有限* @param content*/public static void baseAnay(String content) {Result result = BaseAnalysis.parse(delHTMLTag(content).replace("\n","").replace(" ", "").replace("\t",""));System.out.println("result:" + result);}/*** 精准分词* 它在易用性,稳定性.准确性.以及分词效率上.都取得了一个不错的平衡.* @param content*/public static void toAnay(String content){Result result = ToAnalysis.parse(content);System.out.println("result:" + result);}/*** nlp分词(单条新闻处理7秒)* 可以识别出未登录词.但是它也有它的缺点.速度比较慢.稳定性差.ps:我这里说的慢仅仅是和自己的其他方式比较.应该是40w字每秒的速度吧.* 个人觉得nlp的适用方式.1.语法实体名抽取.未登录词整理.只要是对文本进行发现分析等工作* 会把企业分开* @param content*/public static void nlpAnay(String content){Result result = NlpAnalysis.parse(delHTMLTag(content).replace("\n","").replace(" ", "").replace("\t",""));System.out.println("result:" + result);List<Term> terms = result.getTerms();for (Term term : terms) {String name = term.getName();String nature = term.getNatureStr();if (nature.equals("nt") || nature.equals("nr")) {System.out.println("------------------");System.out.println("getName:" + term.getName());System.out.println("getNatureStr:" + term.getNatureStr());}}}/*** 筛除HTML标签* @param htmlStr* @return*/public static String delHTMLTag(String htmlStr){String regEx_script="<script[^>]*?>[\\s\\S]*?<\\/script>"; //定义script的正则表达式String regEx_style="<style[^>]*?>[\\s\\S]*?<\\/style>"; //定义style的正则表达式String regEx_html="<[^>]+>"; //定义HTML标签的正则表达式Pattern p_script=Pattern.compile(regEx_script,Pattern.CASE_INSENSITIVE);Matcher m_script=p_script.matcher(htmlStr);htmlStr=m_script.replaceAll(""); //过滤script标签Pattern p_style=Pattern.compile(regEx_style,Pattern.CASE_INSENSITIVE);Matcher m_style=p_style.matcher(htmlStr);htmlStr=m_style.replaceAll(""); //过滤style标签Pattern p_html=Pattern.compile(regEx_html,Pattern.CASE_INSENSITIVE);Matcher m_html=p_html.matcher(htmlStr);htmlStr=m_html.replaceAll(""); //过滤html标签return htmlStr.trim(); //返回文本字符串}}

<dependency><groupId>com.hankcs</groupId><artifactId>hanlp</artifactId><version>portable-1.7.1</version>
</dependency>

/*** 新增词*/public static void addWord(String word, NatureEnum nature) {logger.info("==== addWord@word:{},nature:{},weight:{} ====", word, nature.getNature(), nature.getWeight());if (!StringUtils.isBlank(word) && !word.equals("null")) {//大小括号问题if (word.contains("(") || word.contains(")")) {CustomDictionary.insert(word.replace("(", "(").replace(")", ")"), nature.getNature());}else if (word.contains("(") || word.contains(")")) {CustomDictionary.insert(word.replace("(", "(").replace(")", ")"), nature.getNature());}CustomDictionary.insert(word, nature.getNature());}else {logger.warn("==== addWord@ word({})为空 ====", word);}}

/*** 删除词* @param forest* @param word*/public static void deleteWord(String word) {logger.info("==== deleteWord@word:{} ====", word);if (!StringUtils.isBlank(word)) {CustomDictionary.remove(word);}else {logger.warn("==== deleteWord@word为空({}) ====", word);}}

/*** 分词* @param content* @param forests* ToAnalysis精准分词* BaseAnalysis基础分词*/public static SegmentResult segment(String content) {logger.info("==== segment@content:{} ====", content);SegmentResult segmentResult = new SegmentResult();List<Term> terms = HanLP.segment(content);Set<String> companySet = new HashSet<String>();for (Term term : terms) {String name = term.word;String nature = term.nature.toString();if (nature.equals(NatureEnum.Company.getNature())) {companySet.add(name);}}segmentResult.setCompanys(new ArrayList<String>(companySet));logger.info("==== segment@分词结果:{},提取结果:{} ====", terms.toString(), segmentResult.toString());return segmentResult;}

#本配置文件中的路径的根目录，根目录+其他路径=完整路径(支持相对路径，请参考：https://github.com/hankcs/HanLP/pull/254)
#Windows用户请注意，路径分隔符统一使用/
root=/Users/FengZhen/Desktop/accumulate/分词/HanLP/#好了，以上为唯一需要修改的部分，以下配置项按需反注释编辑。#核心词典路径
#CoreDictionaryPath=data/dictionary/CoreNatureDictionary.txt
#2元语法词典路径
#BiGramDictionaryPath=data/dictionary/CoreNatureDictionary.ngram.txt
#自定义词典路径，用;隔开多个自定义词典，空格开头表示在同一个目录，使用“文件名 词性”形式则表示这个词典的词性默认是该词性。优先级递减。
#所有词典统一使用UTF-8编码，每一行代表一个单词，格式遵从[单词] [词性A] [A的频次] [词性B] [B的频次] ... 如果不填词性则表示采用词典的默认词性。
CustomDictionaryPath=data/dictionary/custom/CustomDictionary.txt; Company.txt company; 现代汉语补充词库.txt; 全国地名大全.txt ns; 人名词典.txt; 机构名词典.txt; 上海地名.txt ns;data/dictionary/person/nrf.txt nrf;
#停用词词典路径
#CoreStopWordDictionaryPath=data/dictionary/stopwords.txt
#同义词词典路径
#CoreSynonymDictionaryDictionaryPath=data/dictionary/synonym/CoreSynonym.txt
#人名词典路径
#PersonDictionaryPath=data/dictionary/person/nr.txt
#人名词典转移矩阵路径
#PersonDictionaryTrPath=data/dictionary/person/nr.tr.txt
#繁简词典根目录
#tcDictionaryRoot=data/dictionary/tc
#HMM分词模型
#HMMSegmentModelPath=data/model/segment/HMMSegmentModel.bin
#分词结果是否展示词性
#ShowTermNature=true
#IO适配器，实现com.hankcs.hanlp.corpus.io.IIOAdapter接口以在不同的平台(Hadoop、Redis等)上运行HanLP
#默认的IO适配器如下，该适配器是基于普通文件系统的。
#IOAdapter=com.hankcs.hanlp.corpus.io.FileIOAdapter
#感知机词法分析器
#PerceptronCWSModelPath=data/model/perceptron/pku199801/cws.bin
#PerceptronPOSModelPath=data/model/perceptron/pku199801/pos.bin
#PerceptronNERModelPath=data/model/perceptron/pku199801/ner.bin
#CRF词法分析器
#CRFCWSModelPath=data/model/crf/pku199801/cws.txt
#CRFPOSModelPath=data/model/crf/pku199801/pos.txt
#CRFNERModelPath=data/model/crf/pku199801/ner.txt
#更多配置项请参考 https://github.com/hankcs/HanLP/blob/master/src/main/java/com/hankcs/hanlp/HanLP.java#L59 自行添加

# encoding=utf-8
import jiebaseg_list = jieba.cut("我来到北京清华大学", cut_all=True)
print("Full Mode: " + "/ ".join(seg_list)) # 全模式seg_list = jieba.cut("我来到北京清华大学", cut_all=False)
print("Default Mode: " + "/ ".join(seg_list)) # 精确模式seg_list = jieba.cut("他来到了网易杭研大厦") # 默认是精确模式
print(", ".join(seg_list))seg_list = jieba.cut_for_search("小明硕士毕业于中国科学院计算所，后在日本京都大学深造") # 搜索引擎模式
print(", ".join(seg_list))

# encoding=utf-8
import jieba
import jieba.posseg as pseg
import timeprint("start-time" + time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
file_name = "/home/jenkins/fz/segment/jieba/data/Company.txt"
jieba.load_userdict(file_name) # file_name 为文件类对象或自定义词典的路径
print("end-time" + time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))seg_list = jieba.cut("北京尚伯乐文化发展有限公司")
print("custom dic: " + "/ ".join(seg_list))seg_list = pseg.cut(content) #带词性
for seg in seg_list:
print(seg)

郑汴 1 company
灯塔市栈道村煤矿 1 company
深圳市莱斯达电信有限公司铁岭分公司 1 company
T&T ENERTECHNO 1 company
MMI Holdings Ltd 1 company
：三十门市部 1 company
关岭天利美容化妆品店 1 company
测试一下啦 1 company