[NLP]OpenNLP块检测器(Chunker)的使用

Chunker

分块是将文章的内容分成句法相关的词组，如名词组、动词组，但不指定它们的内部结构，也不说明它们在主句中的作用。
训练数据的输入格式如下：
Rockwell NNP B-NP
International NNP I-NP
Corp. NNP I-NP
's POS B-NP
Tulsa NNP I-NP
unit NN I-NP
said VBD B-VP
it PRP B-NP
signed VBD B-VP
a DT B-NP
tentative JJ I-NP
agreement NN I-NP
extending VBG B-VP
its PRP$ B-NP
contract NN I-NP
with IN B-PP
Boeing NNP B-NP
Co. NNP I-NP
to TO B-VP
provide VB I-VP
structural JJ B-NP
parts NNS I-NP
for IN B-PP
Boeing NNP B-NP
's POS B-NP
747 CD I-NP
jetliners NNS I-NP

标注说明：
用chunker分块后的标志由两部分组成：块在原句中的位置-词性
如B-NP的B表示该词在句子开始的位置，NP表示名词；I-NP的I表示该词在句中中间的位置，NP表示名词。

\B 标注开始
\I 标注的中间
\E 标注的结束
NP 名词块
VB 动词块

模型训练

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import opennlp.tools.chunker.ChunkSample;
import opennlp.tools.chunker.ChunkSampleStream;
import opennlp.tools.chunker.ChunkerEvaluator;
import opennlp.tools.chunker.ChunkerFactory;
import opennlp.tools.chunker.ChunkerME;
import opennlp.tools.chunker.ChunkerModel;
import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.MarkableFileInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.FMeasure;public class ChunkerTrain {public static void main(String[] args) throws IOException {// TODO Auto-generated method stubString rootDir = System.getProperty("user.dir") + File.separator;String fileResourcesDir = rootDir + "resources" + File.separator;String modelResourcesDir = rootDir + "opennlpmodel" + File.separator;//训练数据的路径String filePath = fileResourcesDir + "chunker.txt";//训练后模型的保存路径String modelPath = modelResourcesDir + "en-chunker-my.bin";//按行读取数据InputStreamFactory inputStreamFactory = new MarkableFileInputStreamFactory(new File(filePath));ObjectStream<String> lineStream = new PlainTextByLineStream(inputStreamFactory, StandardCharsets.UTF_8);//按行读取数据ObjectStream<ChunkSample> sampleStream = new ChunkSampleStream(lineStream);ChunkerFactory factory =new ChunkerFactory();//训练模型ChunkerModel model =ChunkerME.train("en",sampleStream, TrainingParameters.defaultParams(), factory);//保存模型FileOutputStream fos=new FileOutputStream(new File(modelPath));OutputStream modelOut = new BufferedOutputStream(fos);model.serialize(modelOut);  //模型评估ChunkerEvaluator evaluator=new ChunkerEvaluator(new ChunkerME(model));FMeasure fm=evaluator.getFMeasure();System.out.println("FMeasure:"+fm.getFMeasure()+";precision="+fm.getPrecisionScore()+";recall"+fm.getRecallScore());      }
}

词句分块


```java
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import opennlp.tools.chunker.ChunkerME;
import opennlp.tools.chunker.ChunkerModel;
import opennlp.tools.util.Sequence;public class ChunkerPredit {public static void main(String[] args) throws IOException {// TODO Auto-generated method stubString rootDir = System.getProperty("user.dir") + File.separator;String fileResourcesDir = rootDir + "resources" + File.separator;String modelResourcesDir = rootDir + "opennlpmodel" + File.separator;//String filePath = fileResourcesDir + "sentenceDetector.txt";String modelPath = modelResourcesDir + "en-chunker.bin";InputStream modelIn = new FileInputStream(modelPath) ;//加载模型ChunkerModel model = new ChunkerModel(modelIn);//实例化模型ChunkerME chunker = new ChunkerME(model);//分块检测String sent[] = new String[] { "Rockwell", "International", "Corp.", "'s","Tulsa", "unit", "said", "it", "signed", "a", "tentative", "agreement","extending", "its", "contract", "with", "Boeing", "Co.", "to","provide", "structural", "parts", "for", "Boeing", "'s", "747","jetliners", "." };String pos[] = new String[] { "NNP", "NNP", "NNP", "POS", "NNP", "NN","VBD", "PRP", "VBD", "DT", "JJ", "NN", "VBG", "PRP$", "NN", "IN","NNP", "NNP", "TO", "VB", "JJ", "NNS", "IN", "NNP", "POS", "CD", "NNS","." };String tag[] = chunker.chunk(sent, pos);//获取概率参数double chunkerProbs[] = chunker.probs();for(String str:tag){System.out.print(str+",");} System.out.println();for(double pro:chunkerProbs){System.out.print(pro+",");}Sequence[] sentens = chunker.topKSequences(sent, pos);System.out.println();for(Sequence se:sentens){System.out.print(se+",");}}
}