1 import java.io.BufferedReader;
 2 import java.io.BufferedWriter;
 3 import java.io.File;
 4 import java.io.FileInputStream;
 5 import java.io.FileWriter;
 6 import java.io.InputStreamReader;
 7 import java.io.StringReader;
 8
 9 import org.apache.lucene.analysis.Analyzer;
10 import org.apache.lucene.analysis.TokenStream;
11 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
12 import org.wltea.analyzer.lucene.IKAnalyzer;
13
14 /**
15  *
16  * @author zps
17  *
18  */
19 public class Test {
20
21     public static void main(String[] args) throws Exception {
22
23         String encoding = "UTF-8";
24         String text = null;
25         File file = new File("C:\\Users\\dell\\Desktop\\大数据\\热词数据\\热词数据.txt");//将要被分词的文本
26         File file1 =new File("C:\\Users\\dell\\Desktop\\大数据\\热词细胞词库\\热词数据1.txt");//分完词的文本
27         if(!file.exists()){
28             file.createNewFile();
29            }
30         FileWriter fw = new FileWriter(file1.getAbsoluteFile());
31         BufferedWriter bw = new BufferedWriter(fw);
32         if (file.isFile() && file.exists()) { //判断文件是否存在
33             InputStreamReader read = new InputStreamReader(
34             new FileInputStream(file), encoding);//考虑到编码格式
35             BufferedReader bufferedReader = new BufferedReader(read);
36            // String text = null;
37
38
39             while ((text = bufferedReader.readLine()) != null) {
40                 Analyzer analyzer = new IKAnalyzer(true);
41                 StringReader reader = new StringReader(text);
42                 TokenStream ts = analyzer.tokenStream("", reader);
43                 CharTermAttribute term=ts.getAttribute(CharTermAttribute.class);
44                    //System.out.println(text);
45                 while(ts.incrementToken()){
46                        System.out.println(term.toString()+"|");
47                        bw.write(term.toString()+" ");
48                    }
49                 bw.write("\r\n");
50                 analyzer.close();
51                 reader.close();
52                   }
53
54             read.close();
55             bw.close();
56
57         } else {
58             System.out.println("找不到指定的文件");
59         }
60
61
62         //String text = "数据的福利局撒放了肯德基萨芬快乐圣诞节疯狂了";
63         //Analyzer analyzer = new IKAnalyzer(false);
64
65     }
66
67 }

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>  <comment>IK Analyzer 扩展配置</comment><!--用户可以在这里配置自己的扩展字典 <entry key="ext_dict">ext.dic;</entry> --><entry key="ext_dict">MyDic.dic;</entry><!--用户可以在这里配置自己的扩展停止词字典--><entry key="ext_stopwords">stopword.dic;</entry> </properties>