一、爬取京东评论

京东评论竟然全部对外开放

public class CommentCrawler {final static PoolingHttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager();final static int MAX_PAGE = 50;static HttpClient getClient() {return HttpClients.custom().setConnectionManager(httpClientConnectionManager).build();}static String getUrl(String productId, int page) {return String.format("http://sclub.jd.com/comment/productPageComments.action?productId=%s&score=0&sortType=3&page=%d&pageSize=10",productId, page);}static Comment commentFromJson(JSONObject json, String productId) {return new Comment(json.getLongValue("id"), productId, json.getString("score"), json.getString("content"));}public static boolean crawlComments(String productId) {try {int maxPage = 1;int nowPage = 0;HttpClient client = getClient();while (nowPage < maxPage) {String url = getUrl(productId, nowPage);HttpGet get = new HttpGet(url);HttpResponse resp = client.execute(get);JSONObject json = JSON.parseObject(EntityUtils.toString(resp.getEntity()));JSONArray comments = json.getJSONArray("comments");if (comments.size() == 0)return false;CommentService ser = new CommentService();for (int i = 0; i < comments.size(); i++) {Comment comment = commentFromJson(comments.getJSONObject(i), productId);ser.insertComment(comment);}if (nowPage == 0) {maxPage = json.getInteger("maxPage");ser.insertProduct(new Product(productId, comments.getJSONObject(0).getString("referenceName")));}nowPage++;}ProductJudger.judge(productId);return true;} catch (Exception e) {e.printStackTrace();}return false;}
}

二、结巴分词

jieba分词原本是Python版的，有人把它改成了Java版，名字也改成了jieba-analysis

JiebaSegmenter segmenter = new JiebaSegmenter();List<Word> list = segmenter.sentenceProcess(str);for (Word i : list) {String token = i.getToken();if (stopWords.contains(token)) {continue;}Integer cnt = map.get(token);if (cnt == null) {cnt = 0;}map.put(token, new Integer(cnt + 1));}

三、词云图

用到d3.js，d3-cloud.js这两个库，d3.js是“Data Driven Document”，d3-cloud这个库还是比较难用的，主要是官方实例代码太少了。

这里给出一个例子：每一个词云图都对应一个字典，这个字典就是“词语：频率”这样的键值对。给定多个字典，每一个字典都要渲染成一个词云图。

<div id="word-clouds" style="text-align: center"></div>
<script>var wordClouds = <%=request.getAttribute("wordClouds")%>;$(document).ready(function () {for (var i in wordClouds) {var it = wordClouds[i];var divId = "product_wordcloud" + it['productId'];$("#word-clouds").append("<h3 align='center'>商品" + it['productId'] + "词云图</h3>").append("<div id='" + divId + "'></div>");createWordCloud(transformWordFraquency(it['words']), "#" + divId)}})
</script>

还需要编写如下JS代码

var fill = d3.scale.category20();// 20种颜色
var wordCloudWidth = 800, wordCloudHeight = 400;
var font_name = "楷体", font_weight = "bold", max_font_size = 50;
var word_count = 50;// 显示词汇个数
var word_max_size = 60;// 显示词汇字体最大字号
var word_min_size = 10;// 显示词汇字体最小字号
/** 函数名称：transformWordFrequency* 参数words：字典类型，形如“{word1:cnt1,word2:cnt2,word3:cnt3}” 返回值：{text，size}数组*/function transformWordFraquency(words) {var ar = []for ( var i in words) {ar.push({"text" : i,"size" : words[i]})}// 按照字体的大小从大到小进行排序，只取出现次数较多的前几名ar.sort(function(x, y) {return y['size'] - x['size'];})ar = ar.slice(0, Math.min(word_count, ar.length));for (var i = 0; i < ar.length; i++) {ar[i]['size'] = word_max_size - (word_max_size - word_min_size)/ ar.length * i;}return ar;
}
/** wordMap是[{text:"",size:""}]形式的数组 selector是即将渲染到的目标位置*/
function createWordCloud(wordMap, selector) {d3.layout.cloud().size([ wordCloudWidth * 2 - 100, wordCloudHeight * 2 - 100 ]).words(wordMap).font(font_name).fontWeight(font_weight).fontSize(function(d) {return d.size;}).rotate(function() {return 0;}).on("end", function(words) {renderWordCloud(words, selector)}).start();
}
/** 像这种风格的函数调用，这些函数的调用顺序不能变*/
function renderWordCloud(words, selector) {d3.select(selector).append("svg").attr("width", wordCloudWidth).attr("height", wordCloudHeight).append("g").attr("transform","translate(" + wordCloudWidth / 2 + "," + wordCloudHeight/ 2 + ")").selectAll("text").data(words).enter()// 进入words，相当于for循环.append("text").style("font-family", font_name).style("font-weight", font_weight)// .attr("text-anchor", "middle").style("font-size", function(d) {// 字体大小return d.size + "px";}).style("fill", function(d, i) {// 字体颜色return fill(i);}).attr("transform",function(d) {return "translate(" + [ d.x, d.y ] + ") rotate("+ d.rotate + ")";}).text(function(d) {return d.text;});
}