java爬取网易云热评并制作词云。

IT小白终于脱离hello world,实现用java爬取网易云热评并制作词云，激动激动激动。

一、自我介绍
二、上代码
- 1、封装各个model类
- 2、实现各项功能的service类
- 3、设计一个制作词云的工具类util
- 4、最后测试类
三、总结

假期被游戏缠绕无法自拔？或是被淡黄的长裙蓬松的头发洗脑？是否因为假期太长而闲着不知道干嘛呢，还不来敲代码！用java一起来爬个虫？

一、自我介绍

南京某高校大学生，今年寒假因为疫情，假期无限延长aaa，自己在家也不知道学点什么，有一段时间很迷茫，后来因为机缘巧合，碰到了一些it大佬，于是乎，我就代码敲敲敲，啊，这不，已经可以做个小爬虫啦哈哈哈

二、上代码

这个小项目是用java爬取网易云热门歌曲并制作热评的热词云图，效果如下：
先列一下pom里加的依赖吧

   <!-- https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp --><dependencies><dependency><groupId>com.squareup.okhttp3</groupId><artifactId>okhttp</artifactId><version>4.1.0</version></dependency><!-- JSON 操作库 --><dependency><groupId>com.alibaba</groupId><artifactId>fastjson</artifactId><version>1.2.62</version></dependency><!-- 下面tokenizers是为了中文分词引入 --><dependency><groupId>com.kennycason</groupId><artifactId>kumo-tokenizers</artifactId><version>1.17</version></dependency>

这个是项目结构

1、封装各个model类

专辑类

package com.youkeda.music.model;/**
*专辑类*/
public class Album {private String id;private String name;private String picUrl;public String getId() {return id;}public void setId(String id) {this.id = id;}public String getName() {return name;}public void setName(String name) {this.name = name;}public String getPicUrl() {return picUrl;}public void setPicUrl(String picUrl) {this.picUrl = picUrl;}
}

歌曲对象

package com.youkeda.music.model;import java.util.List;/*** 歌单对象*/
public class Artist {private String id;private List<String> alias;private String picUrl;private String briefDesc;private String img1v1Url;private String name;// 包含一组歌曲private List<Song> songList;public String getId() {return id;}public void setId(String id) {this.id = id;}public List<String> getAlias() {return alias;}public void setAlias(List<String> alias) {this.alias = alias;}public String getPicUrl() {return picUrl;}public void setPicUrl(String picUrl) {this.picUrl = picUrl;}public String getBriefDesc() {return briefDesc;}public void setBriefDesc(String briefDesc) {this.briefDesc = briefDesc;}public String getImg1v1Url() {return img1v1Url;}public void setImg1v1Url(String img1v1Url) {this.img1v1Url = img1v1Url;}public String getName() {return name;}public void setName(String name) {this.name = name;}public List<Song> getSongList() {return songList;}public void setSongList(List<Song> songList) {this.songList = songList;}
}

评论类

package com.youkeda.music.model;
//评论类
public class Comment {private String id;private String content;private String likedCount;private String time;private User commentUser;public String getId() {return id;}public void setId(String id) {this.id = id;}public String getContent() {return content;}public void setContent(String content) {this.content = content;}public String getLikedCount() {return likedCount;}public void setLikedCount(String likedCount) {this.likedCount = likedCount;}public String getTime() {return time;}public void setTime(String time) {this.time = time;}public User getCommentUser() {return commentUser;}public void setCommentUser(User commentUser) {this.commentUser = commentUser;}
}

歌曲类

package com.youkeda.music.model;
//歌曲类
import java.util.List;public class Song {private String id;private String name;private List<User> singers;private String sourceUrl;private Album album;private List<Comment> hotComments;private List<Comment> comments;public List<User> getSingers() {return singers;}public void setSingers(List<User> singers) {this.singers = singers;}public String getSourceUrl() {return sourceUrl;}public void setSourceUrl(String sourceUrl) {this.sourceUrl = sourceUrl;}public Album getAlbum() {return album;}public void setAlbum(Album album) {this.album = album;}public List<Comment> getHotComments() {return hotComments;}public void setHotComments(List<Comment> hotComments) {this.hotComments = hotComments;}public List<Comment> getComments() {return comments;}public void setComments(List<Comment> comments) {this.comments = comments;}public String getId() {return id;}public void setId(String id) {this.id = id;}public String getName() {return name;}public void setName(String name) {this.name = name;}
}

评论者

package com.youkeda.music.model;/*
评论者*/
public class User {private String id;private String nickName;private String avatar;//头像public String getId() {return id;}public void setId(String id) {this.id = id;}public String getNickName() {return nickName;}public void setNickName(String nickName) {this.nickName = nickName;}public String getAvatar() {return avatar;}public void setAvatar(String avatar) {this.avatar = avatar;}
}

2、实现各项功能的service类

service接口类

package com.youkeda.music.service;import com.youkeda.music.model.Artist;
import com.youkeda.music.model.Song;
//定义service接口public interface SongCrawlerService {void start(String artistId);Artist getArtist(String artistId);Song getSong(String artistId, String songId);}

接口实现类

package com.youkeda.music.service.impl;import com.alibaba.fastjson.JSON;
import com.youkeda.music.model.*;
import com.youkeda.music.service.SongCrawlerService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;import com.youkeda.music.util.WordCloudUtil;
import okhttp3.Call;
import okhttp3.OkHttpClient;
import okhttp3.Request;/*** 音乐抓取服务的实现*/
public class SongCrawlerServiceImpl implements SongCrawlerService {private static final String ARTIEST_API_PREFIX = "http://neteaseapi.youkeda.com:3000/artists?id=";private static final String S_D_API_PREFIX = "http://neteaseapi.youkeda.com:3000/song/detail?ids=";private static final String S_C_API_PREFIX = "http://neteaseapi.youkeda.com:3000/comment/music?id=";private static final String S_F_API_PREFIX = "http://neteaseapi.youkeda.com:3000/song/url?id=";// okHttpClient 实例private OkHttpClient okHttpClient;// 歌单数据仓库private Map<String, Artist> artists;private void init() {//1. 构建 okHttpClient 实例okHttpClient = new OkHttpClient();artists = new HashMap<>();}@Overridepublic void start(String artistId) {// 参数判断，未输入参数则直接返回if (artistId == null || artistId.equals("")) {return;}// 执行初始化init();//各个方法都重构封装，减少耦合性initArtistHotSongs(artistId);assembleSongDetail(artistId);assembleSongComment(artistId);assembleSongUrl(artistId);generateWordCloud(artistId);}@Overridepublic Artist getArtist(String artistId) {return artists.get(artistId);}@Overridepublic Song getSong(String artistId, String songId) {Artist artist = artists.get(artistId);List<Song> songs = artist.getSongList();if (songs == null) {return null;}for (Song song : songs) {if (song.getId().equals(songId)) {return song;}}return null;}@SuppressWarnings("unchecked")private Map getSourceDataObj(String prefix, String postfix) {// 构建歌单urlString aUrl = prefix + postfix;// 调用 okhttp3 获取返回数据String content = getPageContentSync(aUrl);// 反序列化成 Map 对象Map returnData = JSON.parseObject(content, Map.class);return returnData;}@SuppressWarnings("unchecked")private Artist buildArtist(Map returnData) {// 从 Map 对象中取得 歌单 数据。歌单也是一个子 Map 对象。Map artistData = (Map) returnData.get("artist");Artist artist = new Artist();artist.setId(artistData.get("id").toString());if (artistData.get("picUrl") != null) {artist.setPicUrl(artistData.get("picUrl").toString());}artist.setBriefDesc(artistData.get("briefDesc").toString());artist.setImg1v1Url(artistData.get("img1v1Url").toString());artist.setName(artistData.get("name").toString());artist.setAlias((List) artistData.get("alias"));return artist;}private List<Song> buildSongs(Map returnData) {// 从 Map 对象中取得一组 歌曲 数据List songsData = (List) returnData.get("hotSongs");List<Song> songs = new ArrayList<>();for (int i = 0; i < songsData.size(); i++) {Map songData = (Map) songsData.get(i);Song songObj = new Song();songObj.setId(songData.get("id").toString());songObj.setName(songData.get("name").toString());songs.add(songObj);}return songs;}/*** 根据输入的url，读取页面内容并返回*/private String getPageContentSync(String url) {//2.定义一个requestRequest request = new Request.Builder().url(url).build();//3.使用client去请求Call call = okHttpClient.newCall(request);String result = null;try {//4.获得返回结果result = call.execute().body().string();System.out.println("call " + url + " , content's size=" + result.length());} catch (IOException e) {System.out.println("request " + url + " error . ");e.printStackTrace();}return result;}/*** 初始化歌单及热门歌曲** @param artistId*/private void initArtistHotSongs(String artistId) {// 取得整体数据对象。Map returnData = getSourceDataObj(ARTIEST_API_PREFIX, artistId);// 构建填充了属性的 Artist 实例Artist artist = buildArtist(returnData);// 构建一组填充了属性的 Song 实例List<Song> songs = buildSongs(returnData);// 歌曲填入歌单artist.setSongList(songs);// 存入本地artists.put(artist.getId(), artist);}/*** 装配 歌曲详情** @param artistId*/@SuppressWarnings("unchecked")private void assembleSongDetail(String artistId) {Artist artist = getArtist(artistId);// 取不到歌单说明参数输入错误if (artist == null) {return;}List<Song> songs = artist.getSongList();// 一个歌单中所有歌曲的id，组装成用逗号分割的字符串，形如：347230,347231。记住这个用法，很方便String sIdsParam = buildManyIdParam(songs);// 抓取结果Map songsDetailObj = getSourceDataObj(S_D_API_PREFIX, sIdsParam);// 原始数据中的 songs 是歌曲列表List<Map> sourceSongs = (List<Map>) songsDetailObj.get("songs");// 临时的 MapMap<String, Map> sourceSongsMap = new HashMap<>();// 遍历歌曲列表for (Map songSourceData : sourceSongs) {String sId = songSourceData.get("id").toString();// 原始歌曲数据对象放入一个临时的 Map 中sourceSongsMap.put(sId, songSourceData);}// 再次遍历歌单中的歌曲，填入详情数据for (Song song : songs) {String sId = song.getId();// 从临时的Map中取得对应的歌曲源数据，使用id直接获取，比较方便Map songSourceData = sourceSongsMap.get(sId);// 源歌曲数据中，ar 字段是歌手列表List<Map> singersData = (List<Map>) songSourceData.get("ar");// 歌手集合List<User> singers = new ArrayList<>();for (Map singerData : singersData) {// 歌手对象User singer = new User();singer.setId(singerData.get("id").toString());singer.setNickName(singerData.get("name").toString());// 歌手集合放入歌手对象singers.add(singer);}// 歌手集合放入歌曲song.setSingers(singers);// 专辑Map albumData = (Map) songSourceData.get("al");Album album = new Album();album.setId(albumData.get("id").toString());album.setName(albumData.get("name").toString());if (albumData.get("picUrl") != null) {album.setPicUrl(albumData.get("picUrl").toString());}// 专辑对象放入歌曲song.setAlbum(album);}}/*** 装配 歌曲评论** @param artistId*/private void assembleSongComment(String artistId) {Artist artist = getArtist(artistId);List<Song> songs = artist.getSongList();for (Song song : songs) {String sIdsParam = song.getId() + "&limit=5";// 抓取结果Map songsCommontObj = getSourceDataObj(S_C_API_PREFIX, sIdsParam);List<Map> hotCommentsData = (List<Map>) songsCommontObj.get("hotComments");List<Map> commentsData = (List<Map>) songsCommontObj.get("comments");List<Comment> hotComments = getComments(hotCommentsData);List<Comment> commments = getComments(commentsData);song.setComments(commments);song.setHotComments(hotComments);}}/*** 装配 歌曲链接** @param artistId*/
//    private void assembleSongUrl(String artistId) {//        Artist artist = getArtist(artistId);
//        // 删除其它语句，保留必要的语句
//        List<Song> songs = artist.getSongList();
//        String sIdsParam = buildManyIdParam(songs);
//        // 抓取结果
//        Map songsFileObj = getSourceDataObj(S_F_API_PREFIX, sIdsParam);
//        List data = (List) songsFileObj.get("data");
//        Map map = (Map) data.get(0);
//        for (Song song:songs){//            song.setSourceUrl((String) map.get("url"));
//        }
//
//    }@SuppressWarnings("unchecked")private void assembleSongUrl(String artistId) {Artist artist = getArtist(artistId);// 取不到歌单说明参数输入错误if (artist == null) {return;}// 删除其它语句，保留必要的语句List<Song> songs = artist.getSongList();String sIdsParam = buildManyIdParam(songs);// 抓取结果Map songsFileObj = getSourceDataObj(S_F_API_PREFIX, sIdsParam);// 原始数据中的 data 是音乐文件列表List<Map> datas = (List<Map>) songsFileObj.get("data");// 临时的 MapMap<String, Map> sourceSongsMap = new HashMap<>();// 遍历音乐文件列表for (Map songFileData : datas) {String sId = songFileData.get("id").toString();// 原始音乐文件数据对象放入一个临时的 Map 中sourceSongsMap.put(sId, songFileData);}// 再次遍历歌单中的歌曲，填入音乐文件URLfor (Song song : songs) {String sId = song.getId();// 从临时的Map中取得对应的音乐文件源数据，使用id直接获取，比较方便Map songFileData = sourceSongsMap.get(sId);// 源音乐文件数据中，url 字段就是文件地址if (songFileData != null && songFileData.get("url") != null) {String songFileUrl = songFileData.get("url").toString();song.setSourceUrl(songFileUrl);}}}private String buildManyIdParam(List<Song> songs) {// 收集一个歌单中所有歌曲的id，放入一个listList<String> songIds = new ArrayList<>();for (Song song : songs) {songIds.add(song.getId());}// 一个歌单中所有歌曲的id，组装成用逗号分割的字符串，形如：347230,347231。记住这个用法，很方便String sIdsParam = String.join(",", songIds);return sIdsParam;}private List<Comment> getComments(List<Map> rawData) {List<Comment> comments = new ArrayList<>();for (Map map : rawData) {Comment comment = new Comment();User u = new User();Map user = (Map) map.get("user");u.setId( user.get("userId").toString());u.setNickName(user.get("nickname").toString());u.setAvatar( user.get("avatarUrl").toString());comment.setCommentUser(u);comment.setContent((String) map.get("content"));comment.setId(map.get("commentId").toString());comment.setLikedCount( map.get("likedCount").toString());comment.setTime(map.get("time").toString());comments.add(comment);}return comments;}private void generateWordCloud(String artistId) {Artist artist = getArtist(artistId);List<Song> songs = artist.getSongList();List<String> contents = new ArrayList<>();for (Song song : songs) {// 遍历歌曲所有的评论，包括普通评论和热门评论，把评论内容字符串存入 contents 集合contents.add(getCommentContents(song.getHotComments()));contents.add(getCommentContents(song.getComments()));}// 调用方法，制作词云WordCloudUtil.generate(artistId, contents);}private String getCommentContents(List<Comment> comments){String contents = "";for (Comment comment:comments){contents += comment.getContent();}System.out.println(contents);return contents;}}

3、设计一个制作词云的工具类util

package com.youkeda.music.util;import com.kennycason.kumo.CollisionMode;
import com.kennycason.kumo.WordCloud;
import com.kennycason.kumo.WordFrequency;
import com.kennycason.kumo.bg.CircleBackground;
import com.kennycason.kumo.font.FontWeight;
import com.kennycason.kumo.font.KumoFont;
import com.kennycason.kumo.font.scale.SqrtFontScalar;
import com.kennycason.kumo.nlp.FrequencyAnalyzer;
import com.kennycason.kumo.nlp.tokenizers.ChineseWordTokenizer;
import com.kennycason.kumo.palette.ColorPalette;
import java.awt.Color;
import java.awt.Dimension;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.List;/*** 生成图云的工具类*/
public class WordCloudUtil {/*** 生成词云** @param artistId 歌单id* @param texts 文本*/public static void generate(String artistId, List<String> texts) {FrequencyAnalyzer frequencyAnalyzer = new FrequencyAnalyzer();//设置返回的词数frequencyAnalyzer.setWordFrequenciesToReturn(500);//设置返回的词语最小出现频次frequencyAnalyzer.setMinWordLength(4);//引入中文解析器frequencyAnalyzer.setWordTokenizer(new ChineseWordTokenizer());//输入文章数据，进行分词final List<WordFrequency> wordFrequencyList = frequencyAnalyzer.load(texts);//设置图片分辨率大小Dimension dimension = new Dimension(600, 600);//此处的设置采用内置常量即可，生成词云对象WordCloud wordCloud = new WordCloud(dimension, CollisionMode.PIXEL_PERFECT);//设置边界及字体wordCloud.setPadding(2);// 设置字体，字体必须支持中文，不能随便改wordCloud.setKumoFont(new KumoFont("阿里巴巴普惠体 Light", FontWeight.PLAIN));//设置词云显示的三种颜色，越靠前设置表示词频越高的词语的颜色wordCloud.setColorPalette(new ColorPalette(new Color(0x4055F1), new Color(0x408DF1), new Color(0x40AAF1),new Color(0x40C5F1), new Color(0x40D3F1), new Color(0xFFFFFF)));wordCloud.setFontScalar(new SqrtFontScalar(10, 70));//设置背景图层为圆形wordCloud.setBackground(new CircleBackground(300));//生成词云wordCloud.build(wordFrequencyList);//输出到图片文件，用当前的毫秒数作为文件名Long milliSecond = LocalDateTime.now().toInstant(ZoneOffset.of("+8")).toEpochMilli();//输出到图片文件wordCloud.writeToFile("wordCloud-" + artistId + ".png");}
}

4、最后测试类

package com.youkeda.music.test;import com.youkeda.music.model.Artist;
import com.youkeda.music.model.Song;
import com.youkeda.music.service.SongCrawlerService;
import com.youkeda.music.service.impl.SongCrawlerServiceImpl;/*** 检查服务是否可以正确返回对象*/
public class SongCrawlerTest {private static final String SA_DING_DING = "萨顶顶";private static final String A_ID = "9270";private static final String ZUO_SHOU_ZHI_YUE = "左手指月";private static final String S_ID = "536096151";public static void main(String[] args) {SongCrawlerService songService = new SongCrawlerServiceImpl();songService.start(A_ID);Artist artist = songService.getArtist(A_ID);System.out.println("歌单名称：" + artist.getName());if (!SA_DING_DING.equals(artist.getName())) {System.out.println("歌单名称错误，不是本测试用例指定的歌单。");System.exit(1);}Song song = songService.getSong(A_ID, S_ID);System.out.println("歌曲名称：" + song.getName());if (!ZUO_SHOU_ZHI_YUE.equals(song.getName())) {System.out.println("歌曲名称错误，不是本测试用例指定的歌曲。");System.exit(1);}if (!SA_DING_DING.equals(song.getSingers().get(0).getNickName())) {System.out.println("歌曲名称错误，不是本测试用例指定的歌曲。");System.exit(1);}if (!"香蜜沉沉烬如霜 电视原声音乐专辑".equals(song.getAlbum().getName())) {System.out.println("专辑名称错误，不是本测试用例指定的歌曲的专辑。");System.exit(1);}if (song.getSourceUrl() == null) {System.out.println("歌曲名称错误，不是本测试用例指定的歌曲。");System.exit(1);}if (song.getHotComments() == null || song.getHotComments().isEmpty()) {System.out.println("歌曲热门评论错误，没有正确抓取评论数据。");System.exit(1);}System.out.println("歌曲所属专辑名称：" + song.getAlbum().getName());System.out.println("歌曲的歌手名称：" + song.getSingers().get(0).getNickName());System.out.println("歌曲音乐为文件地址：" + song.getSourceUrl());System.out.println("歌曲热门评论：" + song.getHotComments().get(0).getContent());System.out.println("歌曲服务运行成功。非常棒！");System.exit(0);}
}

运行成功就大功告成啦！！啊哈哈哈——出来吧！词云！

三、总结

也不知为什么，会写这一篇文章，（这是我刚做完高数作业的时候，突发奇想就来这里码了）可能就是想记录一下自己的小成功，让自己确信自己在进步，让自己小快乐一下，毕竟自己能做个小项目出来，我真的，哈哈哈蛮激动的。再接再厉！希望看到这里的你也能找到属于自己的学习方法，早日从小白晋升为大佬，啊哈哈哈。

第一次csdn创作，求大佬指正！

求多多鼓励支持一下下，奥利给！

java爬取网易云热评并制作词云。相关推荐

python爬取网易云热评
在爬取了妹子图之后,感觉总想再爬点什么东西增加编程兴趣,因为平时喜欢听歌嘛,然后就想到了爬取网易云热评,一首歌无数个故事23333- 首先打开网易云音乐网页版,找到自己想要爬取的那首歌(我这里搜的是隔 ...
java 爬虫音乐,Java爬取网易云音乐所有歌曲一：爬取所有歌手及其对应id
使用jsoup包进行爬虫 org.jsoup jsoup 1.6.1 具体代码如下下一篇:根据歌手Id获取所有专辑 package com.ssm.jsoup.music; import co ...
python爬取网易云音乐生成王力宏歌曲词云
python爬取网易云音乐生成王力宏歌曲词云 # -*- coding:utf-8 -*- # 网易云音乐,通过歌手id生成词云 import requests import sys,re,os fr ...
Python爬取网易云热歌榜所有音乐及其热评
获取特定歌曲热评: 首先,我们打开网易云网页版,击排行榜,然后点击左侧云音乐热歌榜,如图: 关于如何抓取指定的歌曲的热评,参考这篇文章,很详细,对小白很友好: 手把手教你用Python爬取网易云40万 ...
python爬取网易云热歌榜
Python爬取网易云音乐热歌榜歌曲,并下载到本地找到要下载歌曲排行榜的链接,这里用的是: https://music.163.com/discover/toplist?id=3778678 #网易 ...
java爬取网易云歌单_爬虫爬取网易云歌单
一.主题式网络爬虫设计方案 1.主题式网络爬虫名称:爬取网易云音乐歌单 2.主题式网络爬虫爬取的内容与数据特征分析爬取网易云音乐歌单前十页歌单,轻音乐类型的歌单名称.歌单播放量.歌单链接.用户名称. ...
Python爬取网易云歌曲评论，做词云分析
前言 emmmm 没什么说的,想说的都在代码里环境使用 Python 3.8 解释器 3.10 Pycharm 2021.2 专业版 selenium 3.141.0 本次要用到selenium模块 ...
java爬取网易云歌单_GitHub - th720309/163music_spider: 网易云音乐歌单爬取
今天花了一点时间抓取了网易云音乐的热门民谣歌单,共1500热门民谣歌单,后续有时间会爬取其他分类. 下面记录一下java爬取过程.见下: 爬虫过程 1.首先抓取各个歌单的url与标题 public s ...
爬取微博好友所发微博制作词云
打开一个关注者的微博页面,打开开发者工具,选择图中两个地方刷新页面,向下滚动,中途会出现正在加载,直到最后会出现下一页, 这时看开发者工具,这时真正请求的URL才出现然后找到Cookies,复制修 ...