Java爬虫简解-疫情数据爬取

首先在爬取数据之前需要先找到对应的数据接口，腾讯疫情数据接口

之后会获取到一个json数据，可以使用json解析工具把json数据格式化，之后就可以使用Java来解析该网站中的数据了，在解析之前需要先导入jsoup和fastjson包。

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.jsoup.Jsoup;import java.io.IOException;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;/*** @Author: ChenHC* @Date: 2022/6/17 7:23*/
public class QQData {public static void main(String[] args) throws IOException {getAllData();}public static void get() throws IOException {//腾讯疫情数据   https://api.inews.qq.com/newsqa/v1/query/inner/publish/modules/list?modules=localCityNCOVDataList,diseaseh5Shelf//使用jsoup提供的connect方法连接接口获取一个String字符串String resultBody = Jsoup.connect("https://api.inews.qq.com/newsqa/v1/query/inner/publish/modules/list?modules=localCityNCOVDataList,diseaseh5Shelf").ignoreContentType(true).execute().body();//将获取到的字符串转换为JSONObject格式便于解析JSONObject jsonObject = JSON.parseObject(resultBody);JSONObject resultJson = new JSONObject();}public static Map<String,Object> getAllData() throws IOException {//腾讯疫情数据   https://api.inews.qq.com/newsqa/v1/query/inner/publish/modules/list?modules=localCityNCOVDataList,diseaseh5ShelfString resultBody = Jsoup.connect("https://api.inews.qq.com/newsqa/v1/query/inner/publish/modules/list?modules=localCityNCOVDataList,diseaseh5Shelf").ignoreContentType(true).execute().body();JSONObject jsonObject = JSON.parseObject(resultBody);//获取data部分JSONObject data = jsonObject.getJSONObject("data");//获取高风险地区数据JSONArray localCityNCOVDataList = data.getJSONArray("localCityNCOVDataList");Map<String,Object> highCitysMap = new HashMap<>();System.out.println("高风险疫情地区数据");for(int i = 0;i < localCityNCOVDataList.size();i++){JSONObject highCity = localCityNCOVDataList.getJSONObject(i);//高风险地区疫情数据System.out.println(highCity);// 数据部分String city = highCity.getString("city");String province = highCity.getString("province");//本土确诊int local_confirm_add = highCity.getIntValue("local_confirm_add");//新增无症状int local_wzz_add = highCity.getIntValue("local_wzz_add");//高风险地区int highRiskAreaNum = highCity.getIntValue("highRiskAreaNum");//中风险地区int mediumRiskAreaNum = highCity.getIntValue("mediumRiskAreaNum");Map<String,Object> highCityMap = new HashMap<>();highCityMap.put("city",city);highCityMap.put("province",province);highCityMap.put("local_confirm_add",local_confirm_add);highCityMap.put("local_wzz_add",local_wzz_add);highCityMap.put("highRiskAreaNum",highRiskAreaNum);highCityMap.put("mediumRiskAreaNum",mediumRiskAreaNum);highCitysMap.put("name",highCityMap);}JSONObject diseaseh5Shelf = data.getJSONObject("diseaseh5Shelf");//获取国内34个省市的疫情数据JSONArray areaTree = diseaseh5Shelf.getJSONArray("areaTree");JSONObject allProvinces = areaTree.getJSONObject(0);JSONArray provinces = allProvinces.getJSONArray("children");Map<String,Object> provincesMap = new HashMap<>();System.out.println("各省份疫情数据");for (int i = 0;i < provinces.size();i++){JSONObject province = provinces.getJSONObject(i);//获取省份数据Map<String, Object> provinceMap = getCityValues(province);//打印省份数据System.out.println((String)provinceMap.get("name")+province+"\n\t城市数据：");provincesMap.put((String)provinceMap.get("name"),provinceMap);JSONArray citys = province.getJSONArray("children");Map<String,Object> citysMap = new HashMap<>();for(int j = 0;j < citys.size();j++){JSONObject city = citys.getJSONObject(j);//获取城市数据Map<String, Object> cityMap = getCityValues(city);//打印城市数据System.out.println("\t"+(String)cityMap.get("name")+city);citysMap.put((String)cityMap.get("name"),cityMap);}}//获取国内全国疫情数据Map<String,Object> chinaMap = new HashMap<>();JSONObject chinaTotal = diseaseh5Shelf.getJSONObject("chinaTotal");//已治愈人数int heal = chinaTotal.getIntValue("heal");//int dead = chinaTotal.getIntValue("dead");//新增无症状int localWzzAdd = chinaTotal.getIntValue("localWzzAdd");//当前无症状int nowLocalWzz = chinaTotal.getIntValue("nowLocalWzz");//所有病例（包括已治愈和死亡的）int confirm = chinaTotal.getIntValue("confirm");//新增病例int confirmAdd = chinaTotal.getIntValue("confirmAdd");//当前病例int nowConfirm = chinaTotal.getIntValue("nowConfirm");//本土病例int localConfirm = chinaTotal.getIntValue("localConfirm");//新增死亡int deadAdd = chinaTotal.getIntValue("deadAdd");//本土新增病例int localConfirmAdd = chinaTotal.getIntValue("localConfirmAdd");//中风险地区int mediumRiskAreaNum = chinaTotal.getIntValue("mediumRiskAreaNum");//高风险地区int highRiskAreaNum = chinaTotal.getIntValue("highRiskAreaNum");//全国新增数据//JSONObject chinaAdd = diseaseh5Shelf.getJSONObject("chinaAdd");chinaMap.put("heal",heal);chinaMap.put("dead",dead);chinaMap.put("localWzzAdd",localWzzAdd);chinaMap.put("nowLocalWzz",nowLocalWzz);chinaMap.put("confirm",confirm);chinaMap.put("confirmAdd",confirmAdd);chinaMap.put("nowConfirm",nowConfirm);chinaMap.put("localConfirm",localConfirm);chinaMap.put("deadAdd",deadAdd);chinaMap.put("localConfirmAdd",localConfirmAdd);chinaMap.put("mediumRiskAreaNum",mediumRiskAreaNum);chinaMap.put("highRiskAreaNum",highRiskAreaNum);//数据截止时间Date lastUpdateTime1 = diseaseh5Shelf.getDate("lastUpdateTime");Map<String,Object> resultMap = new HashMap<>();resultMap.put("provincesMap",provincesMap);resultMap.put("chinaMap",chinaMap);resultMap.put("highCitysMap",highCitysMap);resultMap.put("lastUpdateTime1",lastUpdateTime1);return resultMap;}public static Map<String,Object> getCityValues(JSONObject province){//各个省份的数据String name = province.getString("name");JSONObject today = province.getJSONObject("today");int todayConfirm = today.getIntValue("confirm");int wzz_add = today.getIntValue("wzz_add");int local_confirm_add = today.getIntValue("local_confirm_add");JSONObject total = province.getJSONObject("total");int confirm = total.getIntValue("confirm");int nowConfirm = total.getIntValue("nowConfirm");int wzz = total.getIntValue("wzz");//中风险地区数量int mediumRiskAreaNum = total.getIntValue("mediumRiskAreaNum");//高风险地区数量int highRiskAreaNum = total.getIntValue("highRiskAreaNum");int heal = total.getIntValue("heal");int dead = total.getIntValue("confirm");Map<String, Object> provinceMap = new HashMap<>();provinceMap.put("name",name);provinceMap.put("todayConfirm",todayConfirm);provinceMap.put("wzz_add",wzz_add);provinceMap.put("local_confirm_add",local_confirm_add);provinceMap.put("confirm",confirm);provinceMap.put("nowConfirm",nowConfirm);provinceMap.put("wzz",wzz);provinceMap.put("mediumRiskAreaNum",mediumRiskAreaNum);provinceMap.put("highRiskAreaNum",highRiskAreaNum);provinceMap.put("heal",heal);provinceMap.put("dead",dead);return provinceMap;}
}

如图，通过Jsoup.connect（）的方法来获取到网页的数据，接着使用fastjson提供的JSONObject类和JSONArray类解析数据。

JSONObject类：

存储的时对象的格式：如

{'name':"疫情数据",'data':{'comfrim':7,'dead':12}}

JSONArray类：

存储的是数组格式：如

["shanghai",{'dead':300,'comfrim':1200},13]

Java爬虫简解-疫情数据爬取相关推荐

java爬虫实现百度地图数据爬取
本次项目主要实现百度地图地点检索功能的数据爬取,可以获得检索的相关信息.主要是采用百度地图API接口实现,采用的是servlet,数据库采用的是mybatis.话不多说,上代码. 1.DAO层数据 p ...
每日一练：Python国内疫情数据爬取与地图绘制
Python 国内疫情数据爬取与地图绘制效果图累计确诊疫情地图绘制 ① 时时数据抓取 ② 获取省份疫情数据 ③ 视觉配置项分段颜色数据设置 ④ 累计确诊疫情地图绘制现存确诊疫情地图绘制 ① 获取 ...
搜狗·疫情数据爬取（Python）
上周已经分享过搜狗·疫情数据爬取(R语言),这次分享一下搜狗·疫情数据爬取(Python) 不说废话,直接上代码.有什么问题,可以在留言区讨论. from urllib import request ...
Java爬虫历险记 -- （1）爬取百度首页的logo
Java爬虫历险记 – (1)爬取百度首页的logo 在这篇文章里,介绍两种方式来获取百度网页的logo: (1)Httpclient (2) jsoup + Httpclient ,详细的运行结果可 ...
疫情数据爬取，可视化及其预测
疫情数据爬取及可视化数据爬取及保存(provinceDataGet.py) import requests import json import pandas as pd# 地区 areas = [ ...
爬虫项目3 - 股票数据爬取
爬虫项目3 - 股票数据爬取步骤步骤爬取股票名和股票列表,使用gucheng网进行爬取,网址: https://hq.gucheng.com/gpdmylb.html import reques ...
python爬虫案例-陶瓷公司数据爬取
用requests爬取要注意HTTPConnectionPool(host=xxx, port=xxx): Max retries exceeded with url...异常,出现这个异常的解决方法 ...
Python爬虫 —— 以北京天气数据爬取为例
本文以北京天气为例讲解数据爬取的整个流程,不涉及网络爬虫的原理,直接讲爬取代码怎么写! 1.首先找到你要爬取的网站url:'http://www.tianqihoubao.com/lishi/beij ...
python爬虫实例——某二手车数据爬取
某二手车网站数据爬取要求: 找到所要爬取的网站网址(url): 今天案例的网址(url):https://www.guazi.com/gy/dazhong/o1/#bread. 观察网站,点开检查, ...

Java爬虫简解-疫情数据爬取

Java爬虫简解-疫情数据爬取相关推荐

最新文章

热门文章