爬取京东收件地址下得所有数据

1.工具备用

package reptile;import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;/*** 京东服务地址* @author daiyang**/
public class Reptile4 {public static  int i = 0;public static  int j = 0;public static void main(String[] args) throws Exception {//解析省编码String jdProvince = cover("\u5317\u4eac|1|72|1,\u4e0a\u6d77|2|78|1,\u5929\u6d25|3|51035|1,"+ "\u91cd\u5e86|4|113|1,\u6cb3\u5317|5|142,\u5c71\u897f|6|303,\u6cb3\u5357|7|412,"+ "\u8fbd\u5b81|8|560,\u5409\u6797|9|639,\u9ed1\u9f99\u6c5f|10|698,\u5185\u8499\u53e4|11|799,"+ "\u6c5f\u82cf|12|904,\u5c71\u4e1c|13|1000,\u5b89\u5fbd|14|1116,\u6d59\u6c5f|15|1158,\u798f\u5efa|16|1303,"+ "\u6e56\u5317|17|1381,\u6e56\u5357|18|1482,\u5e7f\u4e1c|19|1601,\u5e7f\u897f|20|1715,\u6c5f\u897f|21|1827,"+ "\u56db\u5ddd|22|1930,\u6d77\u5357|23|2121,\u8d35\u5dde|24|2144,\u4e91\u5357|25|2235,\u897f\u85cf|26|2951,"+ "\u9655\u897f|27|2376,\u7518\u8083|28|2487,\u9752\u6d77|29|2580,\u5b81\u590f|30|2628,\u65b0\u7586|31|2652,"+ "\u6e2f\u6fb3|52993|52994,\u53f0\u6e7e|32|2768,\u9493\u9c7c\u5c9b|84|84");//读取市源数据String unicodeCity = readFile("D:\\test\\city.txt");//解析市编码String jdCity = cover(unicodeCity);//获取省数据List<Map<String,Object>> provinceList = provinceDataHandle(jdProvince);//获取市数据List<Map<String, Object>> cityList= cityDataHandle(jdCity);   //线程安全的区县数据ConcurrentLinkedQueue<Map<String,Object>> districtList = new ConcurrentLinkedQueue<Map<String,Object>>();//线程安全的乡镇数据ConcurrentLinkedQueue<Map<String,Object>> courtList = new ConcurrentLinkedQueue<Map<String,Object>>();//数据处理dataHandle(provinceList, cityList, districtList);System.out.println(JSON.toJSON(provinceList));System.out.println(JSON.toJSON(cityList));System.out.println(JSON.toJSON(districtList));//开始表演---->>>>市Id请求 AtomicInteger atoI = new AtomicInteger(0); //查询哪个市下得区县乡镇数据,成都市列表第325个,绵阳市第329个//容量上限为50的线程池ExecutorService es = Executors.newFixedThreadPool(50);System.out.println("===========>>>>>>>>>>>>>>>>>>开始搜索数据");int taskNum  = 1;//while(taskNum<=cityList.size()){//cityList.size()------------------------------------->>>开闸Runnable task = new Runnable() {               @Overridepublic void run() {getDistrictInfo(courtList,districtList,cityList, atoI);}};es.submit(task);taskNum++;             }es.shutdown();while(true){  if(es.isTerminated()){  System.out.println("---END---\n");System.out.println("所有的子线程都结束了！");//*************************************数据正确处理*******************************************//*System.out.println("=================>>>>>>>>>>>>>>>开始存入数据库");//addProvinceData(provinceList);//添加省数据//addCityData(cityList);//添加市数据//addDistructData(districtList);//添加区县数据 3600多个addTownData(courtList);//添加乡镇数据 39836个break;  }Thread.sleep(1000);    }}static Connection conn;static PreparedStatement ps;static ResultSet rs;/*** 写一个连接数据库的方法*/public static Connection getConnection(){String url="jdbc:mysql://localhost:port/database";String userName="username";String password="password";try {Class.forName("com.mysql.jdbc.Driver");} catch (ClassNotFoundException e) {System.out.println("找不到驱动！");e.printStackTrace();}try {conn=DriverManager.getConnection(url, userName, password);if(conn!=null){System.out.println("connection successful");}} catch (SQLException e) {// TODO Auto-generated catch blockSystem.out.println( "connection fail");e.printStackTrace();}return conn;}public static int addTownData(ConcurrentLinkedQueue<Map<String,Object>> list){int row=0;String sql="insert into tb_town(name,districtId,jdTownId) values(?,?,?)";try {conn=getConnection();//连接数据库ps=conn.prepareStatement(sql);// 2.创建Satement并设置参数
//            rs=ps.executeQuery();  // 3.ִ执行SQL语句,緊緊用于查找語句//sql語句中寫了幾個字段，下面就必須要有幾個字段for(Map<String,Object> map:list){System.out.println("FBIWARNING  i....:"+(i++));ps.setString(1, (String)map.get("name"));ps.setInt(2, Integer.valueOf((String)map.get("districtId")));ps.setInt(3, Integer.valueOf((String)map.get("id")));// 4.处理结果集row=ps.executeUpdate();}} catch (SQLException e) {// TODO Auto-generated catch blocke.printStackTrace();}finally{try {ps.close();conn.close();} catch (SQLException e) {e.printStackTrace();}}return row;}public static int addDistructData(ConcurrentLinkedQueue<Map<String,Object>> list){int row=0;String sql="insert into tb_district(name,cityId,jdDistrictId) values(?,?,?)";try {conn=getConnection();//连接数据库ps=conn.prepareStatement(sql);// 2.创建Satement并设置参数
//            rs=ps.executeQuery();  // 3.ִ执行SQL语句,緊緊用于查找語句//sql語句中寫了幾個字段，下面就必須要有幾個字段for(Map<String,Object> map:list){ps.setString(1, (String)map.get("name"));ps.setInt(2, Integer.valueOf((String)map.get("cityId")));ps.setInt(3, Integer.valueOf((String)map.get("id")));// 4.处理结果集row=ps.executeUpdate();}} catch (SQLException e) {// TODO Auto-generated catch blocke.printStackTrace();}finally{try {ps.close();conn.close();} catch (SQLException e) {e.printStackTrace();}}return row;}public static int addCityData(List<Map<String,Object>> list){int row=0;String sql="insert into tb_city(name,provinceId,jdCityId) values(?,?,?)";try {conn=getConnection();//连接数据库ps=conn.prepareStatement(sql);// 2.创建Satement并设置参数
//            rs=ps.executeQuery();  // 3.ִ执行SQL语句,緊緊用于查找語句//sql語句中寫了幾個字段，下面就必須要有幾個字段for(Map<String,Object> map:list){ps.setString(1, (String)map.get("cityName"));ps.setInt(2, Integer.valueOf((String)map.get("provinceId")));ps.setInt(3, Integer.valueOf((String)map.get("cityId")));// 4.处理结果集row=ps.executeUpdate();}} catch (SQLException e) {// TODO Auto-generated catch blocke.printStackTrace();}finally{try {ps.close();conn.close();} catch (SQLException e) {e.printStackTrace();}}return row;}public static int addProvinceData(List<Map<String,Object>> list){int row=0;String sql="insert into tb_province(name,provinceCode,provinceType,jdProvinceId) values(?,?,?,?)";try {conn=getConnection();//连接数据库ps=conn.prepareStatement(sql);// 2.创建Satement并设置参数
//            rs=ps.executeQuery();  // 3.ִ执行SQL语句,緊緊用于查找語句//sql語句中寫了幾個字段，下面就必須要有幾個字段for(Map<String,Object> map:list){ps.setString(1, (String)map.get("provinceName"));ps.setString(2, (String)map.get("provinceCode"));ps.setInt(3, Integer.valueOf((String)map.get("provinceType")));ps.setInt(4, Integer.valueOf((String)map.get("provinceId")));// 4.处理结果集row=ps.executeUpdate();}} catch (SQLException e) {// TODO Auto-generated catch blocke.printStackTrace();}finally{try {ps.close();conn.close();} catch (SQLException e) {e.printStackTrace();}}return row;}private static void dataHandle(List<Map<String, Object>> provinceList,List<Map<String, Object>> cityList,ConcurrentLinkedQueue<Map<String, Object>> districtList) {//1.直辖市下面的区跑到市去了Iterator<Map<String, Object>> iterator = cityList.iterator();while(iterator.hasNext()){Map<String,Object> map = iterator.next();if(Integer.valueOf((String)map.get("provinceId"))<=4){iterator.remove();}}for(int i = 0; i<4;i++){Map<String, Object> map = provinceList.get(i);Map<String, Object> newMap = new HashMap<String, Object>();newMap.put("cityName", map.get("provinceName"));newMap.put("cityId", map.get("provinceId"));newMap.put("provinceId", map.get("provinceId"));cityList.add(newMap);}}//获取乡镇数据private static void getCourtInfo(ConcurrentLinkedQueue<Map<String,Object>> courtList,String districtId) {//爬取第四级乡镇数据String url = "https://d.jd.com/area/get?fid="+districtId;//System.out.println(url);try {String request = request(url);JSONArray jarr=JSONArray.parseArray(request);//JSON.parseArray(jsonStr);  for (Iterator iterator = jarr.iterator(); iterator.hasNext();) {Map<String,Object> one = new HashMap<>();JSONObject job=(JSONObject)iterator.next(); String name=job.get("name").toString();String id = job.getString("id").toString();one.put("name", name);one.put("id", id);one.put("districtId",districtId);courtList.add(one);}} catch (Exception e) {e.printStackTrace();System.out.println("请求地址错误");}}//获取区县数据private static void getDistrictInfo(ConcurrentLinkedQueue<Map<String,Object>> courtList,ConcurrentLinkedQueue<Map<String,Object>> districtList,List<Map<String, Object>> cityList, AtomicInteger atoI) {Map<String, Object> map = cityList.get(atoI.getAndIncrement());//爬取第三级区县数据String url = "https://d.jd.com/area/get?fid="+map.get("cityId");//System.out.println(url);try {String request = request(url);JSONArray jarr=JSONArray.parseArray(request);//JSON.parseArray(jsonStr);  for (Iterator iterator = jarr.iterator(); iterator.hasNext();) {Map<String,Object> one = new HashMap<>();JSONObject job=(JSONObject)iterator.next(); String name=job.get("name").toString();String id = job.getString("id").toString();one.put("name", name);one.put("id", id);one.put("cityId",map.get("cityId"));districtList.add(one);//getCourtInfo(courtList,id);}} catch (Exception e) {e.printStackTrace();System.out.println("请求地址错误");}}//发起请求private static String request(String url) throws Exception {// 定义一个缓冲字符输入流BufferedReader in = null;// 将string转成url对象URL realUrl = new URL(url);// 初始化一个链接到那个url的连接URLConnection connection = realUrl.openConnection();// 开始实际的连接connection.connect();// 初始化 BufferedReader输入流来读取URL的响应in = new BufferedReader(new InputStreamReader(connection.getInputStream()));String line = null;String content = "";while((line = in.readLine())!=null){content+=line;}return content;}//市数据处理private static List<Map<String,Object>> cityDataHandle(String cityStr){String[] cityStrs = cityStr.split("hello,dy");List<Map<String,Object>> cityList = new ArrayList<Map<String,Object>>();for(String city:cityStrs){String delResult = city.replace("\"", "");String split[] = delResult.split(":");String[] cities = split[1].split(",");for(String str:cities){Map<String,Object> one = new HashMap<String, Object>();if(str.trim().isEmpty())//空白串处理continue;String data[] = str.split("\\|");one.put("cityName", data[0]);one.put("cityId", data[1]);one.put("provinceId", split[0].replace("\t", ""));cityList.add(one);}}return cityList;}//读取市数据源private static String readFile(String fileName){File file = new File(fileName);String content = "";try {BufferedReader br = new BufferedReader(new FileReader(file));String line;while((line=br.readLine())!=null){content += (line+"hello,dy");}} catch (Exception e) {e.printStackTrace();}return content.substring(0,content.length()-8);}//省数据处理private static List<Map<String,Object>> provinceDataHandle(String provinceStr){String[] proviceStrs = provinceStr.split(",");List<Map<String,Object>> provinceList = new ArrayList<Map<String,Object>>();for(String provice:proviceStrs){Map<String,Object> one = new HashMap<String, Object>();String data[] = provice.split("\\|");one.put("provinceName", data[0]);one.put("provinceId", data[1]);one.put("provinceCode", data[2]);if(data.length>3)one.put("provinceType", data[3]);//1为直辖市elseone.put("provinceType", "2");//2为省provinceList.add(one);}return provinceList;}//解码规则--16进制unicode编码public static String cover(String s){String re = "",sub = null;char c1,c2;for(int i=0;i<s.length()-1;i++){c1 = s.charAt(i);c2 = s.charAt(i+1);if(c1 == '\\' && c2 =='u'){sub = s.substring(i+2,i+6);re = re + (char)Integer.parseInt(sub,16);i+=5;}else{re = re+c1;}}return re;}//NIO非阻塞式读写@SuppressWarnings("static-access")public static void writeByNIO(String content,File file) {RandomAccessFile fout = null;FileChannel fcout = null;try {fout = new RandomAccessFile(file, "rw");long filelength = fout.length();//获取文件的长度fout.seek(filelength);//将文件的读写指针定位到文件的末尾fcout = fout.getChannel();//打开文件通道FileLock flout = null;while (true) {try {flout = fcout.tryLock();//不断的请求锁，如果请求不到，等一秒再请求break;} catch (Exception e) {System.out.print("lock is exist ......");Thread.currentThread().sleep(1000);}}fout.write(content.getBytes());//将需要写入的内容写入文件flout.release();fcout.close();fout.close();} catch (IOException e1) {e1.printStackTrace();System.out.print("file no find ...");} catch (InterruptedException e) {e.printStackTrace();} finally {if (fcout != null) {try {fcout.close();} catch (IOException e) {e.printStackTrace();fcout = null;}}if (fout != null) {try {fout.close();} catch (IOException e) {e.printStackTrace();fout = null;}}}}
}

能直接爬出京东的全国地址并拷贝到本地数据库中,使用的话注意数据库连接和表结构.

爬取京东收件地址下得所有数据相关推荐

（详细总结）python爬取 163收件箱邮件内容，收件箱列表的几种方法(urllib, requests, selenium)
需求:最近有一个需求,需要将163邮箱收件箱里面的所有邮件的内容全部copy下来,整理到一个word里面,不多也就28页的邮件(不要问我为什么有这需求,不告诉你),自己手动去ctrl+ cv 的话,估 ...
练习：selenium 爬取京东的电脑商品100页的数据并保存到csv文件中
练习:selenium 爬取京东的电脑商品100页的数据并保存到csv文件中 from selenium.webdriver import Chrome, ChromeOptions import t ...
天冷了，我用Python爬取京东4950件羽绒服数据并可视化
前言大家好,我是J哥. 前不久,我还穿着短袖羡慕着北方的下雪气氛.结果就在上周,深圳也迎来了降温,并成功加入"降温群聊"! 为了抵御严寒,我特地爬了下京东的羽绒服数据,以备不时之 ...
python爬京东反爬_京东的验证码和反爬都很烦人吧？那又怎样，照样轻松爬取京东数据...
目标爬取京东商城上iphone x用户评论数据: 使用jieba对评论数据进行分词处理: 使用wordcloud绘制词云图. 进群:548377875即可获取数十套pdf哦! 然后我们会发现这个接口 ...
用python画苹果_Python爬取京东Iphone X用户评论并绘制词云
目标爬取京东商城上iPhone X用户评论数据: 使用jieba对评论数据进行分词处理: 使用wordcloud绘制词云图. 目前京东商城只会展示商品的前100页评论,所以我们能爬取到的评论只有100 ...
爬取京东笔记本电脑销量榜每件商品详情页url，并存入云服务器中的mysql库
文章目录一.目的分析二.爬取页面商品所有详情页链接 1.爬取href链接 2.将数据存入数据库完整代码: 三.多线程优化版已完成 (多线程优化)爬取京东笔记本电脑销量榜data-sku(商品id ...
（多线程优化）爬取京东笔记本电脑销量榜data-sku(商品id)，并存入云服务器中的mysql库（爬取时间较上一版提升十多倍）
文章目录前言一.优化内容 1.cromeOptions()加载优化 2.ThreadPool()中的pool.map()多线程优化二.完整代码三.运行效果截图前言本文为上一篇爬虫文章的多线 ...
八个commit让你学会爬取京东商品信息
我发现现在不用标题党的套路还真不好吸引人,最近在做相关的事情,从而稍微总结出了一些文字.我一贯的想法吧,虽然才疏学浅,但是还是希望能帮助需要的人.博客园实在不适合这种章回体的文章.这里,我贴出正文的前 ...
python制作爬虫爬取京东商品评论教程
python制作爬虫爬取京东商品评论教程作者:蓝鲸类型:转载本文是继前2篇Python爬虫系列文章的后续篇,给大家介绍的是如何使用Python爬取京东商品评论信息的方法,并根据数据绘制成各种统计 ...
爬取京东网页评论（动态网页）
1.当网页打开的方式不同时,在开发者选项找到的包含评论的文件地址不同,比如第一种,当我们找到的评论界面是含有下一页选项的时候(如下图).我们在左边文件界面发现包含评论的网页地址名字为''product ...

爬取京东收件地址下得所有数据

爬取京东收件地址下得所有数据相关推荐

最新文章

热门文章