爬取京东商品数据

我把项目部署到了linux中,进行爬取,爬到了3000条手机信息,只是爬了一些简单的文本信息.

本文爬取的数据为京东手机信息

准备工作

导入爬取数据需要的依赖包

编写httpClient工具类

编写pojo类

编写dao

<dependencies><dependency><groupId>org.apache.httpcomponents</groupId><artifactId>httpclient</artifactId><version>4.5.4</version></dependency><dependency><groupId>org.jsoup</groupId><artifactId>jsoup</artifactId><version>1.10.3</version></dependency><dependency><groupId>mysql</groupId><artifactId>mysql-connector-java</artifactId><version>5.1.38</version></dependency><dependency><groupId>org.springframework</groupId><artifactId>spring-jdbc</artifactId><version>4.2.4.RELEASE</version></dependency><dependency><groupId>c3p0</groupId><artifactId>c3p0</artifactId><version>0.9.1.2</version></dependency><dependency><groupId>com.google.code.gson</groupId><artifactId>gson</artifactId><version>2.8.1</version></dependency></dependencies>

package com.hrh.utils;import com.hrh.pojo.Product;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;public class HttpClientUtils {//创建httpclient连接池private static PoolingHttpClientConnectionManager connectionManager;static{connectionManager=new PoolingHttpClientConnectionManager();//定义连接池最大连接数connectionManager.setMaxTotal(200);//对指定的网址最多只有20个连接connectionManager.setDefaultMaxPerRoute(20);}private static CloseableHttpClient getCloseableHttpClient(){CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(connectionManager).build();return httpClient;}private static String execute(HttpRequestBase httpRequestBase) throws IOException {httpRequestBase.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0");//设置超时时间RequestConfig config = RequestConfig.custom().setConnectionRequestTimeout(5000).setConnectTimeout(5000).setSocketTimeout(10 * 1000).build();httpRequestBase.setConfig(config);CloseableHttpClient httpClient = getCloseableHttpClient();CloseableHttpResponse response = httpClient.execute(httpRequestBase);String html = EntityUtils.toString(response.getEntity(), "utf-8");return html;}public static String doGet(String url) throws IOException {HttpGet httpGet = new HttpGet(url);String html = execute(httpGet);return html;}public static String doPost(String url, Map<String,String> params) throws IOException {HttpPost httpPost = new HttpPost(url);List<BasicNameValuePair> list = new ArrayList<>();for (String key : params.keySet()) {list.add(new BasicNameValuePair(key,params.get(key)));}UrlEncodedFormEntity entity = new UrlEncodedFormEntity(list);httpPost.setEntity(entity);return execute(httpPost);}
}

package com.hrh.pojo;/*** 商品表*/
public class Product {private String pid;private String title;private String brand;private String pname;private String price;public String getPid() {return pid;}public void setPid(String pid) {this.pid = pid;}public String getTitle() {return title;}public void setTitle(String title) {this.title = title;}public String getBrand() {return brand;}public void setBrand(String brand) {this.brand = brand;}public String getPname() {return pname;}public void setPname(String pname) {this.pname = pname;}public String getPrice() {return price;}public void setPrice(String price) {this.price = price;}@Overridepublic String toString() {return "Product{" +"pid=" + pid +", title='" + title + '\'' +", brand='" + brand + '\'' +", pname='" + pname + '\'' +", price=" + price +'}';}
}

package com.hrh.dao;import com.hrh.pojo.Product;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import org.springframework.jdbc.core.JdbcTemplate;import java.beans.PropertyVetoException;public class ProductDao extends JdbcTemplate{public ProductDao(){//定义c3p0连接池ComboPooledDataSource ds = new ComboPooledDataSource();try {ds.setDriverClass("com.mysql.jdbc.Driver");ds.setUser("root");ds.setPassword("123");ds.setJdbcUrl("jdbc:mysql://localhost:3306/crawler?characterEncoding=utf-8");} catch (PropertyVetoException e) {e.printStackTrace();}super.setDataSource(ds);}public void addProduct(Product product){super.update("insert into jd_phone values (?,?,?,?,?)",product.getPid(),product.getTitle(),product.getPname(),product.getBrand(),product.getPrice());}}

核心代码

创建线程池和队列开启线程等待队列中的数据并进行分析
博客: 线程池和队列的基本使用

在获得手机列表时,pid一个一个的解析,效率太低,解析完一个页面的pid,才能进入下一页,继续解析.所以要引入多线程

线程池的使用: 提高程序执行效率

如果使用线程池,就要考虑线程安全问题

pid在存储时,要放到线程安全的容器中, 并且容器时FIFO的

队列的使用: 线程安全(阻塞队列)

基本流程 :

确定手机列表页的URL 进行分析得到本页中所有手机的pid

将pid 放入阻塞队列中等待线程的解析

根据pid获得确定具体手机的URL 进行解析将数据封装到product对象中

调用dao 将product对象存到数据库中

package com.hrh.test;import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.hrh.dao.ProductDao;
import com.hrh.pojo.Product;
import com.hrh.utils.HttpClientUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;public class JDPhone {//创建dao对象static ProductDao productDao = new ProductDao();//创建线程池static ExecutorService threadPool = Executors.newFixedThreadPool(20);//创建原生阻塞队列  队列最大容量为1000static BlockingQueue<String> queue=new ArrayBlockingQueue<String>(1000);public static void main(String[] args) throws IOException, InterruptedException {//监视队列大小的线程threadPool.execute(new Runnable() {@Overridepublic void run() {while(true){try {Thread.sleep(1000);} catch (InterruptedException e) {e.printStackTrace();}//获得队列当前的大小int size = queue.size();System.out.println("当前队列中有"+size+"个pid");}}});//开启10个线程去解析手机列表页获得的pidsfor (int i = 1; i <=10; i++) {threadPool.execute(new Runnable() {@Overridepublic void run() {while (true){String pid=null;try {//从队列中取出pidpid = queue.take();Product product = parsePid(pid);//存入数据库productDao.addProduct(product);} catch (Exception e) {e.printStackTrace();try {//出现异常则放回队列queue.put(pid);} catch (InterruptedException e1) {e1.printStackTrace();}}}}});}//分页查找手机数据 共100页for (int i = 1; i <=100 ; i++) {//京东分页page为 1 3 5 7 .....//         对应第一页 第二页....String url="https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&page="+(2*i-1);String html = HttpClientUtils.doGet(url);parseIndex(html);}}//解析手机列表页private  static void parseIndex(String html) throws IOException, InterruptedException {Document document = Jsoup.parse(html);//手机列表Elements elements = document.select("#J_goodsList>ul>li");if(elements!=null||elements.size()!=0){for (Element element : elements) {//获得每个li的pidString pid = element.attr("data-pid");//将pid放入队列中queue.put(pid);}}}//解析每个手机的页面 获得某个手机的详细数据private static Product parsePid(String pid) throws IOException {//拼接url 进入手机详情页String productUrl="https://item.jd.com/"+pid+".html";String productHtml = HttpClientUtils.doGet(productUrl);Document document = Jsoup.parse(productHtml);Product product = new Product();//获得手机标题if(document.select("div.sku-name").size()>0){String title = document.select("div.sku-name").get(0).text();product.setTitle(title);}//获得手机品牌String brand = document.select("#parameter-brand li").attr("title");product.setBrand(brand);//获得手机名称String pname = document.select("[class=parameter2 p-parameter-list] li:first-child").attr("title");product.setPname(pname);/*  此方案无法获取到价格jd的价格采用异步刷新,price不在返回的html文档中,需要我们去请求价格页面Elements select = document.select("span[class=price J-p-" + pid + "]");System.out.println(select);*///拼接价格页面url 经过测试 返回Json数据  jd对IP进行了限制,加入pduid为随机数,是为了可以获取更多数据,但是依然只能爬取部分String priceUrl="https://p.3.cn/prices/mgets?pduid="+Math.random()+"&skuIds=J_"+pid;String priceJson = HttpClientUtils.doGet(priceUrl);System.out.println(priceJson);Gson gson = new GsonBuilder().create();List<Map<String,String>> list = gson.fromJson(priceJson, List.class);String price = list.get(0).get("p");product.setPrice(price);product.setPid(pid);return product;}}

出现的问题:

SocketTimeException 超时异常,因为jd对IP进行了限制,请求次数太多时,会被限制, 以后的文章会解决这个问题…

---------------------------------------------------------------------更新…-------------------------------------------------------------

爬笔记本数据的代码(只是更换了URL)

package com.hrh.test;import com.google.gson.Gson;
import com.hrh.dao.ProductDao;
import com.hrh.pojo.Product;
import com.hrh.utils.HttpClientUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;import java.util.List;
import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;/*** @author QuietHR* @create 2018/9/22**/
public class JDPC {private static BlockingQueue<String> queue=new ArrayBlockingQueue<String>(1000);private static  ExecutorService executorService = Executors.newFixedThreadPool(50);private static ProductDao productDao=new ProductDao();public static void main(String[] args) throws Exception {executorService.execute(new Runnable() {@Overridepublic void run() {while (true){try {Thread.sleep(1000);} catch (InterruptedException e) {e.printStackTrace();}int size = queue.size();System.out.println("当前队列中有"+size+"个pid");}}});for (int i = 0; i < 30; i++) {executorService.execute(new Runnable() {@Overridepublic void run() {while (true){String pid = null;try {pid = queue.take();Product product = parsePid(pid);productDao.addProduct(product);} catch (Exception e) {e.printStackTrace();try {queue.put(pid);} catch (InterruptedException e1) {e1.printStackTrace();}}}}});}page();}private static void page() throws Exception {for (int i = 1; i <=100 ; i++) {String url="https://search.jd.com/Search?keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC&enc=utf-8&page="+(2*i-1);String html = HttpClientUtils.doGet(url);parseIndex(html);}}private static void parseIndex(String html) throws InterruptedException {Document document = Jsoup.parse(html);Elements liEl = document.select("[class=gl-warp clearfix]>li");for (Element li : liEl) {queue.put(li.attr("data-sku"));}}private static Product parsePid(String pid) throws Exception {String url="https://item.jd.com/"+pid+".html";String html = HttpClientUtils.doGet(url);Document document = Jsoup.parse(html);Product product = new Product();product.setPid(pid);Elements titleEl = document.select("[class=sku-name]");product.setTitle(titleEl.text());Elements brandEl = document.select("#parameter-brand>li");product.setBrand(brandEl.attr("title"));Elements pnameEl = document.select("[class=parameter2 p-parameter-list]>li:first-child");product.setPname(pnameEl.attr("title"));String productUrl="https://p.3.cn/prices/mgets?pduid="+Math.random()+"&skuIds=J_"+pid;String json = HttpClientUtils.doGet(productUrl);Gson gson = new Gson();List<Map<String,String>> list = gson.fromJson(json, List.class);String price = list.get(0).get("p");product.setPrice(price);return product;}
}

Java爬取京东商品数据相关推荐

Python爬取京东商品数据
一.前言由于京东反爬技术较强,使用常规方法爬取其数据行不通,且使用逆向分析技术又具有一定难度,所以本文将直接使用selenium爬取京东商品数据.若不知道怎么安装和配置selenium,请点击查阅笔 ...
java 爬取京东商品详细信息价格
java 爬取京东商品详细信息价格获取图中的价格,因为京东的价格有反扒机制,价格通过js加载的,直接通过页面标签是获取不到的,所以我们要去要相应的js. http://p.3.cn/prices/ ...
Selenium实战之Python+Selenium爬取京东商品数据
实战目标:爬取京东商品信息,包括商品的标题.链接.价格.评价数量. 代码核心在于这几个部分: 其一:使用元素定位来获取页面上指定需要抓取的关键字: 其二:将页面上定位得到的数据永久存储到本地文件中. ...
python爬取京东商品数据要先登录_手把手教你用python 爬取京东评论
本次python实战,主要目标是利用 Python爬取京东商品评论数,如上图所示:爬取"Python之父"推荐的小蓝书,这些信息主要包括用户名.书名.评论等信息. 爬取的网址url ...
Python3爬取京东商品数据，解决赖加载问题
前言在这里我就不再一一介绍每个步骤的具体操作了,因为在上一次爬取今日头条数据的时候都已经讲的非常清楚了,所以在这里我只会在重点上讲述这个是这么实现的,如果想要看具体步骤请先去看我今日头条的文章内容, ...
python+正则表达式爬取京东商品数据信息
爬取数据的方式有很多种,正则表达式,scrapy,从接口中爬取动态网页的数据-今天我们讲的是用正则表达式来爬取京东一个大米商品的具体信息. 正则表达式的优点:可以精准的爬取我们想要的数据信息缺点:爬 ...
python爬取京东商品数据要先登录_3分钟教你不用python也能爬数据
想成为一名Data Scientist,需要掌握收集数据.利用Excel进行简单的描述性信息分析.利用机器学习神经网络等技术进行预测性及挖掘性数据分析.可视化呈现信息.最终得到商业Insights的能 ...
python爬取京东商品数据要先登录_京东商品评论情感分析|文本数据预处理
本文爬取了十款热销手机的文本评论数据后,首先对文本评论数据做文本预处理,文本评论数据中存在大量的无价值信息,引入无用的文本数据,对其作情感分析,显然没有价值,得出的文本挖掘结果也不尽人意.所以对于文本 ...
python爬取京东数据加载失败_Python爬取京东商品数据
对京东某一商品信息页面的HTML代码进行分析,可以发现它的图书产品信息页面都含有这样一段代码(不同类的商品页面有些不同): window.pageConfig={compatible:true,sea ...

Java爬取京东商品数据

爬取京东商品数据

准备工作

核心代码

爬笔记本数据的代码(只是更换了URL)

Java爬取京东商品数据相关推荐

最新文章

热门文章