百度解释

HttpClient 是Apache Jakarta Common 下的子项目，可以用来提供高效的、最新的、功能丰富的支持 HTTP 协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本和建议。

httpClient专为java发送http请求而生的, 如果要httpClient ,需要先进行导包

        <dependency><groupId>org.apache.httpcomponents</groupId><artifactId>httpclient</artifactId><version>4.5.4</version></dependency>

Get


import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;import java.io.IOException;public class HttpClientGet {public static void main(String[] args) throws IOException {//请求地址String toUrl="http://www.manmanbuy.com";//封装地址到get请求中HttpGet httpGet=new HttpGet(toUrl);//设置请求头信息：User-AgenthttpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36");//创建执行请求对象CloseableHttpClient closeableHttpClient = HttpClients.createDefault();//执行请求,获取响应对象CloseableHttpResponse closeableHttpResponse = closeableHttpClient.execute(httpGet);//获取http请求响应信息StatusLine statusLine = closeableHttpResponse.getStatusLine();//获取状态码int respCode=statusLine.getStatusCode();System.out.println("respCode:"+respCode);//获取响应的Content typeHeader[] headersType = closeableHttpResponse.getHeaders("Content-Type");System.out.println(headersType.length);System.out.println(headersType[0].getName()+"----"+headersType[0].getValue());//获取响应数据,并封装到实体对象HttpEntity httpEntity = closeableHttpResponse.getEntity();//从实体类中取出爬取的数据String  pageMessage = EntityUtils.toString(httpEntity, "utf-8");//System.out.println(pageMessage);}
}

结果展示：

Post


import org.apache.http.Consts;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
/*** post请求登录慢慢买网站*/
public class HttpClientPost {public static void main(String[] args) throws IOException {//登录请求地址String toLogin="http://home.manmanbuy.com/login.aspx";//封装请求HttpPost httpPost=new HttpPost(toLogin);//设置请求头信息，模拟浏览器登录//httpPost.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36");httpPost.setHeader("Referer","http://home.manmanbuy.com/login.aspx?tourl=http%3a%2f%2fwww.manmanbuy.com%2fdefault.aspx");//httpPost.setHeader("Content-Type","application/x-www-form-urlencoded");httpPost.setHeader("Cookie","_ga=GA1.2.1458535014.1567059656; _gid=GA1.2.1328820533.1570633933; Hm_lvt_85f48cee3e51cd48eaba80781b243db3=1570634150,1570709948,1570717507,1570756038; ASP.NET_SessionId=rjnpykzbfs1yyvmdrtclz455; Hm_lpvt_85f48cee3e51cd48eaba80781b243db3=1570763535; _gat_gtag_UA_145348783_1=1");httpPost.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");//封装表单数据BasicNameValuePair txtName= new BasicNameValuePair("txtUser", "13699268827");BasicNameValuePair txtPass= new BasicNameValuePair("txtPass", "manmanmai");BasicNameValuePair autoLogin= new BasicNameValuePair("autoLogin", "on");BasicNameValuePair btnLogin= new BasicNameValuePair("btnLogin", "登陆");BasicNameValuePair viewState= new BasicNameValuePair("__VIEWSTATE", "/wEPDwULLTIwNjQ3Mzk2NDFkGAEFHl9fQ29udHJvbHNSZXF1aXJlUG9zdEJhY2tLZXlfXxYBBQlhdXRvTG9naW4voj01ABewCkGpFHsMsZvOn9mEZg==");BasicNameValuePair eventValidation= new BasicNameValuePair("__EVENTVALIDATION", "/wEWBQLW+t7HAwLB2tiHDgLKw6LdBQKWuuO2AgKC3IeGDJ4BlQgowBQGYQvtxzS54yrOdnbC");// 表单数据封装到集合中List<NameValuePair> formParams = new ArrayList<NameValuePair>();formParams.add(txtName);formParams.add(txtPass);formParams.add(autoLogin);formParams.add(btnLogin);formParams.add(viewState);formParams.add(eventValidation);//把表单集合对象封装到表单实体中UrlEncodedFormEntity urlEncodedFormEntity=new UrlEncodedFormEntity(formParams, Consts.UTF_8);//把表单实体封装到post请求中httpPost.setEntity(urlEncodedFormEntity);//*******************************************************//创建请求对象CloseableHttpClient closeableHttpClient = HttpClients.createDefault();//用户登录成功之后的响应对象CloseableHttpResponse closeableHttpResponse = closeableHttpClient.execute(httpPost);int code=closeableHttpResponse.getStatusLine().getStatusCode();if(code==302){//状态码为302意味着登录成功，并且服务器端做了重定向跳转//通过响应对象获取locationHeader[] locations = closeableHttpResponse.getHeaders("Location");String  respUrl=locations[0].getValue();//爬取用户中心页面String toUserCenterPage="http://home.manmanbuy.com"+respUrl;//用户中心get请求HttpGet httpGet=new HttpGet(toUserCenterPage);//--httpGet.setHeader("Referer","http://www.manmanbuy.com/default.aspx");//--//获取第一次登录成功后的cookieHeader[] headerCookies = closeableHttpResponse.getHeaders("Set-Cookie");System.out.println("headerCookies:"+headerCookies.length);//System.out.println(headerCookies[0].getName()+"----"+headerCookies[0].getValue());//获取第一请求响应后的cookie，并把其cookie值赋予第二个请求httpGet.setHeader("Cookie",headerCookies[0].getValue());//Httpclient对象closeableHttpClient = HttpClients.createDefault();//执行请求 user center pageCloseableHttpResponse closeableHttpResponseUserCenter = closeableHttpClient.execute(httpGet);//获取页面信息 user center pageHttpEntity userCenterEntity = closeableHttpResponseUserCenter.getEntity();//String userCenterPage = EntityUtils.toString(userCenterEntity, "gb2312");//把用户中心的页面转换成document对象Document userCenterDocument = Jsoup.parse(userCenterPage);Elements integrals = userCenterDocument.select("#aspnetForm > div.udivright > div:nth-child(2) > table > tbody > tr > td:nth-child(1) > table:nth-child(2) > tbody > tr > td:nth-child(2) > div:nth-child(1) > font");System.out.println(integrals.text());}}
}

结果展示：

总结

总结: httpClient使用步骤:

导包
确定首页URL
创建httpClient对象: HttpClients.createDefault();
设置请求方式: HttpGet HttpPost

设置请求参数和请求头

post设置参数:

```java List list = new ArrayList();

      list.add(new BasicNameValuePair("username","rose"));
      list.add(new BasicNameValuePair("password","123"));
      HttpEntity entity = new UrlEncodedFormEntity(list);
      httpPost.setEntity(entity);

```

发送请求, 获取响应对象CloseableHttpResponse response = httpClient.execute(get/post)
获取数据:
响应行响应头响应体

爬虫 spider05——使用httpclient发送get请求、post请求相关推荐

爬虫之requests模块发送带参数的请求
爬虫之requests模块发送带参数的请求我们在使用百度搜索的时候经常发现url地址中会有一个 ?,那么该问号后边的就是请求参数,又叫做查询字符串 1.1 在url携带参数直接对含有参数的url发 ...
爬虫之requests模块发送带header的请求
爬虫之requests模块发送带header的请求我们先写一个获取百度首页的代码 import requestsurl = 'https://www.baidu.com'response = req ...
Java实现HttpClient发送GET、POST请求（https、http）
原文链接:https://www.cnblogs.com/klslb/p/9121276.html 1.maven <dependency><groupId>org.apach ...
Android使用Apache HttpClient发送GET、POST请求
2019独角兽企业重金招聘Python工程师标准>>> 简单的网页下载,HttpURLConnection可以完成,但是涉及到用户登录等权限相关问题,就需要涉及Session.Coo ...
HttpClient 发送 HTTP、HTTPS 请求的简单封装
序近期这几周.一直在忙同一个项目.刚開始是了解需求.需求有一定了解之后,就開始调第三方的接口.因为第三方给提供的文档非常模糊,在调接口的时候,出了非常多问题,一直在沟通协调,详细的无奈就不说了,因为 ...
HttpClient发送put与post请求代码实例与解析
/** * 实例化HttpClient * maxTotal 最大连接数 * maxPerRoute 最大并发量 * socketTimeout 从服务器 ...
HttpClient发送POST和GET请求
1.使用HttpClient添加的maven依赖(尽量版本平齐或者比我的高,低的可能会出现找不到类的情况) <dependency><groupId>org.apache.ht ...
httpclient工具类，post请求发送json字符串参数,中文乱码处理
在使用httpclient发送post请求的时候,接收端中文乱码问题解决. 正文: 我们都知道,一般情况下使用post请求是不会出现中文乱码的.可是在使用httpclient发送post请求报文含中文 ...
爬虫之requests模块发送post请求
爬虫之requests模块发送post请求思考:哪些地方我们会用到POST请求? 登录注册( 在web工程师看来POST 比 GET 更安全,url地址中不会暴露用户的账号密码等信息) 需要传输大文 ...

爬虫 spider05——使用httpclient发送get请求、post请求

Get

Post

爬虫 spider05——使用httpclient发送get请求、post请求相关推荐

最新文章

热门文章