By default Requests will perform location redirection for all verbs except HEAD.We can use the history property of the Response object to track redirection.The Response.history list contains the Response objects that were created in order to complete the request. The list is sorted from the oldest to the most recent response.For example, GitHub redirects all HTTP requests to HTTPS:>>> r = requests.get('http://github.com')>>> r.url
'https://github.com/'>>> r.status_code
200>>> r.history
[<Response [301]>]
If you're using GET, OPTIONS, POST, PUT, PATCH or DELETE, you can disable redirection handling with the allow_redirects parameter:>>> r = requests.get('http://github.com', allow_redirects=False)>>> r.status_code
301>>> r.history
[]
If you're using HEAD, you can enable redirection as well:>>> r = requests.head('http://github.com', allow_redirects=True)>>> r.url
'https://github.com/'>>> r.history
[<Response [301]>]

# redirect and history
'''
commit: Sign in
utf8: ✓
authenticity_token: login页面中获取
login: kermitjam
password: kermit46709394
'webauthn-support': 'supported'
'''
import requests
import re# 一 访问login获取authenticity_token
login_url = 'https://github.com/login'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36','Referer': 'https://github.com/'
}
login_res = requests.get(login_url, headers=headers)
# print(login_res.text)
authenticity_token = re.findall('name="authenticity_token" value="(.*?)"', login_res.text, re.S)[0]
print(authenticity_token)
login_cookies = login_res.cookies.get_dict()# 二 携带token在请求体内往session发送POST请求
session_url = 'https://github.com/session'
session_headers = {'Referer': 'https://github.com/login','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
}
form_data = {"commit": "Sign in","utf8": "✓","authenticity_token": authenticity_token,"login": "tankjam","password": "kermit46709394",'webauthn-support': 'supported'
}# 三 开始测试history
# 测试1
session_res = requests.post(session_url,data=form_data,cookies=login_cookies,headers=session_headers,# allow_redirects=False
)
print(session_res.url)  # 跳转后的地址>https://github.com/
print(session_res.history)  # 返回的是一个列表，里面有跳转前的response对象
print(session_res.history[0].text)  # 跳转前的response.text# 测试2
session_res = requests.post(session_url,data=form_data,cookies=login_cookies,headers=session_headers,allow_redirects=False
)
print(session_res.status_code)  # 302
print(session_res.url)  # 取消跳转，返回的是当前url
print(session_res.history)  # 返回一个空列表

#证书验证(大部分网站都是https)
import requests
# 如果是ssl请求,首先检查证书是否合法,不合法则报错,程序终端
response = requests.get('https://www.xiaohuar.com')
print(response.status_code)# 改进1:去掉报错,但是会报警告
import requests
response = requests.get('https://www.xiaohuar.com', verify=False)
# 不验证证书,报警告,返回200
print(response.status_code)# 改进2:去掉报错,并且去掉警报信息
import requests
import urllib3
urllib3.disable_warnings()  # 关闭警告
response = requests.get('https://www.xiaohuar.com', verify=False)
print(response.status_code)# 改进3:加上证书
# 很多网站都是https,但是不用证书也可以访问,大多数情况都是可以携带也可以不携带证书
# 知乎\百度等都是可带可不带
# 有硬性要求的,则必须带，比如对于定向的用户,拿到证书后才有权限访问某个特定网站
import requests
import urllib3
# urllib3.disable_warnings()  # 关闭警告
response = requests.get('https://www.xiaohuar.com',# verify=False,cert=('/path/server.crt', '/path/key'))
print(response.status_code)

# 超时设置
# 两种超时:float or tuple
# timeout=0.1  # 代表接收数据的超时时间
# timeout=(0.1,0.2)  # 0.1代表链接超时  0.2代表接收数据的超时时间import requestsresponse = requests.get('https://www.baidu.com',timeout=0.0001)

import requests
import re# response = requests.get("https://movie.douban.com/top250")
# print(response)
# print(response.text)# 获取url
def get_html(url):response = requests.get(url)return response# 获取正则后的标签 re.S是匹配所有的字符包括空格
def parser_html(response):# 使用re正则匹配出图片地址、评分、评论人数res = re.findall(r'<div class="item">.*?<a href=(.*?)>.*?<span class="title">(.*?)</span>.*?<span class="rating_num".*?>(.*?)</span>.*?<span>(\d+)人评价',response.text, re.S)print(res)return res# 文本模式实现数据持久化
def store(ret):with open("douban.text", "a", encoding="utf8") as f:for item in ret:f.write(" ".join(item) + "\n")# 爬虫三部曲
def do_spider(url):# 1.爬取资源# "https://movie.douban.com/top250"response = get_html(url)# 2.解析资源ret = parser_html(response)# 3.数据持久化store(ret)# 启动爬虫函数
def main():import timec = time.time()# 初始页数count = 0for i in range(10):# 豆瓣电影网爬取数据规律url = "https://movie.douban.com/top250?start=%s&filter=" % countdo_spider(url)count += 25print(time.time() - c)if __name__ == '__main__':main()

'''
绿皮书影评:影评接口第一页:https://movie.douban.com/subject/27060077/comments?status=Phttps://movie.douban.com/subject/27060077/comments?start=0&limit=20&sort=new_score&status=P&comments_only=1第二页:https://movie.douban.com/subject/27060077/comments?start=20&limit=20&sort=new_score&status=P&comments_only=1第二十五页:https://movie.douban.com/subject/27060077/comments?start=480&limit=20&sort=new_score&status=P&comments_only=1注意:1.登录前豆瓣影评提供10页2.登录过后影评豆瓣提供25页
''''''登录前'''
import requests
import reHEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
}def get_page(url):response = requests.get(url, headers=HEADERS)movie_json = response.json()# print(movie_json)data = movie_json.get('html')print(data)return datadef parse_data(data):# 正则匹配# 用户id,用户名,用户主页,用户头像,用户评价,评价时间,影评res = re.findall('<div class="comment-item" data-cid="(.*?)">.*?<a title="(.*?)" href="(.*?)">.*?src="(.*?)".*?<span class="allstar50 rating" title="(.*?)">.*?<span class="comment-time " title="(.*?)">.*?<span class="short">(.*?)</span>', data, re.S)print(res)if __name__ == '__main__':base_url = 'https://movie.douban.com/subject/27060077/comments?start={}&limit=20&sort=new_score&status=P&comments_only=1'num = 0for line in range(10):url = base_url.format(num)num += 20# print(url)data = get_page(url)parse_data(data)