import time
import requests
#爬虫三部曲
#1.发送请求
def get_page(url):response = requests.get(url)return response
# 2.解析数据
import re
def parse_index(html):# findall匹配所有# re.findall('正则匹配规则','匹配文本','匹配模式')# re.S:对所有文本进行搜索匹配detail_urls = re.findall('<div class="items"><a class="imglink" href="(.*?)"',html,re.S)return detail_urls
# 解析详情页
def parse_detail(html):movie_url = re.findall('<source src="(.*?)">',html,re.S)#print(movie_url)if movie_url:return movie_url[0]
# 3.保存数据
import uuid
#uuid.uuid4()根据时间戳生成一段世界上唯一的字符串
def save_video(content):with open(f'{uuid.uuid4()}.mp4','wb')as f:f.write(content)print('视频下载完毕')
# main + 回车键
# 测试用例
if __name__ == '__main__':for line in range(6):url=f'http://www.xiaohuar.com/list-3-{line}.html'#发送请求response = requests.get(url)#print(response)# # 返回响应状态码# print(response.status_code)# # 返回响应文本# print(response.text)#解析主页页面detail_urls = parse_index(response.text)#循环遍历详情页urlfor detail_url in detail_urls:# print(detail_url)# 往每一个详情页发送请求detail_res = get_page(detail_url)#print(response.text)#解析详情页获取视频urlmovie_url = parse_detail(detail_res.text)# 判断视频url存在则打印if movie_url:print(movie_url)# 往视频url发送请求获取视频二进制流movie_res = get_page(movie_url)# 把视频的二进制流传给save_video函数保存到本地save_video(movie_res.content)

''''''
'''
POST请求自动登录github请求URL：https://github.com/session请求方式：POST请求头CookieUser-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36请求体commit:Sign inutf8:✓authenticity_token:cFSDQ5zTaUkkx13xtncPeJhxAnqGiKPoWzKBuTvFyapNXsVlD6MIR/7IBtaIjEOT3yOPiYYXCxAEkZwsnjTO4g==login:******password:****webauthn-support:unknown
'''
# 1.获取token随机字符串
'''
1.访问登录页面获取token字符串请求URL：https://github.com/session请求方式：GET请求头CookieUser-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36
2.解析并提取token字符串
#正则
<input type="hidden" name="authenticity_token" value="(.*?)" />
'''
import requests
import re
login_url ='https://github.com/login'
# login页面的请求头信息
login_header ={'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
}
login_res=requests.get(url=login_url,headers=login_header)
# print(login_res.text)
# 解析提取token字符串
authenticity_token = re.findall('<input type="hidden" name="authenticity_token" value="(.*?)" />',login_res.text,re.S
)[0]
print(authenticity_token)
# 获取login页面的cooies信息
# print(type(login_res.cookies))
# print(type(login_res.cookies.get_dict()))
login_cookies = login_res.cookies.get_dict()
# 2.开始登陆github
'''
POST请求自动登录github请求URL：https://github.com/session请求方式：POST请求头CookieUser-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36请求体commit:Sign inutf8:✓authenticity_token:cFSDQ5zTaUkkx13xtncPeJhxAnqGiKPoWzKBuTvFyapNXsVlD6MIR/7IBtaIjEOT3yOPiYYXCxAEkZwsnjTO4g==login:******password:**********webauthn-support:unknown
'''
# session登录url
session_url ='https://github.com/session'
# 请求头信息
session_headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
}
# 请求体信息
form_data ={
'commit':'Sign in','utf8':'✓','authenticity_token':authenticity_token,'login':'********','password':'********','webauthn-support':'unknown',
}
session_res = requests.post(url=session_url,headers=session_headers,cookies=login_cookies,data=form_data)
with open('github3.html','w',encoding='utf-8') as f:f.write(session_res.text)