python 爬虫 requests学习案例

文章目录

requests登录淘宝并访问搜索商品页面

requests登录淘宝并访问搜索商品页面

参考：
猪哥66
零度风格
python格式化

import os
import requests
import re
from bs4 import BeautifulSoup
import time
import json
import csv
import hashlib
import codecs
def getMD5(s) :md5 = hashlib.md5()md5.update(s.encode('utf-8'))return md5.hexdigest()
def bytesToStr(bs) :# s = "%E7%AC%94%E8%AE%B0%E6%9C%AC" #笔记本# s = s.replace('%', '\\x')# '\xE7\xAC\x94\xE8\xAE\xB0\xE6\x9C\xAC'# ss = codecs.escape_decode(s, 'hex-escape')[0]# print(ss.decode('utf-8'))bs = bs.replace('%', '\\x')bss = codecs.escape_decode(bs, 'hex-escape')[0]bss = bss.decode('utf-8')return bss
reqS = requests.Session()
COOKIES_FILE_PATH = 'D:\\taobao_login_cookies.txt'
class TBLogin:"""须知：开始前要自己先打开F12，登录淘宝https://login.taobao.com/member/login.jhtml，获取login_data的一些关键参数，例如：loginId,password2,ua,_csrf_token,umidToken,hsiz。这些参数可能会该改变。功能：参考别人博客的思路是，首先判断是否需要滑块验证，然后验证密码，验证成功的话，返回的页面内容大致如下：{"content":{"data":{"redirect":true,"redirectUrl":"https://i.taobao.com/my_taobao.htm?nekot=xxx=xxx","asyncUrls":["https://passport.alibaba.com/mini_apply_st.js?callback=callback&site=0&token=xxx"],"resultCode":100},"status":0,"success":true},"hasError":false}asyncUrls是申请st码的地址然后通过申请st码的地址来获取st码，最后再使用st码登录。我个人感觉貌似只需要验证密码返回成功，就已经算登录成功了，因为验证完密码，就重定向到自己主页了，至于为什么要获取st码并再登录一次，不太清楚。留坑：能登录成功的关键在于ua，有合法的基于用户身份的ua登录才不用滑块验证。一般输入完账户还没输入密码的时候就会发出一个post请求，带有ua，而且ua是动态的！这个ua是在登录页面的console通过命令：window["_n"] or window[UA_Opt.LogVal]获取。@property什么用参考：[猪哥66](https://blog.csdn.net/u014044812/article/details/99584382)从代码层面将模拟登录淘宝分为以下四个步骤：输入用户名后，浏览器会向淘宝（taobao.com）发起一个post的请求，判断是否出现滑块验证！用户输入密码后，浏览器向淘宝（taobao.com）又发起一个post请求，验证用户名密码是否正确，如果正确则返回一个token。浏览器拿着token去阿里巴巴（alibaba.com）交换st码！浏览器获取st码之后，拿着st码获取cookies，登录成功https://www.cnblogs.com/542684416-qq/p/11427600.htmlhttps://blog.csdn.net/weixin_41624982/article/details/86710995https://mp.weixin.qq.com/s?__biz=MzI2OTQ1NzEyMQ==&mid=2247483923&idx=1&sn=b54293568b411695dac564a6433c2016&chksm=eae1419ddd96c88bb653a1cef4b7cdbeb0ff5b983c173b4e3730f01b0dfec28d391a43c45b76&scene=21#wechat_redirect"""def __init__(self, loginId, password2, ua, _csrf_token, umidToken, hsiz) :"""账号登录对象:param loginId: 用户名:param password2: 加密后的密码:param ua: 淘宝的ua参数:param _csrf_token: _csrf_token:param umidToken: 新版登录新增参数:param hsiz: hsiz"""# 检测是否需要验证码的URLself.user_check_url = "https://login.taobao.com/member/request_nick_check.do?_input_charset=utf-8"# 'https://login.taobao.com/newlogin/account/check.do?appName=taobao&fromSite=0'# 验证淘宝用户名密码URLself.verify_password_url = "https://login.taobao.com/newlogin/login.do?appName=taobao&fromSite=0"# 申请st码地址self.apply_st_url = "https://passport.alibaba.com/mini_apply_st.js?callback=callback&site=0&token=xxx"# 使用st码登录的URLself.vst_url = 'https://login.taobao.com/member/vst.htm?st={}'# 淘宝个人主页URLself.my_taobao_url = 'http://i.taobao.com/my_taobao.htm'self.stma = ""self.loginId = loginIdself.password2 = password2self.ua = uaself._csrf_token = _csrf_tokenself.umidToken = umidTokenself.hsiz = hsizdef user_check_vcode(self) :"""检测账号是否需要验证码:return:"""print("调用账户是否需要滑动验证码接口(True:需要/False:不需要)")user_check_result = Trueuser_check_url = self.user_check_urldata = {"username": self.loginId,'ua': self.ua}try:user_check_response = reqS.post(user_check_url, data=data)user_check_result = user_check_response.json()["needcode"]print("返回结果为: %s" % user_check_result)except Exception as e:print("用户验证接口请求失败，msg:{}".format(e))user_check_result = Trueprint("需要滑动验证，搞不定！休息10秒再来一次试试")return user_check_resultdef get_umidToken(self) :"""获取umidToken参数:return:"""response = reqS.get('https://login.taobao.com/member/login.jhtml')# 登录网址umidToken_match = re.search(r'"umidToken":"(.*?)"', response.text)print("umidToken: ", umidToken_match.group(1))return umidToken_match.group(1)def verify_password(self) :"""验证账户密码是否正确:return:"""verify_ans = Falselogin_headers = {'Connection': 'keep-alive','Cache-Control': 'max-age=0','Content-Type': 'application/x-www-form-urlencoded',"origin": "https://login.taobao.com","referer": "https://login.taobao.com/member/login.jhtml","user-agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36',}login_data = {'loginId': self.loginId,'password2': self.password2,'keepLogin':'false','ua': self.ua,'umidGetStatusVal':' 255','screenPixel':' 1536x864','navlanguage':' zh-CN',"navUserAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.146 Safari/537.36",'navPlatform':' Win32','appName':' taobao','appEntrance':' taobao_pc','_csrf_token': self._csrf_token,'umidToken': self.umidToken,'hsiz': self.hsiz,'style':' default','appkey':' 00000000','from':' tb','isMobile':' false','lang':' zh_CN','returnUrl':' http://i.taobao.com/my_taobao.htm','fromSite':' 0','bx-ua': 'xxx','bx-umidtoken': 'xxx'}try:r = reqS.post(self.verify_password_url, headers = login_headers, data = login_data, timeout = 3)cookieJar = reqS.cookiescookieDict = requests.utils.dict_from_cookiejar(cookieJar)print(r.status_code, r.request.url)# 输出cookieJar和cookieDictprint(cookieJar)print(cookieDict)r.raise_for_status()r.encoding = r.apparent_encodingprint("验证密码返回内容：")print(r.text)self.apply_st_url = r.json()['content']['data']['asyncUrls'][0]if self.apply_st_url:verify_ans = Trueprint("验证密码并获取申请st码地址成功。")else:print("未获取到申请st码地址。")except :verify_ans = Falseprint("验证密码并获取申请st码地址出现bug。")return verify_ansdef apply_st(self) :"""获取st码:return:"""ast_flag = Falsetry:r = reqS.get(self.apply_st_url)r.raise_for_status()st_match = re.search(r'"data":{"st":"(.*?)"}', r.text)self.stma = st_match.group(1)if self.stma:ast_flag = Trueprint('获取st码成功，st码：{}'.format(st_match.group(1)))else :raise RuntimeError('获取st码失败')except Exception as e:print('申请st码请求失败!')ast_flag = Falsereturn ast_flagdef login_with_stma(self) :"""首先判断是否需要滑块验证，然后验证密码，然后获取st码，再使用st码登录:return:"""login_result = Falseucv_r = self.user_check_vcode()if ucv_r :return Falsevp_r = self.verify_password()if not vp_r :return Falseas_r = self.apply_st()if not as_r :return Falselogin_headers = {'Connection': 'keep-alive','Cache-Control': 'max-age=0','Content-Type': 'application/x-www-form-urlencoded',"origin": "https://login.taobao.com","referer": "https://login.taobao.com/member/login.jhtml","user-agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36',}try:url = self.vst_url.format(self.stma)print('vst login url:')print(url)response = reqS.get(self.vst_url.format(self.stma), headers = login_headers, timeout = 3)response.raise_for_status()print("response.text:")print(response.text)print('--end--')my_taobao_match = re.search(r'top.location.href = "(.*?)"', response.text)print('登录淘宝成功，跳转链接：{}'.format(my_taobao_match.group(1)))self.my_taobao_url = my_taobao_match.group(1)login_result = Trueself.serialization_cookies()except Exception as e:print('登录淘宝失败，{}.'.format(e))login_result = Falsereturn login_resultdef serialization_cookies(self):"""序列化cookies并保持cookieDict:return:"""cookies_dict = requests.utils.dict_from_cookiejar(reqS.cookies)print('cookie 如下：')print(reqS.cookies)print(cookies_dict)with open(COOKIES_FILE_PATH, 'w+', encoding='utf-8') as file:json.dump(cookies_dict, file)print('保存cookies文件成功！')def deserialization_cookies(self) :"""反序列化cookies:return:"""cookies = ""with open(COOKIES_FILE_PATH, 'r+', encoding='utf-8') as file:cookies_dict = json.load(file)cookies = requests.utils.cookiejar_from_dict(cookies_dict)return cookiesdef load_cookies(self):"""从COOKIES_FILE_PATH加载之前保存的cookieDict内容:return:"""if not os.path.exists(COOKIES_FILE_PATH) :return FalsereqS.cookies = self.deserialization_cookies()try:self.get_taobao_nick_name()except Exception as e:os.remove(COOKIES_FILE_PATH)print('cookies过期，删除cookies文件！')return Falseprint('加载淘宝cookies登录成功!!!')return Truedef get_taobao_nick_name(self) :"""通过requests.Session()尝试获取淘宝昵称。最主要目的是判断Session中是否有合法的cookieJar。:return: 淘宝昵称"""headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'}try:response = reqS.get(self.my_taobao_url, headers=headers)response.raise_for_status()nick_name_match = re.search(r'<input id="mtb-nickname" type="hidden" value="(.*?)"/>', response.text)if nick_name_match:print('登录淘宝成功，你的用户名是：{}'.format(nick_name_match.group(1)))return nick_name_match.group(1)else:raise RuntimeError('获取淘宝昵称失败！')except Exception as e:print('获取淘宝昵称失败！')return '获取淘宝昵称失败！'def testSessionCookie(self) :"""dic是上面登录淘宝后得到的cookieJar转换成的cookieDict本函数测试通过上面得到的cookie能否访问淘宝商品页面:return:"""dic = {}with open(COOKIES_FILE_PATH, 'r+', encoding='utf-8') as file:dic = json.load(file)cookieStr = ''for key in dic:cookieStr = cookieStr + key + '=' + dic[key] + ';'print(cookieStr)print("")headers = {"user-agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36',"referer": "https://s.taobao.com/",'cookie': cookieStr}testurl = "https://s.taobao.com/search?q=笔记本"try:r = requests.get(testurl, headers = headers, timeout = 3)print(r.status_code, r.encoding, r.apparent_encoding)print(r.request.url, r.url)r.raise_for_status()r.encoding = r.apparent_encodingprint(r.text)except :print("test fail")def test(self) :"""测试:return:"""gu = self.get_umidToken()print(gu)# self.login_with_stma()nick_name = self.get_taobao_nick_name()
def main() :'''自己的参数'''tbl = TBLogin(loginId, password2, ua, _csrf_token, umidToken, hsiz)tbl.test()# with open(COOKIES_FILE_PATH, 'r+', encoding='utf-8') as file:#     dic = json.load(file)#     print("dic:")#     print(dic)
main()

python 爬虫 requests学习案例相关推荐

python爬虫requests简单案例_Python小例子：一个最简单的爬虫
学习编程的过程中总是有些枯燥.成天面对黑乎乎的窗口可能真的有些无聊.来,我们做点好玩的事情吧.比如将网络上你喜欢的图片保存下来.不过千里之行始于足下,在学习将网页上的图片下载下来之前.我们首先要学 ...
python爬虫requests简单案例_python网络爬虫（三）requests库的13个控制访问参数及简单案例...
酱酱~小编又来啦~
day19 学习python爬虫——requests和bs4
day19 学习python爬虫--requests和bs4 一.requests使用详解 import requests 1.发送请求 requests.get(请求地址) - 直接获取请求地址对应 ...
python自学免费课堂-推荐5个Python爬虫免费学习资源（B站高赞！）
这里是法学院毕业的Python程序员阿峰. 写在前面: 和学任何技能一样,学习Python爬虫也需要两点:行动和时间. 所有的爬虫速成课,我都不推荐. 如果您愿意付出时间和行动,每天在Python上进 ...
已解决（Python爬虫requests库报错请求异常SSL错误，证书认证失败问题）requests.exceptions.SSLError: HTTPSConnectionPool
成功解决(Python爬虫requests库报错请求异常,SSL错误,证书认证失败问题)requests.exceptions.SSLError: HTTPSConnectionPool(host= ...
Python爬虫入门学习实战项目（一）
静态数据的采集第一个项目我们来抓取拉勾网的招聘信息,话不多说直接开始吧! 1.首先我们导入相关库: import requests from lxml import etree import pan ...
Python爬虫入门学习例子之煎蛋网（福利哦）
初学Python爬虫,学习资料是这篇博客:Python爬虫入门教程经过前边两个例子(糗事百科爬虫和百度贴吧爬虫)的学习,我自己写了一个练习的例子:爬煎蛋网的妹子图,福利哦(*^__^*) #煎蛋网爬 ...
Python爬虫_音乐案例
Python爬虫_音乐案例 [案例目的]:下载音乐 [第三方库]:1.requests 2.perttytable [开发环境]:1.Python3.8 2.PyCharm 2022.1 # http ...
已解决（Python爬虫requests报错）requests.exceptions.ProxyError: HTTPSConnectionPool
成功解决(Python爬虫requests报错):requests.exceptions.ProxyError: HTTPSConnectionPool 文章目录报错信息报错翻译报错原因解决方 ...

python 爬虫 requests学习案例

文章目录

requests登录淘宝并访问搜索商品页面

python 爬虫 requests学习案例相关推荐

最新文章

热门文章