python +selenium+phantomjs 登录爬取新浪微博动态js页面

登录新浪微博

最近新浪微博好烦，都取消不了验证码这个难搞得东西，而且跳来跳去，一改版以前的代码就都不能用了。目前整理的资料有三种方法：

 1. 设Cookie：简单粗暴，免去了模拟登录的好多麻烦，只是要定期更新2. 模拟登录：验证码是个大麻烦，有把验证码图片截下来，本地识别控制台输入验证3. 扫码登录 ：用app的扫码登录

其实除了设cookie ，在有验证码下，后两种方法差不多，都要人工干预。设cookie就不再说了。下面给出后两种方法的代码，代码来自GitHub - xchaoinfo/fuck-login: 模拟登录一些知名的网站，为了方便爬取需要登录的网站

模拟登录

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
Required
- requests (必须)
- rsa (必须)
- pillow (可选)
Info
- author : "xchaoinfo"
- email  : "xchaoinfo@qq.com"
- date   : "2016.3.7"
'''
import time
import base64
import rsa
import binascii
import requests
import re
import random
try:from PIL import Image
except:pass
try:from urllib.parse import quote_plus
except:from urllib import quote_plus'''
如果没有开启登录保护，不用输入验证码就可以登录
如果开启登录保护，需要输入验证码'''# 构造 Request headers
agent = 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0'
headers = {'User-Agent': agent
}session = requests.session()# 访问 初始页面带上 cookie
index_url = "http://weibo.com/login.php"
try:session.get(index_url, headers=headers, timeout=2)
except:session.get(index_url, headers=headers)
try:input = raw_input
except:passdef get_su(username):"""对 email 地址和手机号码 先 javascript 中 encodeURIComponent对应 Python 3 中的是 urllib.parse.quote_plus然后在 base64 加密后decode"""username_quote = quote_plus(username)username_base64 = base64.b64encode(username_quote.encode("utf-8"))return username_base64.decode("utf-8")# 预登陆获得 servertime, nonce, pubkey, rsakv
def get_server_data(su):pre_url = "http://login.sina.com.cn/sso/prelogin.php?entry=weibo&callback=sinaSSOController.preloginCallBack&su="pre_url = pre_url + su + "&rsakt=mod&checkpin=1&client=ssologin.js(v1.4.18)&_="pre_url = pre_url + str(int(time.time() * 1000))pre_data_res = session.get(pre_url, headers=headers)sever_data = eval(pre_data_res.content.decode("utf-8").replace("sinaSSOController.preloginCallBack", ''))return sever_data# print(sever_data)def get_password(password, servertime, nonce, pubkey):rsaPublickey = int(pubkey, 16)key = rsa.PublicKey(rsaPublickey, 65537)  # 创建公钥message = str(servertime) + '\t' + str(nonce) + '\n' + str(password)  # 拼接明文js加密文件中得到message = message.encode("utf-8")passwd = rsa.encrypt(message, key)  # 加密passwd = binascii.b2a_hex(passwd)  # 将加密信息转换为16进制。return passwddef get_cha(pcid):cha_url = "http://login.sina.com.cn/cgi/pin.php?r="cha_url = cha_url + str(int(random.random() * 100000000)) + "&s=0&p="cha_url = cha_url + pcidcha_page = session.get(cha_url, headers=headers)with open("cha.jpg", 'wb') as f:f.write(cha_page.content)f.close()try:im = Image.open("cha.jpg")im.show()im.close()except:print(u"请到当前目录下，找到验证码后输入")def login(username, password):# su 是加密后的用户名su = get_su(username)sever_data = get_server_data(su)servertime = sever_data["servertime"]nonce = sever_data['nonce']rsakv = sever_data["rsakv"]pubkey = sever_data["pubkey"]showpin = sever_data["showpin"]password_secret = get_password(password, servertime, nonce, pubkey)postdata = {'entry': 'weibo','gateway': '1','from': '','savestate': '7','useticket': '1','pagerefer': "http://login.sina.com.cn/sso/logout.php?entry=miniblog&r=http%3A%2F%2Fweibo.com%2Flogout.php%3Fbackurl",'vsnf': '1','su': su,'service': 'miniblog','servertime': servertime,'nonce': nonce,'pwencode': 'rsa2','rsakv': rsakv,'sp': password_secret,'sr': '1366*768','encoding': 'UTF-8','prelt': '115','url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack','returntype': 'META'}login_url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.18)'if showpin == 0:login_page = session.post(login_url, data=postdata, headers=headers)else:pcid = sever_data["pcid"]get_cha(pcid)postdata['door'] = input(u"请输入验证码")login_page = session.post(login_url, data=postdata, headers=headers)login_loop = (login_page.content.decode("GBK"))# print(login_loop)pa = r'location\.replace\([\'"](.*?)[\'"]\)'loop_url = re.findall(pa, login_loop)[0]# print(loop_url)# 此出还可以加上一个是否登录成功的判断，下次改进的时候写上login_index = session.get(loop_url, headers=headers)uuid = login_index.textuuid_pa = r'"uniqueid":"(.*?)"'print uuiduuid_res = re.findall(uuid_pa, uuid, re.S)[0]web_weibo_url = "http://weibo.com/%s/profile?topnav=1&wvr=6&is_all=1" % uuid_resweibo_page = session.get(web_weibo_url, headers=headers)weibo_pa = r'<title>(.*?)</title>'# print(weibo_page.content.decode("utf-8"))userID = re.findall(weibo_pa, weibo_page.content.decode("utf-8", 'ignore'), re.S)[0]print(u"欢迎你 %s, 你在正在使用 xchaoinfo 写的模拟登录微博" % userID)if __name__ == "__main__":# username = input(u'用户名：')# password = input(u'密码：')username = "*****"password = "*****"login(username, password)

扫码

#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
在python2.7中测试通过
Required
- requests (必须)
- pillow (可选)
Info
- author : "fangc"
- email  : "swjfc22@163.com"
- date   : "2016.3.16"
'''
import requests
import cookielib
import time
import re
import systry:from PIL import Image
except:pass
import threading# 构造 Request headers
agent = 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0'
headers = {'User-Agent': agent
}session = requests.session()
session.cookies = cookielib.LWPCookieJar('weibo_cookies.txt')# 访问 初始页面带上 cookie
index_url = "http://weibo.com/"
try:session.get(index_url, headers=headers, timeout=2)
except:session.get(index_url, headers=headers)def open_img(image_name):"""打开图片:param image_name: 图片的路径:return:"""im = Image.open(image_name)im.show()im.close()def login():"""登录主函数:return:"""image_name, qrcode_qrid = get_qrcode()try:# TODO(@fangc):用此方法打开图片不会退出，可以直接命令行用open打开，粗暴简单.thread = threading.Thread(target=open_img, name="open", args=(image_name,))thread.start()print(u"请用手机微博扫描二维码"u"微博二维码扫描在主页右上角!")except:print(u"请到当前目录下，打开二维码后用手机微博扫描二维码"u"微博二维码扫描在主页右上角!")# 下面判断是否已经扫描了二维码statu = 0while not statu:qrcode_check_page = scan_qrcode(qrcode_qrid, str(long(time.time() * 10000)))if "50114002" in qrcode_check_page:statu = 1print(u"---成功扫描，请在手机点击确认以登录---")time.sleep(2)# 下面判断是否已经点击登录,并获取alt的内容while statu:qrcode_click_page = scan_qrcode(qrcode_qrid, str(long(time.time() * 100000)))if "succ" in qrcode_click_page:# 登录成功后显示的是如下内容,需要获取到alt的内容# {"retcode":20000000,"msg":"succ","data":{"alt":"ALT-MTgxODQ3MTYyMQ==-sdfsfsdfsdfsfsdf-39A12129240435A0D"}}statu = 0alt = re.search(r'"alt":"(?P<alt>[\w\-\=]*)"', qrcode_click_page).group("alt")print(u"---登录成功---")time.sleep(2)# 下面是登录请求获取登录的跨域请求params = {"entry": "weibo","returntype": "TEXT","crossdomain": 1,"cdult": 3,"domain": "weibo.com","alt": alt,"savestate": 30,"callback": "STK_" + str(long(time.time() * 100000))}login_url_list = "http://login.sina.com.cn/sso/login.php"login_list_page = session.get(login_url_list, params=params, headers=headers)# 返回的数据如下所示，需要提取出4个url# STK_145809336258600({"retcode":"0","uid":"1111111","nick":"*****@sina.cn","crossDomainUrlList":# ["http:***************","http:\/\***************","http:\/\/***************","http:\/\/***************"]});url_list = [i.replace("\/", "/") for i in login_list_page.content.split('"') if "http" in i]for i in url_list:session.get(i, headers=headers)time.sleep(0.5)session.cookies.save(ignore_discard=True, ignore_expires=True)print(u"欢迎你, 你在正在使用 fangc 写的模拟登录微博")is_login()def get_qrcode():"""获取二维码图片以及二维码编号:return: qrcode_image, qrcode_qrid"""qrcode_before = "http://login.sina.com.cn/sso/qrcode/image?entry=weibo&size=180&callback=STK_" + str(long(time.time() * 10000))qrcode_before_page = session.get(qrcode_before, headers=headers)if qrcode_before_page.status_code != 200:sys.exit(u"可能微博改了接口!请联系作者修改")qrcode_before_data = qrcode_before_page.contentqrcode_image = re.search(r'"image":"(?P<image>.*?)"', qrcode_before_data).group("image").replace("\/", "/")qrcode_qrid = re.search(r'"qrid":"(?P<qrid>[\w\-]*)"', qrcode_before_data).group("qrid")cha_page = session.get(qrcode_image, headers=headers)image_name = u"cha." + cha_page.headers['content-type'].split("/")[1]with open(image_name, 'wb') as f:f.write(cha_page.content)f.close()return image_name, qrcode_qriddef scan_qrcode(qrcode_qrid, _time):"""判断是否扫码等需要:param qrcode_qrid::return: html"""params = {"entry": "weibo","qrid": qrcode_qrid,"callback": "STK_" + _time}qrcode_check = "http://login.sina.com.cn/sso/qrcode/check"return session.get(qrcode_check, params=params, headers=headers).contentdef is_login():"""判断是否登录成功:return: 登录成功返回True，失败返回False"""try:session.cookies.load(ignore_discard=True, ignore_expires=True)except:print(u"没有检测到cookie文件")return Falseurl = "http://weibo.com/"my_page = session.get(url, headers=headers)if "我的首页" in my_page.content:return Trueelse:return Falseif __name__ == '__main__':login()url = "http://weibo.com/zhangfeng0729?from=friends_dynamic&refer_flag=0000011002_&is_hot=1"page = session.get(url, headers=headers)# print session.cookiesprint page.content

完成了上述工作只是能访问页面而已，但是获取的页面是js动态生成的，什么都搞不到！好伤！
比如 http://weibo.com/zhangfeng0729?from=friends_dynamic&refer_flag=0000011002_&is_hot=1
请求会得到

什么鬼，关键部分什么都没有，再看看浏览器解析的

PhantomJS

基础的爬虫只能爬取单纯的html代码，但是有很多页面是js渲染的，html源码里面神马都没有。这是就需要借助于像浏览器一样渲染js页面的工具了。

PhantomJS是一个无界面的,可脚本编程的WebKit浏览器引擎。它原生支持多种web 标准：DOM 操作，CSS选择器，JSON，Canvas 以及SVG。

下载地址：http://phantomjs.org/download.html

该部分参考
1. 网易新闻评论爬虫python+selenium+PhantomJS
2. Setting timeout on selenium webdriver.PhantomJS

默认方式会加载整个页面，包括css ,js，图片，链接等，上面参考2 设置等待时间，不加载图片，启用缓存等，会让加载速度加快

不说了，直接上代码

# coding=utf-8import requests
from selenium import webdriver
import timeurl = "http://weibo.com/zhangfeng0729?from=friends_dynamic&refer_flag=0000011002_&is_hot=1"# webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.Cookie'] = 'SINAGLOBAL=3955422793326.2764.1451802953297; wb_publish_vip_1888964862=2; YF-Page-G0=9a31b867b34a0b4839fa27a4ab6ec79f; _s_tentry=123.sogou.com; Apache=3233757158741.355.1460597853009; ULV=1460597853022:23:7:4:3233757158741.355.1460597853009:1460533608651; YF-V5-G0=24e0459613d3bbdec61239bc81c89e13; YF-Ugrow-G0=3a02f95fa8b3c9dc73c74bc9f2ca4fc6; login_sid_t=9c9ff740ffffbdf23415b871aa319ec0; TC-Ugrow-G0=e66b2e50a7e7f417f6cc12eec600f517; TC-V5-G0=7e5b74ea4beaaa98b5f592db11c2eeb9; myuid=5898063885; TC-Page-G0=1e758cd0025b6b0d876f76c087f85f2c; wvr=6; SSOLoginState=1460604791; un=1654916845@qq.com; user_active=201604141220; user_unver=b37741d33507619aa07b6d512be0dd67; SUS=SID-5898063885-1460616073-XD-a0vlz-c7f218016d290c52b4297371d3942ce7; SUE=es%3D5ec8e6a39470b9d75e4cf930977a7449%26ev%3Dv1%26es2%3D2b500477fcc5dc1560b4aee4d5ca9be5%26rs0%3Daeb6LPcKR9mfwMB31yJquO6NSXgRd9BpSLAcKEdCQjtUOM7pIzDblRVIOOItI22Uc6pLjTxUc6k7s0zIgIxar9JlVdPjOiHEGijE2ucUP6GjFv%252BJWntWR7szF3qcWknPBJzemeHiThBGRF0bAcNFHcH%252BY9VfrTaIVjXCUsO93B8%253D%26rv%3D0; SUP=cv%3D1%26bt%3D1460616073%26et%3D1460702473%26d%3Dc909%26i%3D2ce7%26us%3D1%26vf%3D0%26vt%3D0%26ac%3D27%26st%3D0%26uid%3D5898063885%26name%3D1654916845%2540qq.com%26nick%3Dforfun2016%26fmp%3D%26lcp%3D; SUB=_2A256C0vZDeTxGeNG4loR9i3EwzmIHXVZYToRrDV8PUJbvNAPLXX2kW9LHet_3CJqcXTqa57ddP1X0MxCpd6bSA..; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhFdfTB9.lOPTcSpGQskYri5JpX5K-t; SUHB=0z1QUpglvHBm_i; ALF=1492143611; UOR=far.tooold.cn,widget.weibo.com,login.sina.com.cn; WBtopGlobal_register_version=ab9111fb56d70a2b'#这个地方可以设置 header, 代理等一些参数
cap = webdriver.DesiredCapabilities.PHANTOMJS
cap["phantomjs.page.settings.resourceTimeout"] = 1000
cap["phantomjs.page.settings.loadImages"] = False
cap["phantomjs.page.settings.disk-cache"] = True
cap["phantomjs.page.customHeaders.Cookie"] = 'SINAGLOBAL=3955422793326.2764.1451802953297; ' #我删掉了一大部分# browser = webdriver.PhantomJS(desired_capabilities=cap)driver = webdriver.PhantomJS(executable_path="G:\\code\\python\\phantomjs-2.1.1\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe", desired_capabilities=cap)
driver.set_window_size(1120, 1000000)
driver.get(url)# if diver.find_element_by_link_text("下一页"):
#    pass
# else:
#     time.sleep(1)contents = driver.find_elements_by_class_name("WB_cardwrap")
for content in contents:print content.text

输出结果为

691
关注
37193
粉丝
3305
微博
添加粉丝群
微关系
共同关注(1)
微博客服
查看更多 a
文章 [9]

十大银行员工薪酬调查：平均年薪最低22万也算寒冬？

据说看过这篇文章的同学都考上了银行
查看更多 a
相册
查看更多a
银行校园招聘张峰的粉丝也关注
银行招聘网官博
银行招聘网(yinhangzhaopin.com)官方微博
+关注
银行招聘资讯...
微信公众账号：yinhangqiuzhi 汇集银行招聘信息，全方位解读银行招聘笔试面试历年真题、考查知识点、复习重点，为你敲开银行大门。
+关注
应届生求职网
应届生求职网YingJieSheng.COM官方微博，网址：http://www.yingjiesheng.com
+关注
更多 a
微盘文件分享
2014年银行校园招聘答疑
分享者：银行校园招聘张峰
浏览量:715
分享
银行面试到底是什么？
分享者：银行校园招聘张峰
浏览量:1201
分享
2013年中国农业银行校园招聘备考专用
分享者：银行校园招聘张峰
浏览量:6274
分享
更多 a
赞
##后面的省略
...

解析是解析出来了，但是还是存在一个大问题

新浪微博在浏览器中也是，他不会一次性给你加载完一页，等到你将页面滑到第才给你加载完，好烦啊！这样模拟点下一页也搞不成！！

废了好长时间，还没找到解决方法，麻烦知道的给我留言通知声，万分感谢！！！

先这样吧，之后再慢慢了解phantomjs 的详情，搞一些高级的用法