爬虫裁判文书网爬取part2

继续上篇：

receive_data.py

import time
import requestsfrom method.save_data import RedisPara
from method.wenshu_method import ParseJs, GenerateFormRequest, ParseDetail, GenerateRequest, Parse_Indexjs
from static.docid import getkey, decode_docidclass ReceiveData(object):def __init__(self):self.temp_count = 0self.page = 0self.wzws_cid = ''self.proxy = ''self.User_Agent = ''self.proxies = {}self.vl5x = ''self.guid = ''self.vjkl5 = ''self.docid = ''self.param = '案件类型:刑事案件,文书类型:判决书'  # 请求列表页的查询条件self.p = ''  # 用来判断参数是否存在# 需要保存的参数self.sync = 0self.title = ''self.court = ''self.pdate = ''self.writ = ''self.reason = ''self.docid = ''def get_cookies(self):# 调用这个方法刷新cookie参数r = RedisPara()cookies = r.get_para()if cookies:self.wzws_cid = cookies.get('wzws_cid')self.proxy = cookies.get('pr')self.User_Agent = cookies.get('ua')self.proxies = {"http": "http://{}".format(self.proxy)}self.vjkl5 = cookies.get('vjkl5')context = ParseJs()self.vl5x, self.guid = context.get_key_para(self.vjkl5)return cookiesdef listpage_2(self):cookies = self.get_cookies()if not cookies:return Nonegen_obj = GenerateFormRequest(self.param, self.page, self.vl5x, self.guid, self.proxy, self.User_Agent,self.wzws_cid, self.vjkl5)try:resp = gen_obj.get_request()except Exception as e:resp =Noneprint('错误5',e)return respdef parse_listpage_2(self):resp = self.listpage_2()if not resp:return None# 提取json中的datalistcontext = ParseDetail(resp.text)datalist = context.parse_list_data()RunEval = datalist[0]['RunEval']key = getkey(RunEval).encode('utf-8')for case in datalist[1:]:parse_obj = ParseDetail(case)self.title, self.court, self.pdate, self.writ, self.reason, Docid = parse_obj.parse_items()# 获得解密后的docidtry:self.docid = decode_docid(Docid, key)  # 这里有问题，可能会找不到_KEYprint('解密后的docid',self.docid)except Exception as e:print('错误6',e)time.sleep(4)# 详情页# -----------------------------------第一次cid-----------------------------------url1 = 'http://wenshu.court.gov.cn/content/content?DocID={}&KeyWord='.format(self.docid)  # 所以拿到了docid之后，就可以直接发起这个请求了，以后可能还可以再拆req_obj = GenerateRequest(url1, self.User_Agent, self.docid, self.proxies)try:resp = req_obj.get_detail_request()except Exception as e:print('错误7', e)print('编号为{}的页面处理失败'.format(self.docid))print('*' * 100)continue  # 如果失败的话，跳过以下步骤，进下个循环self.wzws_cid = requests.utils.dict_from_cookiejar(resp.cookies).get('wzws_cid')# -----------------------------------第二次cid-----------------------------------context = Parse_Indexjs()wzwstemplate, wzwschallenge, dynamicurl = context.get_context_para(resp.text)url = 'http://wenshu.court.gov.cn' + dynamicurlcookies = {"wzws_cid": self.wzws_cid,"wzwstemplate": wzwstemplate,"wzwschallenge": wzwschallenge}req_obj = GenerateRequest(url, self.User_Agent, self.docid, self.proxies, cookies)try:resp = req_obj.get_detail_request()except Exception as e:print('错误8',e)continueself.wzws_cid = requests.utils.dict_from_cookiejar(resp.cookies)["wzws_cid"]# -------------------------------第三次请求详情页--------------------------------detail_url2 = 'http://wenshu.court.gov.cn/CreateContentJS/CreateContentJS.aspx?DocID=' + self.docidcookies = {"wzws_cid": self.wzws_cid,"vl5x": self.vl5x,}req_obj = GenerateRequest(detail_url2, self.User_Agent, self.docid, self.proxies, cookies)try:resp = req_obj.get_detail_request()except Exception as e:print('错误9', e)continue# 提取文章数据context = ParseDetail(resp.text)html_raw = context.parse_detail()# 保存数据item = {}item['content'] = html_rawprint(item)print('当前第{}条'.format(self.temp_count), '*' * 100)self.temp_count += 1a = ReceiveData()
a.parse_listpage_2()

method.py

import json
import re
import js2py
import requestsclass ParseJs(object):'''解析js获得cookie参数return : vl5x, guid'''def __init__(self):self.context = js2py.EvalJs()with open('/var/workspace/mandy/spider/Spider/src/weapon/wenshu_js/getKey.js', 'r') as f:self.js_content = f.read()with open('/var/workspace/mandy/spider/Spider/src/weapon/wenshu_js/md5.js', 'r') as f:self.js_md5 = f.read()with open('/var/workspace/mandy/spider/Spider/src/weapon/wenshu_js/sha1.js', 'r') as f:self.js_sha1 = f.read()with open('/var/workspace/mandy/spider/Spider/src/weapon/wenshu_js/base64.js', 'r') as f:self.js_base64 = f.read()with open('/var/workspace/mandy/spider/Spider/src/weapon/wenshu_js/createGuid.js', 'r') as f:self.js_guid = f.read()def get_key_para(self, vjkl5):self.context.vjkl5 = vjkl5self.context.execute(self.js_md5)self.context.execute(self.js_sha1)self.context.execute(self.js_base64)self.context.execute(self.js_content)self.context.execute(self.js_guid)return (self.context.result, self.context.guid)class Parse_Indexjs(object):def get_context_para(self,resp_text):# 抓取响应中的js代码raw_func = re.findall(r'<script type="text/javascript">(.*)</script>', resp_text, re.DOTALL)[0]# print(raw_func)# 在js中获得wzwstemplate和wzwschallengesub_text = '''aaa=p;return "'zifuchuan'"'''courl_func = re.sub('return p', sub_text, raw_func)context = js2py.EvalJs()context.execute('var aaa')context.execute(courl_func)code = re.findall(r'(.*)function HXXTTKKLLPPP5', context.aaa)[0]context.execute(code)js = '''var wzwstemplate_result = KTKY2RBD9NHPBCIHV9ZMEQQDARSLVFDU(template.toString());var confirm = QWERTASDFGXYSF();var wzwschallenge_result = KTKY2RBD9NHPBCIHV9ZMEQQDARSLVFDU(confirm.toString());'''context.execute(js)wzwstemplate = context.wzwstemplate_result,wzwstemplate = wzwstemplate[0] # 上面返回一个元组，把数据取一下wzwschallenge = context.wzwschallenge_resultdynamicurl = context.dynamicurlreturn (wzwstemplate,wzwschallenge,dynamicurl)class ParseDetail(object):def __init__(self,resp_text):self.resp_text = resp_textself.context = js2py.EvalJs()passdef parse_detail(self):js_code = re.search(r'(var jsonHtmlData(.*?))var jsonData', self.resp_text, re.DOTALL).group(1)self.context.execute(js_code)self.context.execute('var jsonData = eval("(" + jsonHtmlData + ")")')self.context.execute('var jsonHtml = jsonData.Html.replace(/01lydyh01/g, "\'")')html_raw = self.context.jsonHtml  # 这个是个带格式的html文本return html_rawdef parse_list_data(self):# context = js2py.EvalJs()self.context.data = self.resp_textself.context.execute('datalist = eval(data)')datalist = json.loads(self.context.datalist)return datalistdef parse_items(self):case = self.resp_texttitle= case['案件名称']court = case['法院名称']pdate = case['裁判日期']if case['案号'] == '无':writ = ''else:writ = case['案号']try:reason = case['裁判要旨段原文']except Exception as e:print('未匹配出reason',e)reason = ''Docid = case['文书ID']return (title,court,pdate,writ,reason,Docid)class GenerateFormRequest(object):# 生成列表页请求def __init__(self,param,page,vl5x,guid,proxy,User_Agent,wzws_cid,vjkl5):self.url = "http://wenshu.court.gov.cn/List/ListContent"self.data = {"Param": "{}".format(param),  # 案件类型可以替换"Index": "{}".format(page),  # 这里是页数"Page": "10",  # 这个不变"Order": "法院层级","Direction": "asc","number": "wens","vl5x": vl5x,"guid": guid}self.proxy = proxyself.vl5x = vl5xself.User_Agent = User_Agentself.wzws_cid = wzws_cidself.vjkl5 = vjkl5self.proxies = {"http": "http://{}".format(self.proxy)}def get_request(self):resp = requests.get(url=self.url,headers={"User-Agent": self.User_Agent,"X-Requested-With": "XMLHttpRequest","Referer": "http://wenshu.court.gov.cn/List/List",},proxies=self.proxies,timeout=5,data=self.data)return respclass GenerateRequest(object):# 生成详情页request请求def __init__(self,url,User_Agent,docid,proxies,cookies=''):self.url = urlself.User_Agent = User_Agentself.docid = docidself.proxies = proxiesif cookies:self.cookies = cookieselse:self.cookies = {}passdef get_detail_request(self):resp = requests.get(url=self.url,headers={"User-Agent": self.User_Agent,"Referer": "http://wenshu.court.gov.cn/content/content?DocID={}&KeyWord=".format(self.docid)},cookies=self.cookies,proxies=self.proxies,timeout=5,allow_redirects=False,)return resp

爬虫裁判文书网爬取part2相关推荐

python 爬虫裁判文书网 19年4月更新版
完整版代码github地址:https://github.com/Monster2848/caipanwenshu 爬虫主体逻辑: import re import time import pymon ...
python爬取裁判文书并分析_裁判文书网爬虫攻克
最近因为公司需要爬取裁判文书网的某一类别文章数据,于是简单研究了一下,发现网站数据全是js加载的,于是想都没想直接用selenium尝试爬取,没想到竟然有selenium都抓取不到的js(刚毕业的py ...
Python爬虫Selenium手动接管Edge爬取裁判文书网“环境污染”数据（Mac环境）
目标数据:爬取从2007年到2022年,各地级市中级法院历年关于"环境污染"的裁判文书数量. 由于裁判文书网需要登录,Selenium手动接管爬取可避免频繁登录造成的封号风险. S ...
爬取裁判文书网(一)
1.裁判文书网地址 http://wenshu.court.gov.cn/ 2.网站分析 (1)网站类型是动态网站. (2)网页源代码中并没有我们想要的页面内容,且引用许多js代码. (3)查看列表页 ...
爬虫修正：裁判文书网20200901更新增加登录系统
爬虫修正:裁判文书网20200901更新增加登录系统一.登录系统的增加二.增加登录函数三.一个比较纠结的地方一.登录系统的增加导致需要登录,直接是无法进入了.以往可以直接进入具体的类别如现 ...
裁判文书网爬虫升级最新版本0.7 更新时间2020-12-17
接上一篇文件:裁判文书网爬虫最新更新2020-08-12 https://blog.csdn.net/myhot/article/details/108046389 本次升级主要已支持写入mys ...
2021-04-01裁判文书网数据python爬虫更新下载
长期持续更新数据 2020-11-08裁判文书网数据python爬虫更新下载添加链接描述截至3月已从数据库中下载1亿1200万条裁判文书数据,有需要数据的伙伴可以(｡･∀･)ﾉﾞ嗨前台QQ7900- ...
2020-11-08裁判文书网数据python爬虫更新下载
2020年9月1日更新文书网2020年8月31日晚更新之后就需要手机注册验证登录了,所以,账号是个问题. 分析 1. 登录比较简单哦,主要难题是多账号的登录及session的更新维护. 2.其它 ...
爬虫实战6：爬取英雄联盟官网五个位置的综合排行榜保存到excel
申明:资料来源于网络及书本,通过理解.实践.整理成学习笔记. 文章目录英雄联盟官网获取一个位置的综合排行榜所有数据(上单为例) 获取所有位置的综合排行榜所有数据英雄联盟官网获取一个位置的综合排 ...

爬虫裁判文书网爬取part2

爬虫裁判文书网爬取part2相关推荐

最新文章

热门文章

爬虫 裁判文书网爬取part2

爬虫 裁判文书网爬取part2相关推荐

最新文章

热门文章

爬虫裁判文书网爬取part2

爬虫裁判文书网爬取part2相关推荐