爬虫采集全国工商系统的数据（外接打码平台）

javascript的代码用的是java的javascript引擎，用python的jpype去调用java的对象。因为试了好多python的js库，PYv8装起来太麻烦。而且对js的eval函数支持不是很好，后面就用了java 的js引擎。

java的话打包成jar或者class文件，java用的是1.8版本

#coding:UTF-8
import json
import re
import threading
import timeimport jpype
import redis
import requests
from bs4 import BeautifulSoup
from jpype import *jpype.startJVM(jpype.getDefaultJVMPath(), "-ea", "-Djava.class.path=/code/java/forpython/target/classes/")class SearchItem(threading.Thread):session=requests.session()keyword=""proxy=""semaphore=Nonedef getGTChallenge(self):print "getGTChallenge start"loginurl="http://www.gsxt.gov.cn/SearchItemCaptcha"result=self.session.get(loginurl)if "y.replace(" not in result.text:raise Exception("被屏蔽了")mycookies= result.cookiesjpype.attachThreadToJVM()jpype.isThreadAttachedToJVM()A = jpype.JClass("com.GovTest")self.Aobj=A()fu=self.Aobj.challenge(result.text)print "fu="+fujslarr= fu.split("=")jsl_clearance=jslarr[1]self.session.cookies['__jsl_clearance']=jsl_clearanceresult=self.session.get(loginurl)challengeJson=json.loads(result.text)return  challengeJsondef getImageGif(self):print "getImageGif start"url="http://www.gsxt.gov.cn/corp-query-custom-geetest-image.gif?v="localTime=time.localtime(time.time())url=url+str(localTime.tm_min+localTime.tm_sec)resp=self.session.get(url)aaa=self.Aobj.getImageGif(resp.text)matchObj = re.search( 'location_info = (\d+);', aaa)if matchObj:return matchObj.group(1)else:Exception("没有找到location_info")def getValidateInput(self,location_info):print "getValidateInput start"url="http://www.gsxt.gov.cn/corp-query-geetest-validate-input.html?token="+location_inforesp=self.session.get(url)aaa=self.Aobj.getImageGif(resp.text)matchObj = re.search( 'value: (\d+)}', aaa)if matchObj:location_info= matchObj.group(1)token=int(location_info) ^ 536870911;print "token=",tokenreturn str(token)else:Exception("没有找到location_info")def searchTest(self,keyword):print "searchTest start"url="http://www.gsxt.gov.cn/corp-query-search-test.html?searchword="+keywordresp=self.session.get(url);print "searchTest ",resp.textdef jianYan(self,challengeJson):print "jianYan start"url="http://jiyanapi.c2567.com/shibie?user=你的账号&pass=你的密码&gt="+challengeJson["gt"]+"&challenge="+challengeJson["challenge"]+"&referer=http://www.gsxt.gov.cn&return=json&format=utf8"sess=requests.session()resp=sess.get(url);jiyanJson=  json.loads(resp.text)print resp.textreturn jiyanJsondef querySearch(self,jiYanJson,token,keyword):print "querySearch start"url="http://www.gsxt.gov.cn/corp-query-search-1.html"postData={'tab':'ent_tab','province':'','geetest_challenge':jiYanJson['challenge'],'geetest_validate':jiYanJson['validate'],'geetest_seccode':jiYanJson['validate']+'|jordan','token':token,'searchword':keyword}resp=self.session.post(url,postData)return resp.text ,postDatadef dealPageUrl(self,html):print "dealPageUrl start"soup = BeautifulSoup(html,"html.parser")urlsItem=soup.find_all("a",class_="search_list_item db")pageNums=0for urlItem in urlsItem:print "urlItem['href']=",urlItem['href']if len(urlsItem)>1:pageForm=soup.find_all(id="pageForm")tabAs=pageForm[0].find_all("a",text=re.compile("\d+"))pageNums=len(tabAs)return pageNumsdef dealPageUrlNum(self,pageNums,postData):print "dealPageUrlNum start"url="http://www.gsxt.gov.cn/corp-query-search-advancetest.html"for i in range(pageNums):postData['page']=i+1resp=self.session.get(url,params=postData)soup = BeautifulSoup(resp.text)urlsItem=soup.find_all("a",class_="search_list_item db")for urlItem in urlsItem:print "urlItem['href']=",urlItem['href']def getCorpUrl(self):self.session.timeout=1self.session.max_redirects=1if self.proxy:self.session.proxies={ "http": "http://"+self.proxy, "https": "http://"+self.proxy, }headers={'Host': 'www.gsxt.gov.cn','User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2','Accept-Encoding': 'gzip, deflate','Referer': 'http://www.gsxt.gov.cn/SearchItemCaptcha','Connection': 'keep-alive','Upgrade-Insecure-Requests': '1','Cache-Control': 'max-age=0, no-cache'}self.session.headers=headerschallengeJson=self.getGTChallenge()localtion_info= self.getImageGif()token=self.getValidateInput(localtion_info)self.searchTest(self.keyword)jiyanJson=self.jianYan(challengeJson)html,postData=self.querySearch(jiyanJson,token,self.keyword)pageNums=self.dealPageUrl(html)print 'pageNums=',pageNumsself.dealPageUrlNum(pageNums,postData)return 1def run(self):try:self.getCorpUrl()except Exception,e:print "run exception ",e.messageself.session.close()self.semaphore.release()print "search Item run finish"def __init__(self, keyword,proxy,semaphore):threading.Thread.__init__(self)self.keyword = keywordself.proxy = proxyself.semaphore = semaphoresemaphore=threading.Semaphore(1)
while 1:try:semaphore.acquire()t1=SearchItem("百度",None,semaphore)t1.start()except Exception, e:print 'main e.message:\t', e.messagetime.sleep(1)

java代码执行js引擎的代码：

package com;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;public class GovTest {private ScriptEngine scriptEngine;public GovTest() {ScriptEngineManager scriptEngineManager = new ScriptEngineManager();this.scriptEngine = scriptEngineManager.getEngineByName("JavaScript");}public String challenge(String resp){resp = resp.substring(8);String tmp[] = resp.split("</script");resp = tmp[0];resp = resp.replace("eval(y.replace", "var aaa=(y.replace");resp = resp + "aaa=aaa.replace("h=document.createElement('div');","");aaa=aaa.replace("h.innerHTML='<a href=\\\'/\\\'","");\n" +"aaa=aaa.replace(">x</a>';","");aaa=aaa.replace("h=h.firstChild.href;","h='http://www.gsxt.gov.cn/';");aaa=aaa.replace("while(window._phantom||window.__phantomas){};","");bbb=aaa.split("setTimeout");\n" +"    aaa=bbb[0]+"return dc;}}";\n" +"    aaa=aaa.replace("var l=","{fa:");\n" +"  var ffa=eval("("+aaa+")");\n" +"    var fffa=ffa.fa();";System.out.println(resp);String script = resp;try {scriptEngine.eval(script);} catch (ScriptException e) {return e.getMessage();}String bbb = (String) scriptEngine.get("fffa");System.out.println(bbb);return bbb;}public String getImageGif(String resp){String script="function dd(){var json="+resp+";return json.map( function(item){ return String.fromCharCode(item);}).join('');}" +"var ggg=dd();";try {scriptEngine.eval(script);} catch (ScriptException e) {return e.getMessage();}String bbb = (String) scriptEngine.get("ggg");return bbb;}public static void main(String[] s){new GovTest().challenge("<script>var x="while@div@substr@setTimeout@26@window@9@5@03@location@cookie@String@l@dc@37@GMT@a@href@if@0@length@__jsl_clearance@4@toLowerCase@_phantom@var@f@challenge@1517192797@reverse@match@085@join@3@cd@Mon@catch@fromCharCode@charAt@firstChild@Path@createElement@document@Expires@29@innerHTML@addEventListener@__phantomas@i@eval@captcha@h@replace@x@https@Jan@for@try@r@2@function@18@return@1500@e@false@DOMContentLoaded@else@attachEvent@onreadystatechange".replace(/@*$/,"").split("@"),y="1a d=3d(){1(6.19||6.30){};1a 23,e='16=1d.20|14|';1a 1b=[3d(36){3f 32('c.26('+36+')')},(3d(){1a 34=2b.2a('2');34.2e='<11 12=\\\'/\\\'>36</11>';34=34.28.12;1a 3b=34.1f(/37?:\\\\/\\\\//)[14];34=34.3(3b.15).18();3f 3d(36){39(1a 31=14;31<36.15;31++){36[31]=34.27(36[31])};3f 36.21('')}})()];23=[[[3c+8]+[-~-~(+[])],(22+[]+[[]][~~[]])+[3c+8],(8+[[]][~~[]])+[(+[])],[3c+8]+[(+[])],[-~-~(+[])-~[([(-~(+[])<<-~(+[]))]+~~![]>>(-~(+[])<<-~(+[])))]]+[(+[])],(8+[[]][~~[]])+[-~[(-~(+[])<<-~(+[]))]-~[(-~(+[])<<-~(+[]))]],(8+[[]][~~[]])+[3c+8],[3c+8]+[-~[(-~(+[])<<-~(+[]))]-~[(-~(+[])<<-~(+[]))]],(22+[]+[[]][~~[]])+[3c+8],(8+[[]][~~[]])+[(+[])],[-~[(-~(+[])<<-~(+[]))]-~[(-~(+[])<<-~(+[]))]]+[-~[(-~(+[])<<-~(+[]))]-~[(-~(+[])<<-~(+[]))]],[3c+8]+[(+[])],[-~[(-~(+[])<<-~(+[]))]-~[(-~(+[])<<-~(+[]))]]+(8+[[]][~~[]]),(-~(+[])+[[]][~~[]])+[(+[])]+[17],[-~-~(+[])-~[([(-~(+[])<<-~(+[]))]+~~![]>>(-~(+[])<<-~(+[])))]]+(8+[[]][~~[]]),(7+[])+[(+[])]],[[(+[])]],[(-~(+[])+[[]][~~[]])+[(+[])]+[-~-~(+[])],(8+[[]][~~[]])+[(+[])],[-~-~(+[])-~[([(-~(+[])<<-~(+[]))]+~~![]>>(-~(+[])<<-~(+[])))]]+(22+[]+[[]][~~[]])],[[3c+8]],[[-~-~(+[])-~[([(-~(+[])<<-~(+[]))]+~~![]>>(-~(+[])<<-~(+[])))]]+[-~-~(+[])]],[[(+[])],[17],[17]],[[-~[(-~(+[])<<-~(+[]))]-~[(-~(+[])<<-~(+[]))]]+[-~-~(+[])-~[([(-~(+[])<<-~(+[]))]+~~![]>>(-~(+[])<<-~(+[])))]],[-~-~(+[])-~[([(-~(+[])<<-~(+[]))]+~~![]>>(-~(+[])<<-~(+[])))]]+[17],(-~(+[])+[[]][~~[]])+(-~(+[])+[[]][~~[]])+[17],(-~(+[])+[[]][~~[]])+[(+[])]+(7+[]),[-~-~(+[])-~[([(-~(+[])<<-~(+[]))]+~~![]>>(-~(+[])<<-~(+[])))]]+[-~-~(+[])-~[([(-~(+[])<<-~(+[]))]+~~![]>>(-~(+[])<<-~(+[])))]],[-~[(-~(+[])<<-~(+[]))]-~[(-~(+[])<<-~(+[]))]]+(8+[[]][~~[]]),(22+[]+[[]][~~[]])+[3c+8],(8+[[]][~~[]])+(-~(+[])+[[]][~~[]]),[-~[(-~(+[])<<-~(+[]))]-~[(-~(+[])<<-~(+[]))]]+[-~-~(+[])-~[([(-~(+[])<<-~(+[]))]+~~![]>>(-~(+[])<<-~(+[])))]]]];39(1a 31=14;31<23.15;31++){23[31]=1b.1e()[(-~(+[])+[[]][~~[]])](23[31])};23=23.21('');e+=23;4('a.12=a.12.35(/[\\\\?|&]33-1c/,\\\'\\\')',40);2b.b=(e+';2c=24, 2d-38-3e 9:5:f 10;29=/;');};13((3d(){3a{3f !!6.2f;}25(41){3f 42;}})()){2b.2f('43',d,42);}44{2b.45('46',d);}",z=0,f=function(x,y){var a=0,b=0,c=0;x=x.split("");y=y||99;while((a=x.shift())&&(b=a.charCodeAt(0)-77.5))c=(Math.abs(b)<13?(b+48.5):parseInt(a,36))+y*c;return c},g=y.match(/\\b\\w+\\b/g).sort(function(x,y){return f(x)-f(y)}).pop();while(f(g,++z)-x.length){};eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));</script>");}
}

注：本文仅供参考学习，请勿做其它非法用途！

爬虫采集全国工商系统的数据（外接打码平台）相关推荐

[.NET/C#]快速采集全国城市商圈数据
声明:此文章及所涉及的程序代码仅供学习与交流,使用者如有违反相关法律的行为将负法律责任. [商圈]在O2O时代是个常用的数据,但数据在网上难以寻找,或遇数据公司的收费障碍.对于开发人员来说,比较好的方 ...
爬虫实战—拿下最全租房数据 | 附源码
「关注我,和我一起放下灵魂,让灵魂去搬砖.」作者:小一介绍:放不下灵魂的搬砖者全文共6329字,阅读全文需17分钟点赞再看,养成好习惯 Python版本3.8.0,开发工具:Pycharm 写 ...
正方教务系统验证码自动识别(非打码平台)
本人用的是网上的Java版本,经过本人的改良只剩下验证码识别. 详细说明请看 http://www.unique-liu.com/181.html ImagePreProcess.srcPath = ...
58同城被爆简历数据泄露：700元的恶意爬虫软件可采集全国简历数据
雷锋网(公众号:雷锋网)消息,据<21世纪经济报道>记者调查发现,近日,有多个淘宝卖家廉价批发"58同城简历数据":"一次购买2万份以上,3毛一条:10万以上 ...
芝麻换ip软件：大数据爬虫采集的应用流程
数字化时代,大数据信息的采集和应用逐渐普及.这离不开网络网络爬虫的广泛应用.网络爬虫是比较简单且容易上手的项目.1个网络爬虫工程师就可以轻松完成1个简单的小网络爬虫.但是面对现在对数据信息要求如此之大 ...
数据采集时总提示未登录_做电商必须学会这一招！教你用爬虫工具免费采集网易考拉商品数据...
本文主要介绍如何使用后羿采集器的智能模式,免费采集网易考拉商品的价格.累计评价.商品图片等信息. 采集工具简介: 后羿采集器是一款基于人工智能技术的网络爬虫软件,只需要输入网址就能够自动识别网页数据, ...
【毕业设计_课程设计】基于网络爬虫的新闻采集和订阅系统的设计与实现（源码+论文）
文章目录 0 项目说明 1 项目说明 2 系统需求 3 系统架构 4 效果展示 5 论文目录 6 项目工程 0 项目说明基于网络爬虫的新闻采集和订阅系统的设计与实现提示:适合用于课程设计或毕业设计 ...
以企查查为例详解如何用爬虫采集企业信息及电话邮箱数据
近年来"大数据"."人工智能"这些词汇深受热捧,但是其实很多人还搞不懂什么是大数据,更别提知道大数据的用处了. 那到底大数据是什么呢?实际上,所谓大数据,就是算 ...
Python采集全国各地百度地图上店铺POI数据（母婴、美食等）
Python采集全国各地百度地图上店铺POI数据 1. 注册百度地图开放平台账号先注册百度地图开发平台账号,创建应用,获得AK 2. 通过Python批量检索不同城市店铺POI数据 2.1 大致思路 ...

爬虫采集全国工商系统的数据（外接打码平台）

爬虫采集全国工商系统的数据（外接打码平台）相关推荐

最新文章

热门文章