scrapy ,Selenium 企查查模拟登录

middlewares.py文件

**#更换User-agent ,读取cookie**
import time
from selenium import webdriver
from scrapy.http import HtmlResponse, Headersfrom selenium.webdriver.chrome.options import Options
class UAMiddleware(object):user_agent_list = settings['UAPOOL']def process_request(self,request,spider):print("((((((((((()))))))))))))))))))))))))))))))))))))))))))))))))")if request.url == "https://www.qichacha.com/user_login?back=%2F":return HtmlResponse(url=request.url, encoding="utf-8")ua = random.choice(self.user_agent_list)with open('E:cookies.json', 'r', encoding='utf-8') as fp:# 读取login保存的cookies值listcookies = json.loads(fp.read())cookies_dict = '' # 通过构建字典类型的cookiesfor cookie in listcookies:sss=cookie['name']+"="+cookie['value']+";"cookies_dict=cookies_dict+sssprint("@@@@@@@@@@@@@@@@@@@@@@@@@@111111")print(cookies_dict)#cookies_dict2=json.dumps(cookies_dict)hhh = {'Host': 'www.qichacha.com','User-Agent': ua,"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3","Accept-Language": "zh-CN,zh;q=0.9","Accept-Encoding": "gzip, deflate, br","Cache-Control": "max-age= 0",'Cookie': cookies_dict}print("@@@@@@@@@@@@@@@@@@@@@@@@@@")request.headers = Headers(hhh)#return scrapy.http.HtmlResponse(url=request.url,  encoding='utf-8',request=request)****#模拟登录****
from PIL import Image,ImageEnhance
import scrapy
import json
import re
import random
import urllib.parseimport scrapy,time
import urllib.parse
from  openpyxl import  Workbook
from openpyxl  import load_workbook
from selenium import webdriver
wb = Workbook()
ws = wb.activeimport pytesseract
from selenium.webdriver import ActionChains
class JSPageMiddleware(object):#通过chrome 动态访问，滑块移动函数（未用）def get_strack(self):distance = 348track = []current = 0mid = distance * 3/ 5t = 0.2v = 0while current < distance:if current < mid:a = 3else:a = 6v0 = vv = v0 + a * tmove = v0 * t + 1 / 2 * a * t * tprint(move)print('----------------------------')current += moveprint(current)print('+++++++++++++++++++')track.append(round(move))print(track)return track**#模拟登录，保存cookie**def process_request(self, request, spider):#chrome_options = Options()# chrome_options.add_argument('--headless')  # 使用无头谷歌浏览器模式# chrome_options.add_argument('--disable-gpu')# chrome_options.add_argument('--no-sandbox')if request.url=="https://www.qichacha.com/user_login?back=%2F" :chrome_options = Options()# 指定谷歌浏览器路径self.browser = webdriver.Chrome(chrome_options=chrome_options, executable_path="F:chromedriver.exe")self.browser.get(request.url)# time.sleep(3)self.browser.find_element_by_xpath('//*[@id = "normalLogin"]').click()  # 转到登录界面self.browser.find_element_by_xpath(' // *[ @ id = "nameNormal"]').send_keys("13080024006")  # 账号self.browser.find_element_by_xpath('// *[ @ id = "pwdNormal"]').send_keys("13080024006")  # 密码time.sleep(3)button = self.browser.find_element_by_xpath('//*[@id="nc_1__scale_text"]/span')# 开始拖动 perform()用来执行ActionChains中存储的行为action = ActionChains(self.browser)  # 实例化一个action对象action.click_and_hold(button).perform()  # 鼠标左键按下不放action.reset_actions()  # 清除之前的action# print(distance)track = self.get_strack()ttt = [23, 81, 224]a = 0for i in ttt:action.move_by_offset(xoffset=i, yoffset=0).perform()  # 移动滑块action.reset_actions()time.sleep(0.4)a = a + iaction.release().perform()time.sleep(5)# 滑块，有刷新，有验证码# 取刷新,alert1=self.browser.find_element_by_class_name('nc-lang-cnt')alert = self.browser.find_element_by_class_name('nc-lang-cnt').textprint("AAAAAAAAAAAA")print(alert)print("AAAAAAAAAAAAAAAAAAAAAA")if alert == "哎呀，出错了，点击刷新再来一次":while 1:# 点击刷新while True:self.browser.find_element_by_xpath('//*[@id="dom_id_one"]/div/span/a').click()if button:  # 出现滑块，停止print("CCCCCCCCCCCCC")break# 滑块button = self.browser.find_element_by_xpath('//*[@id="nc_1__scale_text"]/span')action.click_and_hold(button).perform()  # 鼠标左键按下不放action.reset_actions()  # 清除之前的actionfor i in ttt:action.move_by_offset(xoffset=i, yoffset=0).perform()  # 移动滑块action.reset_actions()time.sleep(0.4)a = a + iaction.release().perform()time.sleep(5)alert = self.browser.find_element_by_class_name('nc-lang-cnt').textif alert != "哎呀，出错了，点击刷新再来一次":print("RRRRRRRRRRRRRRRRRRRRRR")breakif alert == "请在下方输入验证码":print("WWWWWWWWWWWWWWWWWWWWWWWWWWWWW")# 验证码while 1:self.browser.save_screenshot(r"E:aa.png")  # 截取登录页面# 定位验证码位置及大小location = self.browser.find_element_by_xpath('//*[@id="nc_1__imgCaptcha_img"]/img').location  # 获取验证码x,y轴坐标size = self.browser.find_element_by_xpath('//*[@id="nc_1__imgCaptcha_img"]/img').size  # 获取验证码的长宽coderange = (int(location['x']), int(location['y']), int(location['x'] + size['width']),int(location['y'] + size['height']))  # 计算验证码整体坐标img = Image.open(r"E:aa.png").crop(coderange)  # 打开截图, 使用Image的crop函数，从截图中再次截取我们需要的区域# 下面对图片做了一些处理，能更好识别一些，相关处理再百度看吧img = img.convert('RGBA')  # 转换模式：L | RGBimg = img.convert('L')  # 转换模式：L | RGBimg = ImageEnhance.Contrast(img)  # 增强对比度img = img.enhance(2.0)  # 增加饱和度img.save("E:bb.png")# 再次读取识别验证码img = Image.open("E:bb.png")time.sleep(3)text = pytesseract.image_to_string(img).strip()  # 使用image_to_string识别验证码# 打印识别的验证码print(text)# 识别出来验证码去特殊符号，用到了正则表达式，这是我第一次用，之前也没研究过，所以用的可能粗糙，请见谅b = ''for i in text:pattern = re.compile(r'[a-zA-Z0-9]')m = pattern.search(i)if m != None:b += i# 输出去特殊符号以后的验证码print(b)time.sleep(2)if b == '':print("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE")time.sleep(2)self.browser.find_element_by_xpath('//*[@id="nc_1__btn_1"]').click()  # 点击刷新验证码else:# 把b的值输入验证码输入框time.sleep(10)# spider.browser.find_element_by_xpath('//*[@id="nc_1_captcha_input"]').send_keys( b)  # 输入验证码self.browser.find_element_by_xpath(' // *[ @ id = "nc_1_scale_submit"] / span').click()  # 点击提交time.sleep(2)if self.browser.find_element_by_class_name('nc-lang-cnt').text == "验证通过":  # 出现验证通过，停止breakprint("SSSSSSSSSSSSSSSSSSSSSSSSSSSSSS")self.browser.find_element_by_xpath('//*[@id="user_login_normal"]/button').click()  # 点击登录time.sleep(5)print("TTTTTTTTTTTTT111111111111111111111111111111111")self.browser.find_element_by_xpath('//*[@id="bindwxModal"]/div/div/div/button/span[1]').click()  # 点×dictCookies = self.browser.get_cookies()print(dictCookies)jsonCookies = json.dumps(dictCookies)# 登录完成后,将cookies保存到本地文件with open("E:cookies.json", "w") as fp:fp.write(jsonCookies)print("TTTTTTTTTTTTTTTTTTT22222222222222222222")# 滑块，无刷新，验证码elif alert == '请在下方输入验证码':while 1:  # 验证码self.browser.save_screenshot(r"E:aa.png")  # 截取登录页面# 定位验证码位置及大小location = self.browser.find_element_by_xpath('//*[@id="nc_1__imgCaptcha_img"]/img').location  # 获取验证码x,y轴坐标size = self.browser.find_element_by_xpath('//*[@id="nc_1__imgCaptcha_img"]/img').size  # 获取验证码的长宽coderange = (int(location['x']), int(location['y']), int(location['x'] + size['width']),int(location['y'] + size['height']))  # 计算验证码整体坐标img = Image.open(r"E:aa.png").crop(coderange)  # 打开截图, 使用Image的crop函数，从截图中再次截取我们需要的区域# 下面对图片做了一些处理，能更好识别一些，相关处理再百度看吧img = img.convert('RGBA')  # 转换模式：L | RGBimg = img.convert('L')  # 转换模式：L | RGBimg = ImageEnhance.Contrast(img)  # 增强对比度img = img.enhance(1.0)  # 增加饱和度img.save("E:bb.png")# 再次读取识别验证码img = Image.open("E:bb.png")time.sleep(3)text = pytesseract.image_to_string(img).strip()  # 使用image_to_string识别验证码# 打印识别的验证码print(text)# 识别出来验证码去特殊符号，用到了正则表达式，这是我第一次用，之前也没研究过，所以用的可能粗糙，请见谅b = ''for i in text:pattern = re.compile(r'[a-zA-Z0-9]')m = pattern.search(i)if m != None:b += i# 输出去特殊符号以后的验证码print("QQQQQQQQQQQQQQQQQQQQQ")print(b)print("QQQQQQQQQQQQQQQQQQQQQ")time.sleep(2)if b == '':self.browser.find_element_by_xpath('//*[@id="nc_1__btn_1"]').click()  # 点击刷新验证码else:# 把b的值输入验证码输入框time.sleep(10)# spider.browser.find_element_by_xpath('//*[@id="nc_1_captcha_input"]').send_keys(b)  # 输入验证码self.browser.find_element_by_xpath(' // *[ @ id = "nc_1_scale_submit"] / span').click()  # 点击提交time.sleep(2)if self.browser.find_element_by_class_name('nc-lang-cnt').text == "验证通过":  # 验证通过，停止breakself.browser.find_element_by_xpath('//*[@id="user_login_normal"]/button').click()  # 点击登录time.sleep(5)print("TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT")# self.browser.find_element_by_xpath('/ html / body / section / div / div / div / p[3] / a').click()self.browser.find_element_by_xpath('//*[@id="bindwxModal"]/div/div/div/button/span[1]').click()  # 点×dictCookies = self.browser.get_cookies()print(dictCookies)jsonCookies = json.dumps(dictCookies)# 登录完成后,将cookies保存到本地文件with open("E:cookies.json", "w") as fp:fp.write(jsonCookies)print("TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT")# 滑块直接登录else:self.browser.find_element_by_xpath('//*[@id="user_login_normal"]/button').click()  # 点击登录time.sleep(5)# self.browser.find_element_by_xpath('/ html / body / section / div / div / div / p[3] / a').click()self.browser.find_element_by_xpath('//*[@id="bindwxModal"]/div/div/div/button/span[1]').click()  # 点×dictCookies = self.browser.get_cookies()print(dictCookies)jsonCookies = json.dumps(dictCookies)# 登录完成后,将cookies保存到本地文件with open("E:cookies.json", "w") as fp:fp.write(jsonCookies)

#settings.py文件

DOWNLOADER_MIDDLEWARES = {
#'ScrapyDemo.middlewares.ScrapydemoDownloaderMiddleware': 543,
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 2,
#'ScrapyDemo.uamid.Uamid': 555,'ScrapyDemo.middlewares.UAMiddleware':543, #用来更换useragent#'ScrapyDemo.middlewares.ProxyMiddleware':544,#用来更换代理ip（不好用）'ScrapyDemo.middlewares.JSPageMiddleware':540,#与，middlewares对应 用来实现自动登入'scrapy.downloadermiddlewares.retry.RetryMiddleware':None,

}

spider.py文件

   import scrapy
import urllib.parseimport scrapy,time
import urllib.parse
from ScrapyDemo.items import QiccItem
from scrapy.selector import Selector
from  openpyxl import  Workbook
from openpyxl  import load_workbook
from selenium import webdriver
wb = Workbook()
ws = wb.active
from selenium.webdriver.chrome.options import Optionsclass QichachaSpider(scrapy.Spider):name = 'Qichacha'allowed_domains = ['qichacha.com']#custom_settings = {'ITEM_PIPELINES': {'ScrapyDemo.pipelines.QiccPipline': 300}}x = 1def start_requests(self):print("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")yield scrapy.Request(url='https://www.qichacha.com/user_login?back=%2F', callback=self.parse,dont_filter=True)def parse(self, response):print("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB")# 查询公司f = ["深圳市腾讯计算机系统有限公司","阿里巴巴"]#      wb = load_workbook('F:\\text.xlsx')# for cell in wb.active['A']:#     if cell == wb.active['A1']:  # 从文件第二行开始读取#         continue#     company = urllib.parse.quote(cell.value).replace('\n', "")#     url = 'https://www.qichacha.com/search?key=' + company#     time.sleep(5)for link in f:print(link)company = urllib.parse.quote(link)print(company)url = 'https://www.qichacha.com/search?key='+ companyprint(url)time.sleep(10)yield scrapy.Request(url, callback=self.parse1, errback=self.errparse1,dont_filter=True)def errparse1(self, err):print("errparse111111111111111")def parse1(self, response):# 提取列表中第一个公司，进入该页print("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC")# print(response.text)# print("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC")link = response.xpath('//tbody/tr[1]/td[3]/a/@href').extract_first()if link:detail_link = response.urljoin(link)print("DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD")print(detail_link)print("DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD")time.sleep(2)yield scrapy.Request(url=detail_link, callback=self.parse2, dont_filter=True, errback=self.errparse2)else:print("ERRORERRORERRORERRORERRORparse ERROR")def errparse2(self,err):print("errparse222222222222222222")def parse2(self, response):print("EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE")item = QiccItem()sel = Selector(response)item['name'] = sel.xpath('//div[@class="content"]/div[1]/h1/text()').extract_first() # 公司名称Official_website = sel.xpath('//div[@class="dcontent"]/div/span/a/@href').extract_first() # 公司官网if Official_website:item['Official_website'] = Official_websiteelse:item['Official_website'] = '暂无'item['Listed_company'] = sel.xpath('//header/div[2]/div[1]/a/h2/text()').extract_first()  # 上市公司if item['Listed_company']=='基本信息': # 非上市公司print(item['Listed_company'])item['registered_capital'] = sel.xpath('//section[@id="Cominfo"]/table[@class="ntable"][2]/tr[1]/td[2]/text()').extract_first() #注册资本item['company_type'] = sel.xpath('//section[@id="Cominfo"]/table[@class="ntable"][2]/tr[5]/td[2]/text()').extract_first() #公司类型item['staff_size'] = sel.xpath('//section[@id="Cominfo"]/table[@class="ntable"][2]/tr[9]/td[2]/text()').extract_first() #人员规模yield itemprint(item)else:print(item['Listed_company']) # 上市公司base = response.xpath('//header/div[2]/div[2]/a/@href').extract_first()#print("yyyyyyyyyyy")#print(base)detail_base = response.urljoin(base)#print("eeeeeeeeeeeeeeeee")#print(detail_base)time.sleep(2)yield scrapy.Request(detail_base, meta={'items':item},callback=self.parse3,errback=self.errparse3)

scrapy ,Selenium 企查查模拟登录相关推荐

scrapy抓企查查数据
scrapy抓企查查数据,scrapy爬虫核心代码,可以直接拿来参考注意点: 1.免费用户只能查看100条数据 2.vip收费用户只能查看5000条数据建议各位有需求的小伙伴直接去企查查接口平台, ...
selenium实现企查查自动登录
前言使用python3.6 selenium包实现用户密码自动登录企查查编写python函数通过用户名.密码入参返回登录cookie 编写函数编写get_cookie函数注意安装chromed ...
python爬虫从企查查获取企业信息-手工绕开企查查的登录验证
想要从企查查爬取企业信息,如果没有登录直接检索,邮箱.电话都被隐藏了: 上面的图片是之前截的图,今天再次检索,好像又可见了: 不过单击查看详情时,还是会被隐藏: 不管怎么说,只要企查查想限制登录,总会 ...
【交通领域张同学】-基础selenium企查查企业信用码抓取
目录 1. 需要用到的库 2. 本地打开浏览器,需要手动登录.(selenium库) 3. 从excel文件中读取数据(xlrd库) 4. 模拟爬虫(利用模拟手动操作浏览器来实现) 1. Xpath的 ...
python采集企查查企业信息，手工绕开企查查的登录验证
想要从企查查爬取企业信息,如果没有登录直接检索,邮箱.电话都被隐藏了: 很多人学习python,不知道从何学起. 很多人学习python,掌握了基本语法过后,不知道在哪里寻找案例上手. 很多已经做案例 ...
登录页面怎么弄_python爬虫另辟蹊径绕过企查查的登录验证，我太冇财了
从企查查爬取企业信息,如果没有登录直接检索,邮箱.电话都被隐藏了点击详情,部分信息同样会被隐藏毕竟只是打工的,没钱不能任性! 想要查看更完整的企业信息,只有登录了. 但登录需要滑块验证,有时可能还 ...
python 企查查爬虫_python爬虫另辟蹊径绕过企查查的登录验证，我太冇财了
从企查查爬取企业信息,如果没有登录直接检索,邮箱.电话都被隐藏了点击详情,部分信息同样会被隐藏毕竟只是打工的,没钱不能任性! 想要查看更完整的企业信息,只有登录了. 但登录需要滑块验证,有时可能还 ...
12-基于selenium实现12306模拟登录，及京东登录滑动缺口验证模拟登录
流程分析: 使用selenium打开登录页面对当前selenium打开的这张页面进行截图对当前图片的局部区域(验证码图片区域)进行截图这样验证码图片和模拟登录进行所显示的图片一一对应(如果我们对 ...
在scrapy上使用cookie模拟登录
下面的例子模拟登录知乎的个人设置页面设置代理池 scrapy + tor #安装tor sudo apt-get install tor sudo /etc/init.d/tor restart # ...

scrapy ,Selenium 企查查模拟登录

scrapy ,Selenium 企查查模拟登录相关推荐

最新文章

热门文章