爱企查爬虫selenium

写这个是为了寻求一个可以实现模拟登录功能的大佬，我已经实现了，爬取第一页，但是到了第二页需要登录，我就不会了。或者有想法一起探讨。会写先登录然后再爬取的也行。

import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import openpyxl as  op
import csv
from urllib.parse import urljoin
ws=op.Workbook()
wb=ws.create_sheet(index=0)
wb.cell(row=1,column=1,value='企业名称')
wb.cell(row=1,column=2,value='统一社会信用代码')
wb.cell(row=1,column=3,value='法定代表人')
wb.cell(row=1,column=4,value='经营状态')
wb.cell(row=1,column=5,value='成立日期')
wb.cell(row=1,column=6,value='行政区划')
wb.cell(row=1,column=7,value='注册资本')
wb.cell(row=1,column=8,value='实缴资本')
wb.cell(row=1,column=9,value='企业类型')
wb.cell(row=1,column=10,value='所属行业')
wb.cell(row=1,column=11,value='工商注册号')
wb.cell(row=1,column=12,value='组织机构代码')
wb.cell(row=1,column=13,value='纳税人识别号')
wb.cell(row=1,column=14,value='纳税人资质')
wb.cell(row=1,column=15,value='营业期限')
wb.cell(row=1,column=16,value='核准日期')
wb.cell(row=1,column=17,value='登记机关')
wb.cell(row=1,column=18,value='参保人数')
wb.cell(row=1,column=19,value='曾用名')
wb.cell(row=1,column=20,value='注册地址')
wb.cell(row=1,column=21,value='经营范围')
wb.cell(row=1,column=22,value='主管部门名称')
wb.cell(row=1,column=23,value='持股比例')
wb.cell(row=1,column=24,value='认缴出资额')
wb.cell(row=1,column=25,value='认缴出资日期')#get直接返回，不再等待界面加载完成
desired_capabilities = DesiredCapabilities.CHROME
desired_capabilities["pageLoadStrategy"] = "none"# 设置谷歌驱动器的环境
options = webdriver.ChromeOptions()
# 设置chrome不加载图片，提高速度
#options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
# 设置不显示窗口
#options.add_argument('--headless')
# 创建一个谷歌驱动器
driver = webdriver.Chrome(options=options)
# 设置搜索主题
theme = "施工图审查"
# 设置所需篇数
papers_need = 20# 打开页面
#driver.get("https://aiqicha.baidu.com/s?q=%E6%96%BD%E5%B7%A5%E5%9B%BE%E5%AE%A1%E6%9F%A5&t=0")
driver.get("https://aiqicha.baidu.com/s?q=%E6%96%BD%E5%B7%A5%E5%9B%BE%E5%AE%A1%E6%9F%A5")
# 传入关键字
WebDriverWait( driver, 100 ).until( EC.presence_of_element_located( (By.XPATH ,'''//*[@id="aqc-header-search-input"]''') ) ).send_keys(theme)
# 点击搜索
WebDriverWait( driver, 100 ).until( EC.presence_of_element_located( (By.XPATH ,"/html/body/div[1]/div[1]/header/div/div[2]/button") ) ).click()
time.sleep(3)# 点击切换中文文献
# 获取总文献数和页数# 赋值序号, 控制爬取的文章数量
count = 1
# 当，爬取数量小于需求时，循环网页页码
while count <= papers_need:# 等待加载完全，休眠3Stime.sleep(3)title_list = WebDriverWait( driver,1 ).until( EC.presence_of_all_elements_located( (By.CSS_SELECTOR  ,"body > div.base.page-search.has-search-tab > div.aqc-content-wrapper.has-footer > div > div.main > div.list-wrap > div.company-list > div > div:nth-child(-n+10) > div.info > div > h3 > a") ) )# 循环网页一页中的条目for i in range(len(title_list)):try:term = count%10   # 本页的第几个条目title_xpath = "/html/body/div[1]/div[2]/div/div[1]/div[3]/div[2]/div/div[@class='card']/div[2]/div/h3/text()"#author_xpath = "/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[@class='author']/td[3]/text()"#source_xpath = "/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[@class='source']/td[4]/text()"#date_xpath = "/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[@class='data']/td[5]/text()"#database_xpath = "/html/body/div[3]/div[2]/div[2]/div[2]/form/div/table/tbody/tr[@class='data']/td[6]/text()"#title = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH ,title_xpath) ) ).click()#authors = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH ,author_xpath) ) ).text#source = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH ,source_xpath) ) ).text#date = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH ,date_xpath) ) ).text#database = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH ,database_xpath) ) ).text#date = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH ,"/html/body/div[2]/div[1]/div[3]/div/div/div[3]/div/h1") ) ).text# 点击条目title_list[i].click()# 获取driver的句柄n = driver.window_handles# driver切换至最新生产的页面driver.switch_to.window(n[-1])# 开始获取页面信息title = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH ,"/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[1]/td[2]/span") ) ).textcode = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH ,"/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[1]/td[4]") ) ).textfadindaibiao = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH ,"/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[2]/td[2]/div[2]/a[1]") ) ).textjingyingzhuangtai = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[2]/td[4]') ) ).textchenglidata = WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[3]/td[2]') ) ).textarea=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[3]/td[4]') ) ).textzhuceziben=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[4]/td[2]') ) ).textshijiaoziben=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[4]/td[4]') ) ).textqiyeleixing=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[5]/td[2]') ) ).textsuosuhangye=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[5]/td[4]') ) ).textgongshangzhucehao= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[6]/td[2]/span') ) ).textzuzhijigodaima= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[6]/td[4]/span') ) ).textnasuorenshibiehao= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[7]/td[2]/span') ) ).textnasuorenzizhi= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[7]/td[4]') ) ).textyingyeqingxian= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[8]/td[2]') ) ).texthezhunriqi= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[8]/td[4]') ) ).textdengjijiguan= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[9]/td[2]') ) ).textcanbaorenshu= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[9]/td[4]') ) ).textcengyongming= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[10]/td[2]/p') ) ).textaddress= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[11]/td[2]/span[1]') ) ).textjingyingfanwei= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[12]/td[2]/div') ) ).textgudongmingcheng= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[4]/div[1]/table/tbody/tr/td[2]/div/div/div[2]/div[1]/div/div[1]/a') ) ).textchigubili= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[4]/div[1]/table/tbody/tr/td[3]/div/div/span') ) ).textmoney=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[4]/div[1]/table/tbody/tr/td[4]/div/span') ) ).textdata=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[4]/div[1]/table/tbody/tr/td[5]/div/span') ) ).texttry:zhuceziben=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[4]/td[2]') ) ).textshijiaoziben=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[4]/td[4]') ) ).textnasuorenshibiehao= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[7]/td[2]/span') ) ).textnasuorenzizhi= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[7]/td[4]') ) ).texthezhunriqi= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[8]/td[4]') ) ).textdengjijiguan= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[9]/td[2]') ) ).textcengyongming= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[3]/table/tbody/tr[10]/td[2]/p') ) ).textgudongmingcheng= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[4]/div[1]/table/tbody/tr/td[2]/div/div/div[2]/div[1]/div/div[1]/a') ) ).textchigubili= WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[4]/div[1]/table/tbody/tr/td[3]/div/div/span') ) ).textmoney=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[4]/div[1]/table/tbody/tr/td[4]/div/span') ) ).textdata=  WebDriverWait( driver, 10 ).until( EC.presence_of_element_located((By.XPATH  ,'/html/body/div[1]/div[2]/div/div[7]/div[1]/div[4]/div[1]/table/tbody/tr/td[5]/div/span') ) ).textexcept:zhuceziben= '无'shijiaoziben= '无'nasuorenshibiehao= '无'nasuorenzizhi= '无'hezhunriqi= '无'dengjijiguan= '无'cengyongming= '无'gudongmingcheng= '无'chigubili= '无'money= '无'data= '无'url = driver.current_url# 获取下载链接# link = WebDriverWait( driver, 10 ).until( EC.presence_of_all_elements_located((By.CLASS_NAME  ,"btn-dlcaj") ) )[0].get_attribute('href')# link = urljoin(driver.current_url, link)# 写入文件#res = "+str(title)+str(authors)+str(institute)+str(abstract)+str(keywords)".replace("\n","")+"\n"#print(res)print("企业名称："+str(title)+"统一社会信用代码："+str(code)+"法定代表人："+str(fadindaibiao)+"经营状态："+str(jingyingzhuangtai)+"成立日期："+str(chenglidata)+"行政区划："+str(area)+"注册资本："+str(zhuceziben)+"实缴资本："+str(shijiaoziben)+"企业类型："+str(qiyeleixing)+"所属行业："+str(suosuhangye)+"工商注册号："+str(gongshangzhucehao)+"组织机构代码："+str(zuzhijigodaima)+"纳税人识别号："+str(nasuorenshibiehao)+"纳税人资质："+str(nasuorenzizhi)+"营业期限："+str(yingyeqingxian)+"核准日期："+str(hezhunriqi)+"登记机关："+str(dengjijiguan)+"参保人数："+str(canbaorenshu)+"曾用名："+str(cengyongming)+"注册地址："+str(address)+"经营范围："+str(jingyingfanwei)+"股东："+str(gudongmingcheng)+"持股比列："+str(chigubili)+"认缴出资额："+str(money)+"认缴出资日期："+str(data))wb.cell(row=count,column=1,value=title)wb.cell(row=count,column=2,value=code)wb.cell(row=count,column=3,value=fadindaibiao)wb.cell(row=count,column=4,value=jingyingzhuangtai)wb.cell(row=count,column=5,value=chenglidata)wb.cell(row=count,column=6,value=area)wb.cell(row=count,column=7,value=zhuceziben)wb.cell(row=count,column=8,value=shijiaoziben)wb.cell(row=count,column=9,value=qiyeleixing)wb.cell(row=count,column=10,value=suosuhangye)wb.cell(row=count,column=11,value=gongshangzhucehao)wb.cell(row=count,column=12,value=zuzhijigodaima)wb.cell(row=count,column=13,value=nasuorenshibiehao)wb.cell(row=count,column=14,value=nasuorenzizhi)wb.cell(row=count,column=15,value=yingyeqingxian)wb.cell(row=count,column=16,value=hezhunriqi)wb.cell(row=count,column=17,value=dengjijiguan)wb.cell(row=count,column=18,value=canbaorenshu)wb.cell(row=count,column=19,value=cengyongming)wb.cell(row=count,column=20,value=address)wb.cell(row=count,column=21,value=jingyingfanwei)wb.cell(row=count,column=22,value=gudongmingcheng)wb.cell(row=count,column=23,value=chigubili)wb.cell(row=count,column=24,value=money)wb.cell(row=count,column=25,value=data)#with open('CNKI_res.txt', 'a', encoding='gbk') as f:#f.write(keywords)#except:except:print(count)print("失败")# 跳过本条，接着下一个continuefinally:n2 = driver.window_handlesif len(n2) > 1:driver.close()driver.switch_to.window(n2[0])# 计数,判断需求是否足够count += 1if count == papers_need:break#except:#print(" 第{count} 条爬取失败\n")# 跳过本条，接着下一个#continuews.save('施工图企业.xls')WebDriverWait( driver, 10 ).until( EC.presence_of_element_located( (By.CSS_SELECTOR ,"body > div.base.page-search.has-search-tab > div.aqc-content-wrapper.has-footer > div > div.main > div.list-wrap > div.company-list > ul > li.ivu-page-next > a > i") ) ).click()#WebDriverWait( driver, 10 ).until( EC.presence_of_element_located( (By.CSS_SELECTOR ,"body > div.base.page-search.has-search-tab > div.aqc-content-wrapper.has-footer > div > div.main > div.list-wrap > div.company-list > div > div > div > button > span") ) ).click()#time.sleep(50)#等待20s，完成手动登录操作# 关闭浏览器
driver.close()

第一页爬取成功后出现这个

急需大佬解决。在我的基础上添加一部分。

爱企查爬虫selenium相关推荐

python爬虫爬取爱企查公司信息
目录一.思路 1.1输入网址,查看源代码 1.2 F12抓包,寻找数据包. 1.3查看数据包的URL和是否有传递的参数 1.4编写代码一.思路 1.1输入网址,查看源代码可以看到源代码中没有 ...
爱企查爬取公司基本信息
前言: 前几天老板让我爬下制造业相关的公司,最开始说用企查查,但是你懂的,6页之后就要收费了,不能白嫖.就是前6页页不好爬,超频访问阻拦着我.但是VIP似乎可以直接导出,不用爬虫,遂和老板交涉,哪怕花 ...
使用python程序抓取天眼查，企查查和爱企查上的企业基本信息
本文说明使用python程序自动抓取天眼查或者企查查上的企业基本信息.本文中的程序仅供学习参考,请通过正规渠道获取相应的数据资产. 工程目录如下: w3目录下内容为: 工程中的部分源码如下. 文件pa ...
青龙面板跑爱企查脚本兑换爱奇艺月卡百度网盘会员等
可兑换物品如下: 爱企查脚本文件第一步进入面板后台定时任务如图添加任务填写即可任务名(随意)如:爱企查命令: ql raw https://github.com/shiunke/Scr ...
爱企查显示，Keep赴港上市，冲击“线上健身第一股”
Keep于2月25日向港交所递交了招股书,若IPO成功,将成为国内"线上健身第一股". 有数据显示,2021年按月活跃用户及用户完成的锻炼次数计算,Keep已经成为中国及全球最大的 ...
python爬取企业电话_如何用python抓取爱企查企业信息
前段时间,经理让我去找一些企业的信息,我平常习惯于使用爱企查.所以,便想着写一个程序来实现这个,所以有以下的代码:import json import requests import re from ...
互联网日报 | 百度推出企业查询工具“爱企查”；滴滴货运宣布再开6城；京东健康推出家庭医生服务...
今日看点 ✦ 百度推出企业信息查询工具"爱企查",打破企业查询付费模式 ✦ 淘宝天猫金华商家运营中心正式开业,为阿里全国首个商家运营中心 ✦ 京东健康推出家庭医生服务,未来5年将服 ...
数视来联系方式类似天眼查企查查爱企查探迹励销云精线索源码开发。企业工商信息接口视野数科这种类型的系统开发。
类似天眼查企查查爱企查探痕励销云精线索源码开发. 企业工商资源现在企业工商线索 2.5亿还在不断增加现在有企业维度 230个维度,目前基本市面上有的维度在这个系统上都有.维度可 ...
HW-自动收集爱企查信息
为减少信息收集中[爱企查]子公司及域名等的手工工作量. 可使用ENScan_GO一键收集控股公司ICP备案.APP.小程序.微信公众号等信息聚合导出. 注意:使用程序可能导致账号被封,不过我搜了六 ...
利用python加selenium模拟浏览器爬取百度爱企查企业详细信息
目录说明企查查网页构成方式代码与详解说明本人不是什么爬虫大佬,只是最近在做一个项目的时候需要根据企业的名称把企业的信息爬取下来,例如营业执照.工商注册号等.在网上找了很多的信息与资源,都不是 ...

爱企查爬虫selenium

爱企查爬虫selenium相关推荐

最新文章

热门文章