网页信息

打开东方财富网首页，切换至东方财富网 > 行情中心 > 沪深港通 > 沪股通/深股通切换至网站，主要包含当天的沪股和深股的实时情况，属于动态网页，使用scrapy配合selenium可以完成表格数据的爬取。

爬虫框架

stock_spider.py(爬虫文件)

import scrapy,time
from stock.items import StockItem
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver import ChromeOptionsclass StockSpider(scrapy.Spider):name = 'stock'# 允许爬取的域名allowed_domains = ['quote.eastmoney.com']# 爬虫的起始网页start_urls = ['http://quote.eastmoney.com/center/gridlist.html#sh_hk_board']def __init__(self):# 因为东方财富网行情表为相同网站下爬取不同页的数据# 定义网站页码变量self.page = 1# 手动定义最大页码，暂时还没有编写从网页自行获取self.max_page = 0# 爬取网页的列表索引self.index = 0# 辅助判断沪股还是深股self.name = ''self.nm = ''# 定义标签页的句柄self.first_h = None# 爬取网页列表self.url_list = ["http://quote.eastmoney.com/center/gridlist.html#sh_hk_board","http://quote.eastmoney.com/center/gridlist.html#sz_hk_board"]super(StockSpider, self).__init__(name='stock')# 定义模拟浏览器的路径chrome = '/home/zcreset/Scrapy/爬虫/chromedriver'# 设置无头爬取模式，即后台运行，无界面显示# chorme_options = ChromeOptions()# chorme_options.add_argument("--headless")# chorme_options.add_argument('--disable-gpu')# self.driver = webdriver.Chrome(executable_path=chrome,chrome_options=chorme_options)# 给爬虫设置Chrome浏览器self.driver = webdriver.Chrome(executable_path=chrome)def start_requests(self):url = self.url_list[self.index]self.name = url.split('#')[-1][:2]self.driver.get(url)# 获取当前标签页句柄self.first_h = self.driver.current_window_handle# 在当前标签页进行爬取请求yield scrapy.Request(url, callback=self.parse, dont_filter= True)# 定义如何存百度的单个页面中提取信息的方法def parse(self, response):# 调用字段函数创建数据存储字典# 暂时手动是手动记录最大页码Item = StockItem()# 判断股票归属地if self.name == 'sh':self.max_page = 30self.nm = '沪股通'elif self.name == 'sz':self.max_page = 45self.nm = '深股通'# 根据页码设定重复切换页码爬取的次数while self.page <= self.max_page:# 根据网页源码提取需要的信息，使用selenium的元素查找方法content_list = self.driver.find_elements_by_xpath('//*[@id="table_wrapper-table"]/tbody/tr')for content in content_list:Item['stock_id'] = content.find_element_by_xpath('./td[2]/a').textItem['stock_name'] = content.find_element_by_xpath('./td[3]/a').textItem['cur_value'] = content.find_element_by_xpath('./td[5]/span').textItem['change_value'] = content.find_element_by_xpath('./td[6]/span').textItem['change_rate'] = content.find_element_by_xpath('./td[7]/span').textItem['deal_num'] = content.find_element_by_xpath('./td[8]').textItem['deal_rate'] = content.find_element_by_xpath('./td[10]').textItem['deal_money'] = content.find_element_by_xpath('./td[9]').textItem['trans_rate'] = content.find_element_by_xpath('./td[15]').textItem['market_sale'] = content.find_element_by_xpath('./td[16]').textItem['market_rate'] = content.find_element_by_xpath('./td[17]').textItem['stock_loc'] = self.nmyield Item# 页码更新self.page += 1# 搜索下一页按钮并点击下一页self.driver.find_element_by_class_name('next').click()# 等待页面加载time.sleep(1)# 定义新建标签页并切换至指定网站的脚本js = 'window.open("http://quote.eastmoney.com/center/gridlist.html#sz_hk_board");'# 更新网页索引self.index += 1if self.index < len(self.allowed_domains):url = self.url_list[self.index]self.name = url.split('#')[-1][:2]# 运行脚本self.driver.execute_script(js)# 获取新建标签页的句柄handles = self.driver.window_handlesh = Nonefor handle in handles:if handle != self.first_h:h = handle# 切换当前标签页self.driver.switch_to.window(h)# 在新的标签页进行爬取请求yield scrapy.Request(url, callback=self.parse)

items.py (爬取字段命名)

import scrapy
# 定义要爬起的字段名，即字典的键
class StockItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()stock_id = scrapy.Field()stock_name = scrapy.Field()cur_value = scrapy.Field()change_value = scrapy.Field()change_rate = scrapy.Field()deal_num = scrapy.Field()deal_money = scrapy.Field()deal_rate = scrapy.Field()trans_rate = scrapy.Field()market_sale = scrapy.Field()market_rate = scrapy.Field()stock_loc = scrapy.Field()```

middlewares.py(Selenium中间件)

// 基于Selenium的中间下载件
class SeleniumStockDownloaderMiddleware:
# class DownloaderMiddleware:# Not all methods need to be defined. If a method is not defined,# scrapy acts as if the downloader middleware does not modify the# passed objects.@classmethoddef from_crawler(cls, crawler):# This method is used by Scrapy to create your spiders.s = cls()crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)return s# 处理回调请求def process_request(self, request, spider):if spider.name == "stock":# 调用爬虫的模拟浏览器打开指定网址spider.driver.get(request.url)# 获取已打开网页的源代码origin_code = spider.driver.page_source# 将源代码构造成为一个Response对象，并返回。res = HtmlResponse(url=request.url, encoding='utf8', body=origin_code, request=request)return resreturn Nonedef process_response(self, request, response, spider):print(response.url, response.status)return response## def process_request(self, request, spider):#     # Called for each request that goes through the downloader#     # middleware.##     # Must either:#     # - return None: continue processing this request#     # - or return a Response object#     # - or return a Request object#     # - or raise IgnoreRequest: process_exception() methods of#     #   installed downloader middleware will be called#     return None## def process_response(self, request, response, spider):#     # Called with the response returned from the downloader.##     # Must either;#     # - return a Response object#     # - return a Request object#     # - or raise IgnoreRequest#     return response#def process_exception(self, request, exception, spider):# Called when a download handler or a process_request()# (from other downloader middleware) raises an exception.# Must either:# - return None: continue processing this exception# - return a Response object: stops process_exception() chain# - return a Request object: stops process_exception() chainpassdef spider_opened(self, spider):spider.logger.info('Spider opened: %s' % spider.name)

pipelines.py

from itemadapter import ItemAdapter
import pymysql
import datetime
import settingsclass StockPipeline:def __init__(self):# 连接MySQL数据库self.con = pymysql.connect(host=settings.MYSQL_HOST,  # host为数据库所在服务的ip地址port=settings.MYSQL_PORT,  # 端口号user=settings.MYSQL_USER,  # 用户名passwd=settings.MYSQL_PASSWD,  # 密码db=settings.MYSQL_DB,  # 数据库名charset='utf8',  # 编码格式)# 定位游标self.cursor = self.con.cursor()def process_item(self, item, spider):# 定义sql语句，比如创建表、插入数据sql = "insert into stocks values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,curdate())"# 执行sql语句，插入数据self.cursor.execute(sql,(item['stock_id'],item['stock_name'],item['cur_value'],item['change_value'],item['change_rate'],item['deal_num'],item['deal_money'],item['deal_rate'],item['trans_rate'],item['market_sale'],item['market_rate'],item['stock_loc']))self.con.commit()return item# 定义一个爬虫结束调用函数def close_spider(self,spider):# 将爬取的数据从MySQL数据库中输出到文件中filename = str(datetime.date.today()) + '.txt'path = '/home/zcreset/Mysql学习/爬虫数据/Stock/' + filenamesql = "select * from stocks into outfile %s"self.cursor.execute(sql,path)# 关闭游标self.cursor.close()# 关闭数据库连接self.con.close()

settings.py(设置文件)

BOT_NAME = 'stock'SPIDER_MODULES = ['stock.spiders']
NEWSPIDER_MODULE = 'stock.spiders'# Crawl responsibly by identifying yourself (and your website) on the user-agent
# 用户代理
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"# 数据库地址
MYSQL_HOST = 'localhost'
# 端口
MYSQL_PORT = 3306
# 数据库用户名
MYSQL_USER = 'root'
# 密码写自己的
MYSQL_PASSWD = '123'
# 插入的数据库
MYSQL_DB = 'test'
# 下载延迟
DOWNLOAD_DELAY = 2# 字段管道处理函数
ITEM_PIPELINES = {'stock.pipelines.StockPipeline': 200,
}# ip代理池，到代理网站获取
# IPPOOL=[
#     {"ipaddr":"114.233.189.60:9999"},
#     {"ipaddr":"182.87.136.50:9999"},
#     {"ipaddr":"27.152.192.225:8888"},
#     {"ipaddr":"223.243.245.33:9999"},
# ]# 下载中间件
DOWNLOADER_MIDDLEWARES = {'stock.middlewares.SeleniumStockDownloaderMiddleware':543,# 如果启用代理池需要包含如下两个设置# 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware':123,# 'stock.middlewares.IPPOOLS':125
}# Obey robots.txt rules
# false为不遵守网站爬虫规则
ROBOTSTXT_OBEY = False

运行Scrapy框架

命令：scrapy crawl stock(爬虫名)

存在问题

不知道是网页设置了爬取时间限制，或者是selenium的持续操作时间未设置，大概五分钟后爬虫就自动停止了，还请大佬帮忙指正！
GitHub：https://github.com/Nightkisser/Scrapy-stock-

跌跌撞撞尝试Scrapy+Selenium+MySQL爬取与存储东方财富网股票数据相关推荐

python爬虫爬取股票软件数据_Python爬虫抓取东方财富网股票数据并实现MySQL数据库存储（转载）...
完整代码实际上,整个事情完成了两个相对独立的过程:1.爬虫获取网页股票数据并保存到本地文件:2.将本地文件数据储存到MySQL数据库.并没有直接的考虑把从网页上抓取到的数据实时(或者通过一个临时文件 ...
python+selenium+phantomJS爬取国家地表水水质自动监测实时数据发布系统——动态网页爬虫
一.关于phantomjs 1.介绍 PhantomJS是一个为自动化而生的利器,它本质上是一个基于webkit内核的无界面浏览器,并可使用JavaScript或CoffeeScript进行编程.由于 ...
使用selenium + chrome爬取中国大学Mooc网的计算机学科的所有课程链接
目的:使用selenium + chrome爬取中国大学Mooc网计算机学科的所有的课程链接列表思路:找到每个分页的节点属性为class="m-course-list" 的div ...
Python爬虫抓取东方财富网股票数据并实现MySQL数据库存储（转）
Python爬虫可以说是好玩又好用了.现想利用Python爬取网页股票数据保存到本地csv数据文件中,同时想把股票数据保存到MySQL数据库中.需求有了,剩下的就是实现了. 在开始之前,保证已经安装好 ...
python抓取数据库数据_Python爬虫抓取东方财富网股票数据并实现MySQL数据库存储...
Python爬虫可以说是好玩又好用了.现想利用Python爬取网页股票数据保存到本地csv数据文件中,同时想把股票数据保存到MySQL数据库中.需求有了,剩下的就是实现了. 在开始之前,保证已经安装好 ...
python爬虫实现股票数据存储_Python爬虫抓取东方财富网股票数据并实现MySQL数据库存储！...
Python爬虫可以说是好玩又好用了.现想利用Python爬取网页股票数据保存到本地csv数据文件中,同时想把股票数据保存到MySQL数据库中.需求有了,剩下的就是实现了. 在开始之前,保证已经安装好 ...
MySQL 怎么插入10天前的日期_Python爬虫抓取东方财富网股票数据并实现MySQL数据库存储！...
Python爬虫可以说是好玩又好用了.现想利用Python爬取网页股票数据保存到本地csv数据文件中,同时想把股票数据保存到MySQL数据库中.需求有了,剩下的就是实现了. 在开始之前,保证已经安装好 ...
爬虫：东方财富网股票数据爬取
前言因为东方财富网的Js限制,第一页很好爬取,但是第二页开始的网页地址并没有改变,看了下xpath页面元素也和第一页没什么区别,所以只好曲线救国,用selenium找到"下一页" ...
用scrapy+selenium + phantomjs 爬取vip网页,保存为json格式,写入到mysql数据库,下载图片(二)
接上一编 weipin.py文件的代码 : # -*- coding: utf-8 -*- import scrapy from weipinhui.items import WeipinhuiIte ...

跌跌撞撞尝试Scrapy+Selenium+MySQL爬取与存储东方财富网股票数据

目录