1 import scrapy
 2 from selenium import webdriver
 3 from wangyi.items import WangyiItem
 4
 5
 6 class TestSpider(scrapy.Spider):
 7     name = 'test'
 8     # allowed_domains = ['www.xxx.com']
 9     start_urls = ['http://war.163.com/']
10
11     def __init__(self):
12         self.bro = webdriver.Chrome(executable_path=r'F:\爬虫加数据\练习\第三天\chromedriver.exe')
13
14     def parse(self, response):
15
16         div_list = response.xpath('//div[@class="data_row news_article clearfix "]')
17         for div in div_list:
18             item = WangyiItem()
19             title = div.xpath('./div/div/h3/a/text()').extract_first()
20             time = div.xpath('./div/div[2]/span/text()').extract_first()
21             print(title)
22             item['title'] = title
23             item['time'] = time
24             yield item
25
26     def closed(self, spider):
27         print('关闭浏览器对象!')
28         self.bro.quit()

 1 from scrapy.http import HtmlResponse
 2 def process_response(self, request, response, spider):
 3         # Called with the response returned from the downloader.
 4
 5         # Must either;
 6         # - return a Response object
 7         # - return a Request object
 8         # - or raise IgnoreRequest
 9         bro = spider.bro
10         bro.get(url=request.url)
11         sleep(3)
12         # 获取爬取的数据
13         page_text = bro.page_source
14         sleep(3)
15
16         return HtmlResponse(url=spider.bro.current_url, body=page_text, encoding='utf8', request=request)

DOWNLOADER_MIDDLEWARES = {'wangyi.middlewares.WangyiDownloaderMiddleware': 543,
}