# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rulefrom crawlPro.items import CrawlproItem
class CrawldemoSpider(CrawlSpider):name = 'crawlDemo'# allowed_domains = ['www.qiushibaike.com']start_urls = ['http://www.qiushibaike.com/']#rules元祖中存放的是不同规则解析器（封装好了某种解析规则）rules = (# Rule: 规则解析器，可以将连接提取器提取到的所有连接表示的页面进行指定规则（有中间的回调函数决定）的解析#LinkBxtractor:连接提取器，会去上面起始url响应回来的页面中，提取指定的urlRule(LinkExtractor(allow=r'/8hr/page/d+'), callback='parse_item', follow=True), #follow=True可以跟进保证将所有页面都提取出来（实际就是去重功能）)def parse_item(self, response):# i = {}# #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()# #i['name'] = response.xpath('//div[@id="name"]').extract()# #i['description'] = response.xpath('//div[@id="description"]').extract()# return idivs=response.xpath('//div[@id="content-left"]/div')for div in divs:item=CrawlproItem()#提取糗百中段子的作者item['author'] = div.xpath('./div[@class="author clearfix"]/a[2]/h2/text()').extract_first().strip('n')# 提取糗百中段子的内容item['content'] = div.xpath('.//div[@class="content"]/span/text()').extract_first().strip('n')yield item  #将item提交到管道

# -*- coding: utf-8 -*-# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.htmlimport scrapyclass CrawlproItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()author=scrapy.Field()content=scrapy.Field()

# -*- coding: utf-8 -*-# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.htmlclass CrawlproPipeline(object):def __init__(self):self.fp = Nonedef open_spider(self,spider):print('开始爬虫')self.fp = open('./data.txt','w',encoding='utf-8')def process_item(self, item, spider):# 将爬虫文件提交的item写入文件进行持久化存储self.fp.write(item['author']+':'+item['content']+'n')return itemdef close_spider(self,spider):print('结束爬虫')self.fp.close()

DOWNLOADER_MIDDLEWARES = {'crawlPro.middlewares.Mydaili': 543,  #Mydaili名字就是中间件里面的类名
}

class Mydaili(object):def process_request(self,request,spider):request.meta['proxy'] = "http://119.28.195.93:8888"