爬取过程中 遇见 百度蜘蛛反爬 robot.txt，我们可以在scrapy 的setting.py 配置文件下配置

ROBOTSTXT_OBEY = False

最终代码

# -*- coding: utf-8 -*-
from scrapy.spider import Spider
from scrapy.contrib.spiders import CrawlSpider, Rule
#from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.linkextractors import LinkExtractor
from scrapy.selector import Selector
from scrapy.http import Request, HtmlResponse
from scrapy import logfrom items import BDzdItemclass BDzdSpider(CrawlSpider):global qa_number;qa_number=0;"""爬取百度知道 银行"""log.msg("log",level=log.DEBUG)def _requests_to_follow(self, response):if not isinstance(response, HtmlResponse):returnseen = set()for n, rule in enumerate(self._rules):links = [lnk for lnk in rule.link_extractor.extract_links(response)if lnk not in seen]if links and rule.process_links:links = rule.process_links(links)for link in links:if link.text.find("银行") == -1:continue;seen.add(link)r = Request(url=link.url, callback=self._response_downloaded)r.meta.update(rule=n, link_text=link.text)yield rule.process_request(r)name = "bankSpider"download_delay = 1allowed_domains = ["zhidao.baidu.com"]start_urls = ["https://zhidao.baidu.com/question/1796062605517856547.html?fr=iks&word=%D2%F8%D0%D0&ie=gbk"]rules = [Rule(LinkExtractor(allow=('/question/.*'),restrict_xpaths=('//a[@class="related-link"]')),callback='parse_item',follow=True)]def parse_item(self, response):#return;# open("aa.txt", 'wb').write(response.body)sel = Selector(response)url=response._url;question=sel.xpath('//span[@class="ask-title "]/text()').extract()answer = sel.xpath('//pre[@class="best-text mb-10"]/text()').extract()otherAnswer=sel.xpath('//div[@class="answer-text line"]/span/text()').extract()#sites=sel.xpath('//a[@class="related-link"]')item = BDzdItem()item["question"] = ''.join(question);if len(answer) > 0:item["answer"] = ''.join(answer);#因为xpath text()截出来可能是字符数组，要转成字符elif len(otherAnswer) > 0:item["answer"] = ''.join(otherAnswer[0]);else:return;global qa_numberqa_number=qa_number+1;item["number"]=qa_numberprint "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ 第" + str(qa_number)+" 条";print "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@" + url;print "##########################################" + item["question"];print "*******************************************" +  item["answer"];yield item

如果有多个spider在一个项目中，可以在pipelines.py中这样写

# -*- coding: utf-8 -*-# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.htmlimport json
import codecsclass TutorialPipeline(object):def process_item(self, item, spider):print "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%0"return itemclass BDzdPipeline(object):def __init__(self):self.bankFile = codecs.open('data_bank.json', 'wb', encoding='utf-8')#银行self.mobileFile = codecs.open('data_mobile.json', 'wb', encoding='utf-8')#移动self.baoxianFile = codecs.open('data_baoxian.json', 'wb', encoding='utf-8')#保险self.jinrongFile = codecs.open('data_jinrong.json', 'wb', encoding='utf-8')#金融def process_item(self, item, spider):line = json.dumps(dict(item)) + '\n'if spider.name=='bankSpider':self.bankFile.write(line.decode("unicode_escape"))elif spider.name == 'mobileSpider':self.mobileFile.write(line.decode("unicode_escape"))elif spider.name == 'baoxianSpider':self.baoxianFile.write(line.decode("unicode_escape"))elif spider.name == 'jinrongSpider':self.jinrongFile.write(line.decode("unicode_escape"))return item