"""
Base class for Scrapy spidersSee documentation in docs/topics/spiders.rst
"""
import logging
import warningsfrom scrapy import signals
from scrapy.http import Request
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import method_is_overriddenclass Spider(object_ref):"""Base class for scrapy spiders. All spiders must inherit from thisclass."""name = Nonecustom_settings = Nonedef __init__(self, name=None, **kwargs):if name is not None:self.name = nameelif not getattr(self, 'name', None):raise ValueError("%s must have a name" % type(self).__name__)self.__dict__.update(kwargs)if not hasattr(self, 'start_urls'):self.start_urls = []@propertydef logger(self):logger = logging.getLogger(self.name)return logging.LoggerAdapter(logger, {'spider': self})def log(self, message, level=logging.DEBUG, **kw):"""Log the given message at the given log levelThis helper wraps a log call to the logger within the spider, but youcan use it directly (e.g. Spider.logger.info('msg')) or use any otherPython logger too."""self.logger.log(level, message, **kw)@classmethoddef from_crawler(cls, crawler, *args, **kwargs):spider = cls(*args, **kwargs)spider._set_crawler(crawler)return spiderdef set_crawler(self, crawler):warnings.warn("set_crawler is deprecated, instantiate and bound the ""spider to this crawler with from_crawler method ""instead.",category=ScrapyDeprecationWarning, stacklevel=2)assert not hasattr(self, 'crawler'), "Spider already bounded to a " "crawler"self._set_crawler(crawler)def _set_crawler(self, crawler):self.crawler = crawlerself.settings = crawler.settingscrawler.signals.connect(self.close, signals.spider_closed)def start_requests(self):cls = self.__class__if method_is_overridden(cls, Spider, 'make_requests_from_url'):warnings.warn("Spider.make_requests_from_url method is deprecated; it ""won't be called in future Scrapy releases. Please ""override Spider.start_requests method instead (see %s.%s)." % (cls.__module__, cls.__name__),)for url in self.start_urls:yield self.make_requests_from_url(url)else:for url in self.start_urls:yield Request(url, dont_filter=True)def make_requests_from_url(self, url):""" This method is deprecated. """return Request(url, dont_filter=True)def parse(self, response):raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))@classmethoddef update_settings(cls, settings):settings.setdict(cls.custom_settings or {}, priority='spider')@classmethoddef handles_request(cls, request):return url_is_from_spider(request.url, cls)@staticmethoddef close(spider, reason):closed = getattr(spider, 'closed', None)if callable(closed):return closed(reason)def __str__(self):return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self))__repr__ = __str__BaseSpider = create_deprecated_class('BaseSpider', Spider)class ObsoleteClass(object):def __init__(self, message):self.message = messagedef __getattr__(self, name):raise AttributeError(self.message)spiders = ObsoleteClass('"from scrapy.spider import spiders" no longer works - use ''"from scrapy.spiderloader import SpiderLoader" and instantiate ''it with your project settings"'
)# Top-level imports
from scrapy.spiders.crawl import CrawlSpider, Rule
from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
from scrapy.spiders.sitemap import SitemapSpider

"""
This modules implements the CrawlSpider which is the recommended spider to use
for scraping typical web sites that requires crawling pages.See documentation in docs/topics/spiders.rst
"""import copy
import sixfrom scrapy.http import Request, HtmlResponse
from scrapy.utils.spider import iterate_spider_output
from scrapy.spiders import Spiderdef identity(x):return xclass Rule(object):def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):self.link_extractor = link_extractorself.callback = callbackself.cb_kwargs = cb_kwargs or {}self.process_links = process_linksself.process_request = process_requestif follow is None:self.follow = False if callback else Trueelse:self.follow = followclass CrawlSpider(Spider):rules = ()def __init__(self, *a, **kw):super(CrawlSpider, self).__init__(*a, **kw)self._compile_rules()def parse(self, response):return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)def parse_start_url(self, response):return []def process_results(self, response, results):return resultsdef _build_request(self, rule, link):r = Request(url=link.url, callback=self._response_downloaded)r.meta.update(rule=rule, link_text=link.text)return rdef _requests_to_follow(self, response):if not isinstance(response, HtmlResponse):returnseen = set()for n, rule in enumerate(self._rules):links = [lnk for lnk in rule.link_extractor.extract_links(response)if lnk not in seen]if links and rule.process_links:links = rule.process_links(links)for link in links:seen.add(link)r = self._build_request(n, link)yield rule.process_request(r)def _response_downloaded(self, response):rule = self._rules[response.meta['rule']]return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)def _parse_response(self, response, callback, cb_kwargs, follow=True):if callback:cb_res = callback(response, **cb_kwargs) or ()cb_res = self.process_results(response, cb_res)for requests_or_item in iterate_spider_output(cb_res):yield requests_or_itemif follow and self._follow_links:for request_or_item in self._requests_to_follow(response):yield request_or_itemdef _compile_rules(self):def get_method(method):if callable(method):return methodelif isinstance(method, six.string_types):return getattr(self, method, None)self._rules = [copy.copy(r) for r in self.rules]for rule in self._rules:rule.callback = get_method(rule.callback)rule.process_links = get_method(rule.process_links)rule.process_request = get_method(rule.process_request)@classmethoddef from_crawler(cls, crawler, *args, **kwargs):spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)return spiderdef set_crawler(self, crawler):super(CrawlSpider, self).set_crawler(crawler)self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)

class scrapy.spiders.Rule(link_extractor, callback = None, cb_kwargs = None, follow = None, process_links = None, process_request = None
)

class scrapy.linkextractors.LinkExtractor

class scrapy.linkextractors.LinkExtractor(allow = (),deny = (),allow_domains = (),deny_domains = (),deny_extensions = None,restrict_xpaths = (),tags = ('a','area'),attrs = ('href'),canonicalize = True,unique = True,process_value = None
)

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Ruleclass TestSpider(CrawlSpider):name = 'Test'allowed_domains = ['Test.com']start_urls = ['http://Test.com/']rules = (Rule(LinkExtractor(allow=r'Items/'), callback='parse_test', follow=True),)def parse_test(self, response):items = {}
············return items

{优先级0 : 队列0优先级1 : 队列1优先级2 : 队列2}

def request_seen(self, request):# self.request_figerprints就是一个指纹集合  fp = self.request_fingerprint(request)# 这就是判重的核心操作  if fp in self.fingerprints:return Trueself.fingerprints.add(fp)if self.file:self.file.write(fp + os.linesep)

from scrapy import signals
from scrapy.exceptions import DontCloseSpider
from scrapy.spiders import Spider, CrawlSpiderfrom . import connection, defaults
from .utils import bytes_to_strclass RedisMixin(object):"""Mixin class to implement reading urls from a redis queue."""redis_key = Noneredis_batch_size = Noneredis_encoding = None# Redis client placeholder.server = Nonedef start_requests(self):"""Returns a batch of start requests from redis."""return self.next_requests()def setup_redis(self, crawler=None):"""Setup redis connection and idle signal.This should be called after the spider has set its crawler object."""if self.server is not None:returnif crawler is None:# We allow optional crawler argument to keep backwards# compatibility.# XXX: Raise a deprecation warning.crawler = getattr(self, 'crawler', None)if crawler is None:raise ValueError("crawler is required")settings = crawler.settingsif self.redis_key is None:self.redis_key = settings.get('REDIS_START_URLS_KEY', defaults.START_URLS_KEY,)self.redis_key = self.redis_key % {'name': self.name}if not self.redis_key.strip():raise ValueError("redis_key must not be empty")if self.redis_batch_size is None:# TODO: Deprecate this setting (REDIS_START_URLS_BATCH_SIZE).self.redis_batch_size = settings.getint('REDIS_START_URLS_BATCH_SIZE',settings.getint('CONCURRENT_REQUESTS'),)try:self.redis_batch_size = int(self.redis_batch_size)except (TypeError, ValueError):raise ValueError("redis_batch_size must be an integer")if self.redis_encoding is None:self.redis_encoding = settings.get('REDIS_ENCODING', defaults.REDIS_ENCODING)self.logger.info("Reading start URLs from redis key '%(redis_key)s' ""(batch size: %(redis_batch_size)s, encoding: %(redis_encoding)s",self.__dict__)self.server = connection.from_settings(crawler.settings)# The idle signal is called when the spider has no requests left,# that's when we will schedule new requests from redis queuecrawler.signals.connect(self.spider_idle, signal=signals.spider_idle)def next_requests(self):"""Returns a request to be scheduled or none."""use_set = self.settings.getbool('REDIS_START_URLS_AS_SET', defaults.START_URLS_AS_SET)fetch_one = self.server.spop if use_set else self.server.lpop# XXX: Do we need to use a timeout here?found = 0# TODO: Use redis pipeline execution.while found < self.redis_batch_size:data = fetch_one(self.redis_key)if not data:# Queue empty.breakreq = self.make_request_from_data(data)if req:yield reqfound += 1else:self.logger.debug("Request not made from data: %r", data)if found:self.logger.debug("Read %s requests from '%s'", found, self.redis_key)def make_request_from_data(self, data):"""Returns a Request instance from data coming from Redis.By default, ``data`` is an encoded URL. You can override this method toprovide your own message decoding.Parameters----------data : bytesMessage from redis."""url = bytes_to_str(data, self.redis_encoding)return self.make_requests_from_url(url)def schedule_next_requests(self):"""Schedules a request if available"""# TODO: While there is capacity, schedule a batch of redis requests.for req in self.next_requests():self.crawler.engine.crawl(req, spider=self)def spider_idle(self):"""Schedules a request if available, otherwise waits."""# XXX: Handle a sentinel to close the spider.self.schedule_next_requests()raise DontCloseSpiderclass RedisSpider(RedisMixin, Spider):"""Spider that reads urls from redis queue when idle.Attributes----------redis_key : str (default: REDIS_START_URLS_KEY)Redis key where to fetch start URLs from..redis_batch_size : int (default: CONCURRENT_REQUESTS)Number of messages to fetch from redis on each attempt.redis_encoding : str (default: REDIS_ENCODING)Encoding to use when decoding messages from redis queue.Settings--------REDIS_START_URLS_KEY : str (default: "<spider.name>:start_urls")Default Redis key where to fetch start URLs from..REDIS_START_URLS_BATCH_SIZE : int (deprecated by CONCURRENT_REQUESTS)Default number of messages to fetch from redis on each attempt.REDIS_START_URLS_AS_SET : bool (default: False)Use SET operations to retrieve messages from the redis queue. If False,the messages are retrieve using the LPOP command.REDIS_ENCODING : str (default: "utf-8")Default encoding to use when decoding messages from redis queue."""@classmethoddef from_crawler(self, crawler, *args, **kwargs):obj = super(RedisSpider, self).from_crawler(crawler, *args, **kwargs)obj.setup_redis(crawler)return objclass RedisCrawlSpider(RedisMixin, CrawlSpider):"""Spider that reads urls from redis queue when idle.Attributes----------redis_key : str (default: REDIS_START_URLS_KEY)Redis key where to fetch start URLs from..redis_batch_size : int (default: CONCURRENT_REQUESTS)Number of messages to fetch from redis on each attempt.redis_encoding : str (default: REDIS_ENCODING)Encoding to use when decoding messages from redis queue.Settings--------REDIS_START_URLS_KEY : str (default: "<spider.name>:start_urls")Default Redis key where to fetch start URLs from..REDIS_START_URLS_BATCH_SIZE : int (deprecated by CONCURRENT_REQUESTS)Default number of messages to fetch from redis on each attempt.REDIS_START_URLS_AS_SET : bool (default: True)Use SET operations to retrieve messages from the redis queue.REDIS_ENCODING : str (default: "utf-8")Default encoding to use when decoding messages from redis queue."""@classmethoddef from_crawler(self, crawler, *args, **kwargs):obj = super(RedisCrawlSpider, self).from_crawler(crawler, *args, **kwargs)obj.setup_redis(crawler)return obj

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rulefrom scrapy_redis.spiders import RedisCrawlSpiderclass TestSpider(RedisCrawlSpider):name = 'test'allowed_domains = ['www.test.com']redis_key = 'testspider:start_urls'rules = [# 获取每一页的链接Rule(link_extractor=LinkExtractor(allow=('/?page=d+'))),# 获取每一个公司的详情Rule(link_extractor=LinkExtractor(allow=('/d+')), callback='parse_item')]def parse_item(self, response):······return item