scrapy startproject konachan  scrapy genspider kona konachan.net

# -*- coding: utf-8 -*-# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.htmlimport scrapyclass KonachanItem(scrapy.Item):# define the fields for your item here like:#由于我们使用的是scrapy 自带的图片下载通道，所以url 和图片名称必须按照配置文件设置# 即 Item里面 url 信息设置为 image_urls, 图片名称信息 设置为imagesimage_urls = scrapy.Field()  # 即 Item里面 url 信息设置为 image_urlsimages=scrapy.Field()        # 图片名称信息 设置为imagesdetailed=scrapy.Field()      # 高清页面的url

    def parse(self, response):#获得要爬取的url的相对地址url_list = response.xpath("//div[@class='inner']//a[@class='thumb']/@href").extract()# 获得url的绝对地址for url in url_list:item1=KonachanItem()item1["detailed"] = urlitem1["detailed"] = "http://konachan.net" + item1["detailed"]

next_url= response.xpath("//a[@class='next_page']/@href").extract_first()next_url='http://konachan.net'+ next_url#这里仅仅是测试哈，考虑内存只爬取两页if(self.cnt<2):self.cnt+=1# 回调给自己，继续解析yield scrapy.Request(next_url,callback=self.parse)

    def parse(self, response):#获得要爬取的url的相对地址url_list = response.xpath("//div[@class='inner']//a[@class='thumb']/@href").extract()# 获得url的绝对地址for url in url_list:item1=KonachanItem()item1["detailed"] = urlitem1["detailed"] = "http://konachan.net" + item1["detailed"]#将绝对地址给第二个解析函数 parse_detail 进行解析yield scrapy.Request(item1["detailed"],callback = self.parse_detail,meta={"item":item1}  #meta 以字典的形式传递item信息)

 yield scrapy.Request(item1["detailed"],callback = self.parse_detail,meta={"item":item1}  #meta 以字典的形式传递item信息)

  #定义解析详情页的函数def parse_detail(self,response):#接受parse函数传递的请求，接受item信息item1=response.meta["item"]#解析网页item1["image_urls"] = response.xpath("//div[@class='content']//img/@src").extract()#确定每次都传一个urlif len(item1["image_urls"])<2:item1["image_urls"] = item1["image_urls"]yield item1

# -*- coding: utf-8 -*-# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.htmlimport logging
logger= logging.getLogger(__name__)
class KonachanPipeline(object):def process_item(self, item, spider):logger.warning(item["inner_url"])return item

# -*- coding: utf-8 -*-
import scrapy
#引入定义的item
from  konachan.items import  KonachanItem
# 引入 scrapy 自带的图片爬取通道，注意在setting里面要进行注释。
from  scrapy.pipelines.images import ImagesPipeline

ITEM_PIPELINES = {'konachan.pipelines.KonachanPipeline': 300,# 对图片通道进行注册'scrapy.pipelines.images.ImagesPipeline':299}

MEDIA_NAME = 'image'# Uppercase attributes kept for backward compatibility with code that subclasses# ImagesPipeline. They may be overridden by settings.MIN_WIDTH = 0MIN_HEIGHT = 0EXPIRES = 90THUMBS = {}#这就是之前我们设置item属性为 image_urls以及images的原因：DEFAULT_IMAGES_URLS_FIELD = 'image_urls'DEFAULT_IMAGES_RESULT_FIELD = 'images'def __init__(self, store_uri, download_func=None, settings=None):super(ImagesPipeline, self).__init__(store_uri, settings=settings,download_func=download_func)if isinstance(settings, dict) or settings is None:settings = Settings(settings)resolve = functools.partial(self._key_for_pipe,base_class_name="ImagesPipeline",settings=settings)self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)if not hasattr(self, "IMAGES_RESULT_FIELD"):self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELDif not hasattr(self, "IMAGES_URLS_FIELD"):self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELDself.images_urls_field = settings.get(resolve('IMAGES_URLS_FIELD'),self.IMAGES_URLS_FIELD)self.images_result_field = settings.get(resolve('IMAGES_RESULT_FIELD'),self.IMAGES_RESULT_FIELD)self.min_width = settings.getint(resolve('IMAGES_MIN_WIDTH'), self.MIN_WIDTH)self.min_height = settings.getint(resolve('IMAGES_MIN_HEIGHT'), self.MIN_HEIGHT)self.thumbs = settings.get(resolve('IMAGES_THUMBS'), self.THUMBS)

import  os
BOT_NAME = 'konachan'SPIDER_MODULES = ['konachan.spiders']
NEWSPIDER_MODULE = 'konachan.spiders'
# 定义log等级，只显示warning
LOG_LEVEL="WARNING"project_dir=os.path.abspath(os.path.dirname(__file__))  #获取当前爬虫项目的绝对路径
IMAGES_STORE = os.path.join(project_dir,'images')  #组装新的图片路径IMAGES_URLS_FIELD='image_urls'  #图片使用的字段# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/66.0'# Obey robots.txt rules
ROBOTSTXT_OBEY = True