#启动图片管道
ITEM_PIPELINES = {'mztu.pipelines.ImagesPipelinse': 300,
}
#设置默认目录地址  注意下载图片的话默认地址必须设置！！！
IMAGES_STORE = "E:\study\Python\scrapy\mztu\imges"
#设置图片通道失效时间
IMAGES_EXPIRES =90
#缩略图生成
#IMAGES_THUMBS = {#   'small': (50, 50),
#    'big': (270, 270),
#}

# -*- coding: utf-8 -*-
import scrapy
from mztu.items import MztuItemclass ZimdgSpider(scrapy.Spider):name = 'zimdg'allowed_domains = ['mzitu.com']#生成链接列表start_urls = ['http://www.mzitu.com/xinggan/page/{}/'.format(str(x)) for x in range(118)]def parse(self, response):#解析出链接set_li = response.xpath("//div[@class='postlist']/ul/li")for ecth in set_li:ed = ecth.xpath('./a/@href').extract()#进行二次分类解析yield scrapy.Request(ed[0],callback=self.parse_item)def parse_item(self,response):itme = MztuItem()# 获取页数链接进行访问offset = int(response.xpath('//div[@class="pagenavi"]/a/span/text()')[4].extract())#生成链接访问#遍历链接访问for i in [response.url+"/{}".format(str(x))  for x in range(1,offset+1)]:itme['Referer']=i#将meta传入链接yield scrapy.Request(itme['Referer'],meta={'meta_1':itme}, callback=self.parse_ponse)# for i in url:def parse_ponse(self,response):#获取itme资源itme = response.meta['meta_1']#获取图片地址imgs = response.xpath('//div[@class="main-image"]/p/a/img/@src')[0].extract()#获取图片目录title = response.xpath('//div[@class="main-image"]/p/a/img/@alt')[0].extract()itme["title"]= titleitme["imge_url"]= imgs#itme["nickname"] = itme["Referer"][itme["Referer"].rfind("/"):]+itme["imge_url"][itme["imge_url"].rfind('/')+1:itme["imge_url"].rfind('.')]#itme["nickname"] = itme["imge_url"][itme["imge_url"].rfind('/')+1:itme["imge_url"].rfind('.')]yield itme

import scrapyclass MztuItem(scrapy.Item):#目录title = scrapy.Field()#图片地址imge_url = scrapy.Field()#请求头Referer = scrapy.Field()image_Path = scrapy.Field()#图片名称# nickname = scrapy.Field()

# -*- coding: utf-8 -*-# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
# 导入这个包为了移动文件
import shutil
#此包不解释
import scrapy
# 导入项目设置
from scrapy.utils.project import get_project_settings
# 导入scrapy框架的图片下载类
from scrapy.pipelines.images import ImagesPipeline
#此包不解释
import osclass ImagesPipelinse(ImagesPipeline):#def process_item(self, item, spider):#    return item# 获取settings文件里设置的变量值IMAGES_STORE = get_project_settings().get("IMAGES_STORE")# 重写ImagesPipeline类的此方法# 发送图片下载请求def get_media_requests(self, item, info):image_url = item["imge_url"]#headers是请求头主要是防反爬虫yield scrapy.Request(image_url,headers={'Referer':item['Referer']})def item_completed(self, result, item, info):image_path = [x["path"] for ok, x in result if ok]# 定义分类保存的路径img_path = "%s\%s" % (self.IMAGES_STORE, item['title'])# 目录不存在则创建目录if os.path.exists(img_path) == False:os.mkdir(img_path)# 将文件从默认下路路径移动到指定路径下shutil.move(self.IMAGES_STORE + "\\" +image_path[0], img_path + "\\" +image_path[0][image_path[0].find("full\\")+6:])item['image_Path'] = img_path + "\\" + image_path[0][image_path[0].find("full\\")+6:]return item