1 from scrapy import cmdline
2
3 cmdline.execute("scrapy crawl Meizitu".split())

 1 BOT_NAME = 'CrawlMeiziTu'
 2
 3 SPIDER_MODULES = ['CrawlMeiziTu.spiders']
 4 NEWSPIDER_MODULE = 'CrawlMeiziTu.spiders'
 5 ITEM_PIPELINES = {
 6    'CrawlMeiziTu.pipelines.CrawlmeizituPipeline': 300,
 7 }
 8 IMAGES_STORE = 'D://pic2'
 9 DOWNLOAD_DELAY = 0.3
10
11 USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'
12 ROBOTSTXT_OBEY = True

# -*- coding: utf-8 -*-# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.htmlimport scrapyclass CrawlmeizituItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()#title为文件夹名字title = scrapy.Field()url = scrapy.Field()tags = scrapy.Field()#图片的连接src = scrapy.Field()#alt为图片名字alt = scrapy.Field()

# -*- coding: utf-8 -*-
import os
import requests
from CrawlMeiziTu.settings import IMAGES_STOREclass CrawlmeizituPipeline(object):def process_item(self, item, spider):fold_name = "".join(item['title'])header = {'USER-Agent': 'User-Agent:Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36','Cookie': 'b963ef2d97e050aaf90fd5fab8e78633',#需要查看图片的cookie信息，否则下载的图片无法查看
        }images = []# 所有图片放在一个文件夹下dir_path = '{}'.format(IMAGES_STORE)if not os.path.exists(dir_path) and len(item['src']) != 0:os.mkdir(dir_path)if len(item['src']) == 0:with open('..//check.txt', 'a+') as fp:fp.write("".join(item['title']) + ":" + "".join(item['url']))fp.write("\n")for jpg_url, name, num in zip(item['src'], item['alt'],range(0,100)):file_name = name + str(num)file_path = '{}//{}'.format(dir_path, file_name)images.append(file_path)if os.path.exists(file_path) or os.path.exists(file_name):continuewith open('{}//{}.jpg'.format(dir_path, file_name), 'wb') as f:req = requests.get(jpg_url, headers=header)f.write(req.content)return item

# -*- coding: utf-8 -*-
import scrapy
from CrawlMeiziTu.items import CrawlmeizituItem
#from CrawlMeiziTu.items import CrawlmeizituItemPage
import time
class MeizituSpider(scrapy.Spider):name = "Meizitu"#allowed_domains = ["meizitu.com/"]
start_urls = []last_url = []with open('..//url.txt', 'r') as fp:crawl_urls = fp.readlines()for start_url in crawl_urls:last_url.append(start_url.strip('\n'))start_urls.append("".join(last_url[-1]))def parse(self, response):selector = scrapy.Selector(response)#item = CrawlmeizituItemPage()
next_pages = selector.xpath('//*[@id="wp_page_numbers"]/ul/li/a/@href').extract()next_pages_text = selector.xpath('//*[@id="wp_page_numbers"]/ul/li/a/text()').extract()all_urls = []if '下一页' in next_pages_text:next_url = "http://www.meizitu.com/a/{}".format(next_pages[-2])with open('..//url.txt', 'a+') as fp:fp.write('\n')fp.write(next_url)fp.write("\n")request = scrapy.http.Request(next_url, callback=self.parse)time.sleep(2)yield requestall_info = selector.xpath('//h3[@class="tit"]/a')#读取每个图片夹的连接for info in all_info:links = info.xpath('//h3[@class="tit"]/a/@href').extract()for link in links:request = scrapy.http.Request(link, callback=self.parse_item)time.sleep(1)yield request# next_link = selector.xpath('//*[@id="wp_page_numbers"]/ul/li/a/@href').extract()# next_link_text = selector.xpath('//*[@id="wp_page_numbers"]/ul/li/a/text()').extract()# if '下一页' in next_link_text:#     nextPage = "http://www.meizitu.com/a/{}".format(next_link[-2])#     item['page_url'] = nextPage#     yield item#抓取每个文件夹的信息def parse_item(self, response):item = CrawlmeizituItem()selector = scrapy.Selector(response)image_title = selector.xpath('//h2/a/text()').extract()image_url = selector.xpath('//h2/a/@href').extract()image_tags = selector.xpath('//div[@class="metaRight"]/p/text()').extract()if selector.xpath('//*[@id="picture"]/p/img/@src').extract():image_src = selector.xpath('//*[@id="picture"]/p/img/@src').extract()else:image_src = selector.xpath('//*[@id="maincontent"]/div/p/img/@src').extract()if selector.xpath('//*[@id="picture"]/p/img/@alt').extract():pic_name = selector.xpath('//*[@id="picture"]/p/img/@alt').extract()else:pic_name = selector.xpath('//*[@id="maincontent"]/div/p/img/@alt').extract()#//*[@id="maincontent"]/div/p/img/@altitem['title'] = image_titleitem['url'] = image_urlitem['tags'] = image_tagsitem['src'] = image_srcitem['alt'] = pic_nameprint(item)time.sleep(1)yield item