ROBOTSTXT_OBEY = False #不遵寻reboot协议

DOWNLOAD_DELAY = 2 #下载延迟时间
DOWNLOAD_TIMEOUT = 15 #下载超时时间
COOKIES_ENABLED = False #禁用cookie

DOWNLOADER_MIDDLEWARES = {   #请求头   'jandan.middlewares.RandomUserAgent': 100,   #代理ip   'jandan.middlewares.RandomProxy': 200,}

#请求列表USER_AGENTS = [   #遨游   "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",   #火狐   "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",   #谷歌   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"]

#代理ip列表PROXIES = [   {"ip_port":"119.177.90.103:9999","user_passwd":""},   #代理ip无密码   {"ip_port":"101.132.122.230:3128","user_passwd":""},   #代理ip有密码   # {"ip_port":"123.139.56.238:9999","user_passwd":"root:admin"}]

#管道文件，取消注释ITEM_PIPELINES = {   'jandan.pipelines.JandanPipeline': 300,}

IMAGES_STORE = "images"

import random
import base64
from jandan.settings import USER_AGENTS
from jandan.settings import PROXIESclass RandomUserAgent(object):def process_request(self,request,spider):useragent = random.choice(USER_AGENTS)request.headers.setdefault("User-Agent",useragent)class RandomProxy(object):def process_request(self,request,spider):proxy = random.choice(PROXIES)if proxy["user_passwd"] is None:request.meta["proxy"] = "http://" + proxy["ip_port"]else:# b64编码接收字节对象,在py3中str是unicode，需要转换,返回是字节对象base64_userpasswd = base64.b16encode(proxy["user_passwd"].encode())request.meta["proxy"] = "http://" + proxy["ip_port"]#拼接是字符串，需要转码request.headers["Proxy-Authorization"] = "Basic " + base64_userpasswd.decode()

import scrapyclass JandanItem(scrapy.Item):name = scrapy.Field()url = scrapy.Field()

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from jandan.items import JandanItem
from selenium import webdriver
from bs4 import BeautifulSoup as bs4class JdSpider(CrawlSpider):name = 'jd'allowed_domains = ['jandan.net']start_urls = ['http://jandan.net/pic/page-1#comments/']rules = (Rule(LinkExtractor(allow=r'pic/page-\d+'), callback='parse_item', follow=True),)def parse_item(self, response):item = JandanItem()driver = webdriver.PhantomJS()driver.get(response.url)soup = bs4(driver.page_source, 'html.parser')all_data = soup.find_all('div', {'class': 'row'})for i in all_data:name = i.find("strong")item["name"] = name.get_text().strip()link = i.find('a', {'class': 'view_img_link'})url = link.get("href")if len(url) == 0:returnitem["url"] = "http://" + url.split("//")[-1]yield item

import json
import os
import requests
from scrapy.conf import settingsclass JandanPipeline(object):　　　#保存为json文件# def __init__(self):#     self.filename = open("jandan.json","wb")#     self.num = 0# # def process_item(self, item, spider):#     text = json.dumps(dict(item),ensure_ascii=False) + "\n"#     self.filename.write(text.encode("utf-8"))#     self.num += 1#     return item# # def close_spider(self,spider):#     self.filename.close()#     print("总共有" + str(self.num) + "个资源")　　#下载到本地def process_item(self, item, spider):if 'url' in item:dir_path = settings["IMAGES_STORE"]if not os.path.exists(dir_path):os.makedirs(dir_path)su = "." + item["url"].split(".")[-1]path = item["name"] + sunew_path = '%s/%s' % (dir_path, path)if not os.path.exists(new_path):with open(new_path, 'wb') as handle:response = requests.get(item["url"], stream=True)for block in response.iter_content(1024):if not block:breakhandle.write(block)return item