scrapy startproject weibo #创建工程
scrapy genspider -t basic weibo.com weibo.com #创建spider

import scrapyclass WeiboItem(scrapy.Item):# define the fields for your item here like:image_urls = scrapy.Field()dirname = scrapy.Field()

# -*- coding: utf-8 -*-# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import hashlib
# from scrapy.contrib.pipeline.images import ImagesPipeline
# from scrapy.http import Request
from scrapy.utils.python import to_bytes
import urllib
import os
import redis#Scrapy 自带图片下载器，对gif动图支持不好，原因没找，所以为支持gif动图，自定义下载器
# class WeboPipeline(ImagesPipeline):
#     # def process_item(self, item, spider):
#     #     return item
#     def get_media_requests(self, item, info):
#         for image_url in item['image_urls']:
#             request =  Request('http:'+image_url)
#             request.meta['item'] = {'dirname':item['dirname']}
#             yield request#定义存储目录和文件扩展名
#     def file_path(self, request, response=None, info=None):
#         url = request.url
#         item = request.meta['item']
#         image_guid = hashlib.sha1(to_bytes(url)).hexdigest()
#         ext = url.split('.')[-1]
#         url = 'full/%s/%s.%s' % (item['dirname'],image_guid,ext)
#         return url #     def item_completed(self, results, item, info):
#         return item#没有用原生的Scrapy文件下载功能，因为gif图片下载不全，不能动
class WeboPipeline(object):#用redis判重，简单粗暴了点，def open_spider(self,spider):self.client=redis.Redis(host='127.0.0.1', port=6379)def process_item(self, item, spider):file_path = item['dirname']#redis判重复yn = self.client.get(file_path)if yn is not None:print '已经下载过'return itemfor image_url in item['image_urls']:imageurl = 'http:'+image_urlsavepath = self.get_file_path(file_path,imageurl)print imageurl,savepathtry:#下载图片到指定地址urllib.urlretrieve(imageurl,savepath)except Exception as e:print str(e)#根据微博内容中每组图片地址的hash做唯一标识self.client.set(file_path,1)return itemdef get_file_path(self,dirname, url):```获取新的文件名```image_guid = hashlib.sha1(to_bytes(url)).hexdigest()ext = url.split('.')[-1]#文件存储目录写死了，可在setting中设置file_dir = './full/%s'%(dirname)if os.path.exists(file_dir) == False:os.makedirs(file_dir)return '%s/%s.%s' % (file_dir,image_guid,ext)

# -*- coding: utf-8 -*-
import scrapy
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import json
import os
import time
from collections import defaultdict
from scrapy.selector import Selector
import re
import urllib
from weibo.items import WeiboItem
import time
import hashlibclass WebComSpider(scrapy.Spider):name = 'weibo.com'allowed_domains = ['weibo.com']start_urls = ['https://weibo.com/']#cookie存储地址cookie_file_path = './cookies.json'#要抓取博主的IDuids = ['5139152205']def saveCookie(self,cookie):```保存cookie到文件```with open(self.cookie_file_path,'w') as outputfile:json.dump(cookie, outputfile)def getCookie(self):```从文件中获取cookie```if os.path.exists(self.cookie_file_path) == False:self.cookie = Nonereturnwith open(self.cookie_file_path,'r') as inputfile:self.cookie = json.load(inputfile)def start_requests(self):```抓取微博```self.getCookie()#如果没有cookie,模拟登陆获取cookieif self.cookie is None:#用PhantomJS模拟浏览器driver = webdriver.PhantomJS(executable_path='/data/software/phantomjs-2.1.1-linux-x86_64/bin/phantomjs')driver.get('https://weibo.com') try:#设置窗口大小，这个很重要，如果不设置，获取不到下面的元素driver.set_window_size(1920,1080)#等待获取到 loginname 对象后才会进行下一步操作userElen = WebDriverWait(driver,10).until(EC.presence_of_element_located((By.ID,'loginname')))#等待时间，反扒time.sleep(3);print 'sleep 3'#设置登陆用户名userElen.send_keys('登陆用户名')print 'sleep 5'time.sleep(5)pasElen = driver.find_element_by_xpath('//*[@id="pl_login_form"]/div/div[3]/div[2]/div/input') #设置登陆密码pasElen.send_keys('登陆密码')print 'sleep 1'time.sleep(1)sumbButton =  driver.find_element_by_xpath('//*[@id="pl_login_form"]/div/div[3]/div[6]/a')#登陆sumbButton.click()#当页面title中包含我的首页之后，表示登陆成功，绩效下一步element = WebDriverWait(driver,10).until(EC.title_contains(u'我的首页'))except Exception as e:print '22222222222222222',str(e)#获取cookieck = driver.get_cookies()self.cookie = defaultdict()for item in ck:self.cookie[item['name']] = item['value']#保存cookieself.saveCookie(self.cookie)#用得到的cookie抓取你想要的博主的图片微博for uid in self.uids:#博主带图片的微博列表，根据具体需求自己定义url = 'https://weibo.com/u/%s?profile_ftype=1&is_pic=1#_0' %(uid,)request = scrapy.Request(url=url,cookies=self.cookie,callback=self.parse)request.meta['item'] =  {'uid':uid,'page_num':1}yield requestdef parse(self,response):```解析页面，因为微博采用页面跳转和Ajax两种翻页模式，每次页面跳转之后都会有两次Ajax请求获取数据，微博页面全部由js渲染html字符生成，所以没法用xpath,css选择器，只能采用正则查找方法找到自己想要的内容```title = response.xpath('//title/text()').extract()[0]print titleseletor = Selector(text=response.body)#获取Ajax请求参数pageId = seletor.re(r"\$CONFIG\[\'page_id\'\]=\'(\d+)\'")[0]domain = seletor.re(r"\$CONFIG\[\'domain\'\]=\'(\d+)\'")[0]#分析页面跳转后html页面内容for itemObj in self.parse_content(seletor):yield itemObj#Ajax请求数据item = response.meta['item']ajaxUrl = 'https://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=%s&profile_ftype=1&is_pic=1&pagebar=%s&pl_name=Pl_Official_MyProfileFeed__21&id=%s&script_uri=/u/%s&feed_type=0&page=%s&pre_page=1&domain_op=%s&__rnd=%s'for num in range(0,2):rand = str(int(time.time()*1000))url = ajaxUrl%(domain,num,pageId,item['uid'],item['page_num'],domain,rand)print urlprint '------------sleep 10------------'time.sleep(10)yield scrapy.Request(url=url,cookies=self.cookie,callback=self.parse_ajax)item['page_num'] +=  1nexpage = 'https://weibo.com/u/%s?is_search=0&visible=0&is_pic=1&is_tag=0&profile_ftype=1&page=%s#feedtop'%(item['uid'],item['page_num'])request = scrapy.Request(url = nexpage,cookies=self.cookie,callback=self.parse)request.meta['item'] = itemyield requestdef parse_ajax(self,response):```解析Ajax内容```bodyObj = json.loads(response.body)seletor = Selector(text=bodyObj['data'])for itemObj in self.parse_content(seletor):yield itemObjdef parse_content(self,seletor):```获取图片地址```pre = re.compile(r'clear_picSrc=(.*?)[\&|\\"]')imagelist =  seletor.re(pre)for row in imagelist:hs = hashlib.md5()hs.update(row)row = urllib.unquote(row)#用每组图片的地址做唯一标识，和子目录名imgset = row.split(',')yield WeiboItem(image_urls=imgset,dirname=hs.hexdigest())

ROBOTSTXT_OBEY = False #不遵循robots规则
注册 WeiboPipeline
ITEM_PIPELINES = {'webo.pipelines.WeiboPipeline': 300,
}

scrapy crawl weibo.com