# -*- coding: utf-8 -*-
import scrapy
from bs4 import BeautifulSoup
from scrapy.selector import HtmlXPathSelector
from ..items import SqlItemclass ChoutiSpider(scrapy.Spider):name = 'chouti'allowed_domains = ['chouti.com']start_urls = ['http://chouti.com/']def parse(self, response):# print('已经下载完成',response.text)# 获取页面上所有的新闻，将标题和URL写入文件"""方法一：soup = BeautifulSoup(response.text,'lxml')tag = soup.find(name='div',id='content-list')# tag = soup.find(name='div', attrs={'id':'content_list})"""# 方法二："""// 表示从跟开始向下找标签//div[@id="content-list"]//div[@id="content-list"]/div[@class="item"].// 相对当前对象"""hxs = HtmlXPathSelector(response)tag_list = hxs.select('//div[@id="content-list"]/div[@class="item"]')for item in tag_list:text1 = (item.select('.//div[@class="part1"]/a/text()')).extract_first().strip()url1 = (item.select('.//div[@class="part1"]/a/@href')).extract_first()yield SqlItem(text=text1,url=url1)

import scrapyclass SqlItem(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()text = scrapy.Field()url = scrapy.Field()

class DbPipeline(object):def process_item(self, item, spider):# print('数据DbPipeline',item)return itemclass FilePipeline(object):def open_spider(self, spider):"""爬虫开始执行时，调用:param spider::return:"""print('开始>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')self.f = open('123.txt','a+',encoding='utf-8')def process_item(self, item, spider):self.f.write(item['text']+'\n')self.f.write(item['url']+'\n')self.f.flush()return itemdef close_spider(self, spider):"""爬虫关闭时，被调用:param spider::return:"""self.f.close()print('结束>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')

#建议300-1000,谁小谁先执行
ITEM_PIPELINES = {'sql.pipelines.DbPipeline': 300,'sql.pipelines.FilePipeline': 400,
}

from scrapy.exceptions import DropItemclass CustomPipeline(object):def __init__(self,v):self.value = vdef process_item(self, item, spider):# 操作并进行持久化# return表示会被后续的pipeline继续处理return item# 表示将item丢弃，不会被后续pipeline处理# raise DropItem()
@classmethoddef from_crawler(cls, crawler):"""初始化时候，用于创建pipeline对象:param crawler: :return: """val = crawler.settings.getint('MMMM')return cls(val)def open_spider(self,spider):"""爬虫开始执行时，调用:param spider: :return: """print('000000')def close_spider(self,spider):"""爬虫关闭时，被调用:param spider: :return: """print('111111')

class DBPipeline(object):def process_item(self, item, spider):#print('DBPipeline',item)raise DropItem()        

# -*- coding: utf-8 -*-
import scrapy
from bs4 import BeautifulSoup
from scrapy.selector import HtmlXPathSelector
from ..items import SqlItem
from scrapy.http import Requestclass ChoutiSpider(scrapy.Spider):name = 'chouti'allowed_domains = ['chouti.com']start_urls = ['http://chouti.com/']def parse(self, response):# print('已经下载完成',response.text)# 1.获取页面上所有的新闻，将标题和URL写入文件"""方法一：soup = BeautifulSoup(response.text,'lxml')tag = soup.find(name='div',id='content-list')# tag = soup.find(name='div', attrs={'id':'content_list})"""# 方法二："""// 表示从跟开始向下找标签//div[@id="content-list"]//div[@id="content-list"]/div[@class="item"].// 相对当前对象"""hxs = HtmlXPathSelector(response)tag_list = hxs.select('//div[@id="content-list"]/div[@class="item"]')for item in tag_list:text1 = (item.select('.//div[@class="part1"]/a/text()')).extract_first().strip()url1 = (item.select('.//div[@class="part1"]/a/@href')).extract_first()yield SqlItem(text=text1,url=url1)# 2.找到所有页码，访问页码，页码下载完成后，继续执行持久化的逻辑+继续找页码page_list = hxs.select('//div[@id="dig_lcpage"]//a/@href').extract()# page_list = hxs.select('//div[@id="dig_lcpage"]//a[re:test(@href,"/all/hot/recent/\d+")]/@href').extract()base_url = "https://dig.chouti.com/{0}"for page in page_list:url = base_url.format(page)yield Request(url=url,callback=self.parse)

class RepeatUrl:def __init__(self):self.visited_url = set()@classmethoddef from_settings(cls, settings):"""初始化时，调用:param settings: :return: """return cls()def request_seen(self, request):"""检测当前请求是否已经被访问过:param request: :return: True表示已经访问过；False表示未访问过"""if request.url in self.visited_url:return Trueself.visited_url.add(request.url)return Falsedef open(self):"""开始爬去请求时，调用:return: """print('open replication')def close(self, reason):"""结束爬虫爬取时，调用:param reason: :return: """print('close replication')def log(self, request, spider):"""记录日志:param request: :param spider: :return: """pass

# 自定义过滤规则
DUPEFILTER_CLASS = 'spl.new.RepeatUrl'

cookie_dict = {}has_request_set = {}def start_requests(self):url = 'http://dig.chouti.com/'yield Request(url=url, callback=self.login)def login(self, response):# 去响应头中获取cookiecookie_jar = CookieJar()cookie_jar.extract_cookies(response, response.request)for k, v in cookie_jar._cookies.items():for i, j in v.items():for m, n in j.items():self.cookie_dict[m] = n.value

req = Request(url='http://dig.chouti.com/login',method='POST',headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'},body='phone=86138xxxxxxxx&password=xxxxxxxxx&oneMonth=1',cookies=self.cookie_dict,# 未认证cookie带过去callback=self.check_login)yield req#需要登录帐号

# -*- coding: utf-8 -*-
import scrapy
from scrapy.selector import HtmlXPathSelector
from scrapy.http.request import Request
from scrapy.http.cookies import CookieJar
from scrapy import FormRequestclass ChouTiSpider(scrapy.Spider):# 爬虫应用的名称，通过此名称启动爬虫命令name = "chouti"# 允许的域名allowed_domains = ["chouti.com"]cookie_dict = {}has_request_set = {}def start_requests(self):url = 'http://dig.chouti.com/'yield Request(url=url, callback=self.login)def login(self, response):# 去响应头中获取cookiecookie_jar = CookieJar()cookie_jar.extract_cookies(response, response.request)for k, v in cookie_jar._cookies.items():for i, j in v.items():for m, n in j.items():self.cookie_dict[m] = n.value# 未认证cookiereq = Request(url='http://dig.chouti.com/login',method='POST',headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'},body='phone=86138xxxxxxxx&password=xxxxxx&oneMonth=1',cookies=self.cookie_dict,# 未认证cookie带过去callback=self.check_login)yield reqdef check_login(self, response):print('登录成功，返回code:9999',response.text)req = Request(url='http://dig.chouti.com/',method='GET',callback=self.show,cookies=self.cookie_dict,dont_filter=True)yield reqdef show(self, response):# print(response)hxs = HtmlXPathSelector(response)news_list = hxs.select('//div[@id="content-list"]/div[@class="item"]')for new in news_list:# temp = new.xpath('div/div[@class="part2"]/@share-linkid').extract()link_id = new.xpath('*/div[@class="part2"]/@share-linkid').extract_first()# 赞# yield Request(#     url='http://dig.chouti.com/link/vote?linksId=%s' %(link_id,),#     method='POST',#     cookies=self.cookie_dict,#     callback=self.result# )# 取消赞yield Request(url='https://dig.chouti.com/vote/cancel/vote.do',method='POST',headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'},body='linksId=%s' %(link_id),cookies=self.cookie_dict,callback=self.result)page_list = hxs.select('//div[@id="dig_lcpage"]//a[re:test(@href, "/all/hot/recent/\d+")]/@href').extract()for page in page_list:page_url = 'http://dig.chouti.com%s' % page"""手动去重import hashlibhash = hashlib.md5()hash.update(bytes(page_url,encoding='utf-8'))key = hash.hexdigest()if key in self.has_request_set:passelse:self.has_request_set[key] = page_url"""yield Request(url=page_url,method='GET',callback=self.show)def result(self, response):print(response.text)

class SqlDownloaderMiddleware(object):def process_request(self, request, spider):# Called for each request that goes through the downloader# middleware.# Must either:# - return None: continue processing this request# - or return a Response object# - or return a Request object# - or raise IgnoreRequest: process_exception() methods of#   installed downloader middleware will be calledprint('中间件>>>>>>>>>>>>>>>>',request)# 对所有请求做统一操作# 1. 请求头处理request.headers['User-Agent'] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36"# 2. 添加代理# request.headers['asdfasdf'] = "asdfadfasdf"def process_response(self, request, response, spider):# Called with the response returned from the downloader.# Must either;# - return a Response object# - return a Request object# - or raise IgnoreRequestreturn responsedef process_exception(self, request, exception, spider):# Called when a download handler or a process_request()# (from other downloader middleware) raises an exception.# Must either:# - return None: continue processing this exception# - return a Response object: stops process_exception() chain# - return a Request object: stops process_exception() chainpass

DOWNLOADER_MIDDLEWARES = {'sql.middlewares.SqlDownloaderMiddleware': 543,
}

import random
import base64
import sixdef to_bytes(text, encoding=None, errors='strict'):if isinstance(text, bytes):return textif not isinstance(text, six.string_types):raise TypeError('to_bytes must receive a unicode, str or bytes ''object, got %s' % type(text).__name__)if encoding is None:encoding = 'utf-8'return text.encode(encoding, errors)class ProxyMiddleware(object):def process_request(self, request, spider):PROXIES = [{'ip_port': '111.11.228.75:80', 'user_pass': ''},{'ip_port': '120.198.243.22:80', 'user_pass': ''},{'ip_port': '111.8.60.9:8123', 'user_pass': ''},{'ip_port': '101.71.27.120:80', 'user_pass': ''},{'ip_port': '122.96.59.104:80', 'user_pass': ''},{'ip_port': '122.224.249.122:8088', 'user_pass': ''},]proxy = random.choice(PROXIES)if proxy['user_pass'] is not None:request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])encoded_user_pass = base64.encodestring(to_bytes(proxy['user_pass']))request.headers['Proxy-Authorization'] = to_bytes('Basic ' + encoded_user_pass)else:request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])

DOWNLOADER_MIDDLEWARES = {'spl.pronxies.ProxyMiddleware':600,
}

from scrapy.commands import ScrapyCommand
from scrapy.utils.project import get_project_settingsclass Command(ScrapyCommand):requires_project = Truedef syntax(self):return '[options]'def short_desc(self):return 'Runs all of the spiders'def run(self, args, opts):spider_list = self.crawler_process.spiders.list()for name in spider_list:self.crawler_process.crawl(name, **opts.__dict__)self.crawler_process.start()