python16_day37【爬虫2】

一、异步非阻塞

　　1.自定义异步非阻塞

 1 import socket
 2 import select
 3
 4 class Request(object):
 5     def __init__(self,sock,func,url):
 6         self.sock = sock
 7         self.func = func
 8         self.url = url
 9
10     def fileno(self):
11         return self.sock.fileno()
12
13 def async_request(url_list):
14
15     input_list = []
16     conn_list = []
17
18     for url in url_list:
19         client = socket.socket()
20         client.setblocking(False)
21         # 创建连接,不阻塞
22         try:
23             client.connect((url[0],80,)) # 100个向百度发送的请求
24         except BlockingIOError as e:
25             pass
26
27         obj = Request(client,url[1],url[0])
28
29         input_list.append(obj)
30         conn_list.append(obj)
31
32     while True:
33         # 监听socket是否已经发生变化 [request_obj,request_obj....request_obj]
34         # 如果有请求连接成功：wlist = [request_obj,request_obj]
35         # 如果有响应的数据：  rlist = [request_obj,request_obj....client100]
36         rlist,wlist,elist = select.select(input_list,conn_list,[],0.05)
37         for request_obj in wlist:
38             # print('连接成功')
39             # # # # 发送Http请求
40             # print('发送请求')
41             request_obj.sock.sendall("GET / HTTP/1.0\r\nhost:{0}\r\n\r\n".format(request_obj.url).encode('utf-8'))
42             conn_list.remove(request_obj)
43
44         for request_obj in rlist:
45             data = request_obj.sock.recv(8096)
46             request_obj.func(data)
47             request_obj.sock.close()
48             input_list.remove(request_obj)
49
50         if not input_list:
51             break

　　2. 调用上面自定义模块

 1 import s2
 2
 3 def callback1(data):
 4     print('百度回来了',data)
 5
 6 def callback2(data):
 7     print('必应回来了',data)
 8
 9 url_list = [
10     ['www.baidu.com',callback1],
11     ['www.bing.com',callback2]
12 ]
13 s2.async_request(url_list)
14
15 # #################################  twisted #################################
16 from twisted.web.client import getPage, defer
17 from twisted.internet import reactor
18
19
20 def all_done(arg):
21     reactor.stop()
22
23
24 def callback1(contents):
25     print(contents)
26
27 def callback2(contents):
28     print(contents)
29 deferred_list = []
30
31 url_list = [
32     ('http://www.bing.com',callback1),
33     ('http://www.baidu.com',callback2)
34 ]
35 for url in url_list:
36     deferred = getPage(bytes(url[0], encoding='utf8'))
37     deferred.addCallback(url[1])
38     deferred_list.append(deferred)
39
40 dlist = defer.DeferredList(deferred_list)
41 dlist.addBoth(all_done)

二、scrapy

　　1. 安装

pip3 install scrapy

　　2. 基本命令

1. scrapy startproject p1
2. scrapy genspider chouti chouti.com
3. scrapy list
4. scrapy crawl chouti

　　3. 项目文件介绍

scrapy.cfg  项目的主配置信息。（真正爬虫相关的配置信息在settings.py文件中）
items.py    设置数据存储模板，用于结构化数据，如：Django的Model
pipelines    数据处理行为，如：一般结构化的数据持久化
settings.py 配置文件，如：递归的层数、并发数，延迟下载等
spiders      爬虫目录，如：创建文件，编写爬虫规则

　　4. 小试牛刀

 1 import scrapy
 2 from scrapy.selector import HtmlXPathSelector
 3 from scrapy.http.request import Request
 4
 5
 6 class DigSpider(scrapy.Spider):
 7     # 爬虫应用的名称，通过此名称启动爬虫命令
 8     name = "dig"
 9
10     # 允许的域名
11     allowed_domains = ["chouti.com"]
12
13     # 起始URL
14     start_urls = [
15         'http://dig.chouti.com/',
16     ]
17
18     has_request_set = {}
19
20     def parse(self, response):
21         print(response.url)
22
23         hxs = HtmlXPathSelector(response)
24         page_list = hxs.select('//div[@id="dig_lcpage"]//a[re:test(@href, "/all/hot/recent/\d+")]/@href').extract()
25         for page in page_list:
26             page_url = 'http://dig.chouti.com%s' % page
27             key = self.md5(page_url)
28             if key in self.has_request_set:
29                 pass
30             else:
31                 self.has_request_set[key] = page_url
32                 obj = Request(url=page_url, method='GET', callback=self.parse)
33                 yield obj
34
35     @staticmethod
36     def md5(val):
37         import hashlib
38         ha = hashlib.md5()
39         ha.update(bytes(val, encoding='utf-8'))
40         key = ha.hexdigest()
41         return key

View Code

　　运行：scrapy crawl chouti.com --nolog

　　5. 选择器

 1 from scrapy.selector import Selector, HtmlXPathSelector
 2 from scrapy.http import HtmlResponse
 3 html = """<!DOCTYPE html>
 4 <html>
 5     <head lang="en">
 6         <meta charset="UTF-8">
 7         <title></title>
 8     </head>
 9     <body>
10         <ul>
11             <li class="item-"><a id='i1' href="link.html">first item</a></li>
12             <li class="item-0"><a id='i2' href="llink.html">first item</a></li>
13             <li class="item-1"><a href="llink2.html">second item<span>vv</span></a></li>
14         </ul>
15         <div><a href="llink2.html">second item</a></div>
16     </body>
17 </html>
18 """
19 response = HtmlResponse(url='http://example.com', body=html,encoding='utf-8')
20 # hxs = HtmlXPathSelector(response)
21 # print(hxs)
22 # hxs = Selector(response=response).xpath('//a')
23 # print(hxs)
24 # hxs = Selector(response=response).xpath('//a[2]')
25 # print(hxs)
26 # hxs = Selector(response=response).xpath('//a[@id]')
27 # print(hxs)
28 # hxs = Selector(response=response).xpath('//a[@id="i1"]')
29 # print(hxs)
30 # hxs = Selector(response=response).xpath('//a[@href="link.html"][@id="i1"]')
31 # print(hxs)
32 # hxs = Selector(response=response).xpath('//a[contains(@href, "link")]')
33 # print(hxs)
34 # hxs = Selector(response=response).xpath('//a[starts-with(@href, "link")]')
35 # print(hxs)
36 # hxs = Selector(response=response).xpath('//a[re:test(@id, "i\d+")]')
37 # print(hxs)
38 # hxs = Selector(response=response).xpath('//a[re:test(@id, "i\d+")]/text()').extract()
39 # print(hxs)
40 # hxs = Selector(response=response).xpath('//a[re:test(@id, "i\d+")]/@href').extract()
41 # print(hxs)
42 # hxs = Selector(response=response).xpath('/html/body/ul/li/a/@href').extract()
43 # print(hxs)
44 # hxs = Selector(response=response).xpath('//body/ul/li/a/@href').extract_first()
45 # print(hxs)
46
47 # ul_list = Selector(response=response).xpath('//body/ul/li')
48 # for item in ul_list:
49 #     v = item.xpath('./a/span')
50 #     # 或
51 #     # v = item.xpath('a/span')
52 #     # 或
53 #     # v = item.xpath('*/a/span')
54 #     print(v)

View Code

 1 import scrapy
 2 from scrapy.selector import HtmlXPathSelector
 3 from scrapy.http.request import Request
 4 from scrapy.http.cookies import CookieJar
 5 from scrapy import FormRequest
 6
 7
 8 class ChouTiSpider(scrapy.Spider):
 9     # 爬虫应用的名称，通过此名称启动爬虫命令
10     name = "chouti"
11     # 允许的域名
12     allowed_domains = ["chouti.com"]
13
14     cookie_dict = {}
15     has_request_set = {}
16
17     def start_requests(self):
18         url = 'http://dig.chouti.com/'
19         # return [Request(url=url, callback=self.login)]
20         yield Request(url=url, callback=self.login)
21
22     def login(self, response):
23         cookie_jar = CookieJar()
24         cookie_jar.extract_cookies(response, response.request)
25         for k, v in cookie_jar._cookies.items():
26             for i, j in v.items():
27                 for m, n in j.items():
28                     self.cookie_dict[m] = n.value
29
30         req = Request(
31             url='http://dig.chouti.com/login',
32             method='POST',
33             headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'},
34             body='phone=8615131255089&password=pppppppp&oneMonth=1',
35             cookies=self.cookie_dict,
36             callback=self.check_login
37         )
38         yield req
39
40     def check_login(self, response):
41         req = Request(
42             url='http://dig.chouti.com/',
43             method='GET',
44             callback=self.show,
45             cookies=self.cookie_dict,
46             dont_filter=True
47         )
48         yield req
49
50     def show(self, response):
51         # print(response)
52         hxs = HtmlXPathSelector(response)
53         news_list = hxs.select('//div[@id="content-list"]/div[@class="item"]')
54         for new in news_list:
55             # temp = new.xpath('div/div[@class="part2"]/@share-linkid').extract()
56             link_id = new.xpath('*/div[@class="part2"]/@share-linkid').extract_first()
57             yield Request(
58                 url='http://dig.chouti.com/link/vote?linksId=%s' %(link_id,),
59                 method='POST',
60                 cookies=self.cookie_dict,
61                 callback=self.do_favor
62             )
63
64         page_list = hxs.select('//div[@id="dig_lcpage"]//a[re:test(@href, "/all/hot/recent/\d+")]/@href').extract()
65         for page in page_list:
66
67             page_url = 'http://dig.chouti.com%s' % page
68             import hashlib
69             hash = hashlib.md5()
70             hash.update(bytes(page_url,encoding='utf-8'))
71             key = hash.hexdigest()
72             if key in self.has_request_set:
73                 pass
74             else:
75                 self.has_request_set[key] = page_url
76                 yield Request(
77                     url=page_url,
78                     method='GET',
79                     callback=self.show
80                 )
81
82     def do_favor(self, response):
83         print(response.text)

自动登录抽屉点赞

　　6. 格式化处理

spider/xiahuar.py

1 import scrapy
2
3
4 class XiaoHuarItem(scrapy.Item):
5     name = scrapy.Field()
6     school = scrapy.Field()
7     url = scrapy.Field()

items

 1 import json
 2 import os
 3 import requests
 4
 5
 6 class JsonPipeline(object):
 7     def __init__(self):
 8         self.file = open('xiaohua.txt', 'w')
 9
10     def process_item(self, item, spider):
11         v = json.dumps(dict(item), ensure_ascii=False)
12         self.file.write(v)
13         self.file.write('\n')
14         self.file.flush()
15         return item
16
17
18 class FilePipeline(object):
19     def __init__(self):
20         if not os.path.exists('imgs'):
21             os.makedirs('imgs')
22
23     def process_item(self, item, spider):
24         response = requests.get(item['url'], stream=True)
25         file_name = '%s_%s.jpg' % (item['name'], item['school'])
26         with open(os.path.join('imgs', file_name), mode='wb') as f:
27             f.write(response.content)
28         return item

pipelines

1 ITEM_PIPELINES = {
2    'spider1.pipelines.JsonPipeline': 100,
3    'spider1.pipelines.FilePipeline': 300,
4 }
5 # 每行后面的整型值，确定了他们运行的顺序，item按数字从低到高的顺序，通过pipeline，通常将这些数字定义在0-1000范围内。

settings

　　7. 自定义pipelines

 1 from scrapy.exceptions import DropItem
 2
 3 class CustomPipeline(object):
 4     def __init__(self,v):
 5         self.value = v
 6
 7     def process_item(self, item, spider):
 8         # 操作并进行持久化
 9
10         # return表示会被后续的pipeline继续处理
11         return item
12
13         # 表示将item丢弃，不会被后续pipeline处理
14         # raise DropItem()
15
16
17     @classmethod
18     def from_crawler(cls, crawler):
19         """
20         初始化时候，用于创建pipeline对象
21         :param crawler:
22         :return:
23         """
24         val = crawler.settings.getint('MMMM')
25         return cls(val)
26
27     def open_spider(self,spider):
28         """
29         爬虫开始执行时，调用
30         :param spider:
31         :return:
32         """
33         print('000000')
34
35     def close_spider(self,spider):
36         """
37         爬虫关闭时，被调用
38         :param spider:
39         :return:
40         """
41         print('111111')
42
43 自定义pipeline

View Code

　　8. 中间件　　

 1 class SpiderMiddleware(object):
 2
 3     def process_spider_input(self,response, spider):
 4         """
 5         下载完成，执行，然后交给parse处理
 6         :param response:
 7         :param spider:
 8         :return:
 9         """
10         pass
11
12     def process_spider_output(self,response, result, spider):
13         """
14         spider处理完成，返回时调用
15         :param response:
16         :param result:
17         :param spider:
18         :return: 必须返回包含 Request 或 Item 对象的可迭代对象(iterable)
19         """
20         return result
21
22     def process_spider_exception(self,response, exception, spider):
23         """
24         异常调用
25         :param response:
26         :param exception:
27         :param spider:
28         :return: None,继续交给后续中间件处理异常；含 Response 或 Item 的可迭代对象(iterable)，交给调度器或pipeline
29         """
30         return None
31
32
33     def process_start_requests(self,start_requests, spider):
34         """
35         爬虫启动时调用
36         :param start_requests:
37         :param spider:
38         :return: 包含 Request 对象的可迭代对象
39         """
40         return start_requests
41
42 爬虫中间件

爬虫中间件

 1 class DownMiddleware1(object):
 2     def process_request(self, request, spider):
 3         """
 4         请求需要被下载时，经过所有下载器中间件的process_request调用
 5         :param request:
 6         :param spider:
 7         :return:
 8             None,继续后续中间件去下载；
 9             Response对象，停止process_request的执行，开始执行process_response
10             Request对象，停止中间件的执行，将Request重新调度器
11             raise IgnoreRequest异常，停止process_request的执行，开始执行process_exception
12         """
13         pass
14
15
16
17     def process_response(self, request, response, spider):
18         """
19         spider处理完成，返回时调用
20         :param response:
21         :param result:
22         :param spider:
23         :return:
24             Response 对象：转交给其他中间件process_response
25             Request 对象：停止中间件，request会被重新调度下载
26             raise IgnoreRequest 异常：调用Request.errback
27         """
28         print('response1')
29         return response
30
31     def process_exception(self, request, exception, spider):
32         """
33         当下载处理器(download handler)或 process_request() (下载中间件)抛出异常
34         :param response:
35         :param exception:
36         :param spider:
37         :return:
38             None：继续交给后续中间件处理异常；
39             Response对象：停止后续process_exception方法
40             Request对象：停止中间件，request将会被重新调用下载
41         """
42         return None
43
44 下载器中间件

下载中间件

　　9. 自定制命令

　　A.在spiders同级创建任意目录,在目录创建crawlall.py

 1 from scrapy.commands import ScrapyCommand
 2     from scrapy.utils.project import get_project_settings
 3
 4
 5     class Command(ScrapyCommand):
 6
 7         requires_project = True
 8
 9         def syntax(self):
10             return '[options]'
11
12         def short_desc(self):
13             return 'Runs all of the spiders'
14
15         def run(self, args, opts):
16             spider_list = self.crawler_process.spiders.list()
17             for name in spider_list:
18                 self.crawler_process.crawl(name, **opts.__dict__)
19             self.crawler_process.start()

crawlall.py

　　B.在settings.py 中添加配置 COMMANDS_MODULE = '项目名称.目录名称

　　C.scrapy crawlall

　　10. 避免重复访问

　　　　scrapy默认使用 scrapy.dupefilter.RFPDupeFilter 进行去重，相关配置有：

DUPEFILTER_CLASS = 'scrapy.dupefilter.RFPDupeFilter'
DUPEFILTER_DEBUG = False
JOBDIR = "保存范文记录的日志路径，如：/root/"  # 最终路径为 /root/requests.seen

　　11. settings其它配置

  1 # -*- coding: utf-8 -*-
  2
  3 # Scrapy settings for step8_king project
  4 #
  5 # For simplicity, this file contains only settings considered important or
  6 # commonly used. You can find more settings consulting the documentation:
  7 #
  8 #     http://doc.scrapy.org/en/latest/topics/settings.html
  9 #     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
 10 #     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
 11
 12 # 1. 爬虫名称
 13 BOT_NAME = 'step8_king'
 14
 15 # 2. 爬虫应用路径
 16 SPIDER_MODULES = ['step8_king.spiders']
 17 NEWSPIDER_MODULE = 'step8_king.spiders'
 18
 19 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 20 # 3. 客户端 user-agent请求头
 21 # USER_AGENT = 'step8_king (+http://www.yourdomain.com)'
 22
 23 # Obey robots.txt rules
 24 # 4. 禁止爬虫配置
 25 # ROBOTSTXT_OBEY = False
 26
 27 # Configure maximum concurrent requests performed by Scrapy (default: 16)
 28 # 5. 并发请求数
 29 # CONCURRENT_REQUESTS = 4
 30
 31 # Configure a delay for requests for the same website (default: 0)
 32 # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
 33 # See also autothrottle settings and docs
 34 # 6. 延迟下载秒数
 35 # DOWNLOAD_DELAY = 2
 36
 37
 38 # The download delay setting will honor only one of:
 39 # 7. 单域名访问并发数，并且延迟下次秒数也应用在每个域名
 40 # CONCURRENT_REQUESTS_PER_DOMAIN = 2
 41 # 单IP访问并发数，如果有值则忽略：CONCURRENT_REQUESTS_PER_DOMAIN，并且延迟下次秒数也应用在每个IP
 42 # CONCURRENT_REQUESTS_PER_IP = 3
 43
 44 # Disable cookies (enabled by default)
 45 # 8. 是否支持cookie，cookiejar进行操作cookie
 46 # COOKIES_ENABLED = True
 47 # COOKIES_DEBUG = True
 48
 49 # Disable Telnet Console (enabled by default)
 50 # 9. Telnet用于查看当前爬虫的信息，操作爬虫等...
 51 #    使用telnet ip port ，然后通过命令操作
 52 # TELNETCONSOLE_ENABLED = True
 53 # TELNETCONSOLE_HOST = '127.0.0.1'
 54 # TELNETCONSOLE_PORT = [6023,]
 55
 56
 57 # 10. 默认请求头
 58 # Override the default request headers:
 59 # DEFAULT_REQUEST_HEADERS = { 60 #     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 61 #     'Accept-Language': 'en',
 62 # }
 63
 64
 65 # Configure item pipelines
 66 # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
 67 # 11. 定义pipeline处理请求
 68 # ITEM_PIPELINES = { 69 #    'step8_king.pipelines.JsonPipeline': 700,
 70 #    'step8_king.pipelines.FilePipeline': 500,
 71 # }
 72
 73
 74
 75 # 12. 自定义扩展，基于信号进行调用
 76 # Enable or disable extensions
 77 # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
 78 # EXTENSIONS = { 79 #     # 'step8_king.extensions.MyExtension': 500,
 80 # }
 81
 82
 83 # 13. 爬虫允许的最大深度，可以通过meta查看当前深度；0表示无深度
 84 # DEPTH_LIMIT = 3
 85
 86 # 14. 爬取时，0表示深度优先Lifo(默认)；1表示广度优先FiFo
 87
 88 # 后进先出，深度优先
 89 # DEPTH_PRIORITY = 0
 90 # SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleLifoDiskQueue'
 91 # SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.LifoMemoryQueue'
 92 # 先进先出，广度优先
 93
 94 # DEPTH_PRIORITY = 1
 95 # SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleFifoDiskQueue'
 96 # SCHEDULER_MEMORY_QUEUE = 'scrapy.squeue.FifoMemoryQueue'
 97
 98 # 15. 调度器队列
 99 # SCHEDULER = 'scrapy.core.scheduler.Scheduler'
100 # from scrapy.core.scheduler import Scheduler
101
102
103 # 16. 访问URL去重
104 # DUPEFILTER_CLASS = 'step8_king.duplication.RepeatUrl'
105
106
107 # Enable and configure the AutoThrottle extension (disabled by default)
108 # See http://doc.scrapy.org/en/latest/topics/autothrottle.html
109
110 """
111 17. 自动限速算法
112     from scrapy.contrib.throttle import AutoThrottle
113     自动限速设置
114     1. 获取最小延迟 DOWNLOAD_DELAY
115     2. 获取最大延迟 AUTOTHROTTLE_MAX_DELAY
116     3. 设置初始下载延迟 AUTOTHROTTLE_START_DELAY
117     4. 当请求下载完成后，获取其"连接"时间 latency，即：请求连接到接受到响应头之间的时间
118     5. 用于计算的... AUTOTHROTTLE_TARGET_CONCURRENCY
119     target_delay = latency / self.target_concurrency
120     new_delay = (slot.delay + target_delay) / 2.0 # 表示上一次的延迟时间
121     new_delay = max(target_delay, new_delay)
122     new_delay = min(max(self.mindelay, new_delay), self.maxdelay)
123     slot.delay = new_delay
124 """
125
126 # 开始自动限速
127 # AUTOTHROTTLE_ENABLED = True
128 # The initial download delay
129 # 初始下载延迟
130 # AUTOTHROTTLE_START_DELAY = 5
131 # The maximum download delay to be set in case of high latencies
132 # 最大下载延迟
133 # AUTOTHROTTLE_MAX_DELAY = 10
134 # The average number of requests Scrapy should be sending in parallel to each remote server
135 # 平均每秒并发数
136 # AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
137
138 # Enable showing throttling stats for every response received:
139 # 是否显示
140 # AUTOTHROTTLE_DEBUG = True
141
142 # Enable and configure HTTP caching (disabled by default)
143 # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
144
145
146 """
147 18. 启用缓存
148     目的用于将已经发送的请求或相应缓存下来，以便以后使用
149
150     from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
151     from scrapy.extensions.httpcache import DummyPolicy
152     from scrapy.extensions.httpcache import FilesystemCacheStorage
153 """
154 # 是否启用缓存策略
155 # HTTPCACHE_ENABLED = True
156
157 # 缓存策略：所有请求均缓存，下次在请求直接访问原来的缓存即可
158 # HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
159 # 缓存策略：根据Http响应头：Cache-Control、Last-Modified 等进行缓存的策略
160 # HTTPCACHE_POLICY = "scrapy.extensions.httpcache.RFC2616Policy"
161
162 # 缓存超时时间
163 # HTTPCACHE_EXPIRATION_SECS = 0
164
165 # 缓存保存路径
166 # HTTPCACHE_DIR = 'httpcache'
167
168 # 缓存忽略的Http状态码
169 # HTTPCACHE_IGNORE_HTTP_CODES = []
170
171 # 缓存存储的插件
172 # HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
173
174
175 """
176 19. 代理，需要在环境变量中设置
177     from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware
178
179     方式一：使用默认
180         os.environ
181         {
182             http_proxy:http://root:woshiniba@192.168.11.11:9999/
183             https_proxy:http://192.168.11.11:9999/
184         }
185     方式二：使用自定义下载中间件
186
187     def to_bytes(text, encoding=None, errors='strict'):
188         if isinstance(text, bytes):
189             return text
190         if not isinstance(text, six.string_types):
191             raise TypeError('to_bytes must receive a unicode, str or bytes '
192                             'object, got %s' % type(text).__name__)
193         if encoding is None:
194             encoding = 'utf-8'
195         return text.encode(encoding, errors)
196
197     class ProxyMiddleware(object):
198         def process_request(self, request, spider):
199             PROXIES = [
200                 {'ip_port': '111.11.228.75:80', 'user_pass': ''},
201                 {'ip_port': '120.198.243.22:80', 'user_pass': ''},
202                 {'ip_port': '111.8.60.9:8123', 'user_pass': ''},
203                 {'ip_port': '101.71.27.120:80', 'user_pass': ''},
204                 {'ip_port': '122.96.59.104:80', 'user_pass': ''},
205                 {'ip_port': '122.224.249.122:8088', 'user_pass': ''},
206             ]
207             proxy = random.choice(PROXIES)
208             if proxy['user_pass'] is not None:
209                 request.meta['proxy'] = to_bytes（"http://%s" % proxy['ip_port']）
210                 encoded_user_pass = base64.encodestring(to_bytes(proxy['user_pass']))
211                 request.headers['Proxy-Authorization'] = to_bytes('Basic ' + encoded_user_pass)
212                 print "**************ProxyMiddleware have pass************" + proxy['ip_port']
213             else:
214                 print "**************ProxyMiddleware no pass************" + proxy['ip_port']
215                 request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])
216
217     DOWNLOADER_MIDDLEWARES = {
218        'step8_king.middlewares.ProxyMiddleware': 500,
219     }
220
221 """
222
223 """
224 20. Https访问
225     Https访问时有两种情况：
226     1. 要爬取网站使用的可信任证书(默认支持)
227         DOWNLOADER_HTTPCLIENTFACTORY = "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
228         DOWNLOADER_CLIENTCONTEXTFACTORY = "scrapy.core.downloader.contextfactory.ScrapyClientContextFactory"
229
230     2. 要爬取网站使用的自定义证书
231         DOWNLOADER_HTTPCLIENTFACTORY = "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
232         DOWNLOADER_CLIENTCONTEXTFACTORY = "step8_king.https.MySSLFactory"
233
234         # https.py
235         from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
236         from twisted.internet.ssl import (optionsForClientTLS, CertificateOptions, PrivateCertificate)
237
238         class MySSLFactory(ScrapyClientContextFactory):
239             def getCertificateOptions(self):
240                 from OpenSSL import crypto
241                 v1 = crypto.load_privatekey(crypto.FILETYPE_PEM, open('/Users/wupeiqi/client.key.unsecure', mode='r').read())
242                 v2 = crypto.load_certificate(crypto.FILETYPE_PEM, open('/Users/wupeiqi/client.pem', mode='r').read())
243                 return CertificateOptions(
244                     privateKey=v1,  # pKey对象
245                     certificate=v2,  # X509对象
246                     verify=False,
247                     method=getattr(self, 'method', getattr(self, '_ssl_method', None))
248                 )
249     其他：
250         相关类
251             scrapy.core.downloader.handlers.http.HttpDownloadHandler
252             scrapy.core.downloader.webclient.ScrapyHTTPClientFactory
253             scrapy.core.downloader.contextfactory.ScrapyClientContextFactory
254         相关配置
255             DOWNLOADER_HTTPCLIENTFACTORY
256             DOWNLOADER_CLIENTCONTEXTFACTORY
257
258 """
259
260
261
262 """
263 21. 爬虫中间件
264     class SpiderMiddleware(object):
265
266         def process_spider_input(self,response, spider):
267             '''
268             下载完成，执行，然后交给parse处理
269             :param response:
270             :param spider:
271             :return:
272             '''
273             pass
274
275         def process_spider_output(self,response, result, spider):
276             '''
277             spider处理完成，返回时调用
278             :param response:
279             :param result:
280             :param spider:
281             :return: 必须返回包含 Request 或 Item 对象的可迭代对象(iterable)
282             '''
283             return result
284
285         def process_spider_exception(self,response, exception, spider):
286             '''
287             异常调用
288             :param response:
289             :param exception:
290             :param spider:
291             :return: None,继续交给后续中间件处理异常；含 Response 或 Item 的可迭代对象(iterable)，交给调度器或pipeline
292             '''
293             return None
294
295
296         def process_start_requests(self,start_requests, spider):
297             '''
298             爬虫启动时调用
299             :param start_requests:
300             :param spider:
301             :return: 包含 Request 对象的可迭代对象
302             '''
303             return start_requests
304
305     内置爬虫中间件：
306         'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50,
307         'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': 500,
308         'scrapy.contrib.spidermiddleware.referer.RefererMiddleware': 700,
309         'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware': 800,
310         'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': 900,
311
312 """
313 # from scrapy.contrib.spidermiddleware.referer import RefererMiddleware
314 # Enable or disable spider middlewares
315 # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
316 SPIDER_MIDDLEWARES = {
317    # 'step8_king.middlewares.SpiderMiddleware': 543,
318 }
319
320
321 """
322 22. 下载中间件
323     class DownMiddleware1(object):
324         def process_request(self, request, spider):
325             '''
326             请求需要被下载时，经过所有下载器中间件的process_request调用
327             :param request:
328             :param spider:
329             :return:
330                 None,继续后续中间件去下载；
331                 Response对象，停止process_request的执行，开始执行process_response
332                 Request对象，停止中间件的执行，将Request重新调度器
333                 raise IgnoreRequest异常，停止process_request的执行，开始执行process_exception
334             '''
335             pass
336
337
338
339         def process_response(self, request, response, spider):
340             '''
341             spider处理完成，返回时调用
342             :param response:
343             :param result:
344             :param spider:
345             :return:
346                 Response 对象：转交给其他中间件process_response
347                 Request 对象：停止中间件，request会被重新调度下载
348                 raise IgnoreRequest 异常：调用Request.errback
349             '''
350             print('response1')
351             return response
352
353         def process_exception(self, request, exception, spider):
354             '''
355             当下载处理器(download handler)或 process_request() (下载中间件)抛出异常
356             :param response:
357             :param exception:
358             :param spider:
359             :return:
360                 None：继续交给后续中间件处理异常；
361                 Response对象：停止后续process_exception方法
362                 Request对象：停止中间件，request将会被重新调用下载
363             '''
364             return None
365
366
367     默认下载中间件
368     {
369         'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100,
370         'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
371         'scrapy.contrib.downloadermiddleware.downloadtimeout.DownloadTimeoutMiddleware': 350,
372         'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
373         'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
374         'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550,
375         'scrapy.contrib.downloadermiddleware.redirect.MetaRefreshMiddleware': 580,
376         'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 590,
377         'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
378         'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
379         'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750,
380         'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830,
381         'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
382         'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
383     }
384
385 """
386 # from scrapy.contrib.downloadermiddleware.httpauth import HttpAuthMiddleware
387 # Enable or disable downloader middlewares
388 # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
389 # DOWNLOADER_MIDDLEWARES = {390 #    'step8_king.middlewares.DownMiddleware1': 100,
391 #    'step8_king.middlewares.DownMiddleware2': 500,
392 # }
393
394 settings

View Code

转载于:https://www.cnblogs.com/weibiao/p/7715986.html

python16_day37【爬虫2】相关推荐

html,xml_网页开发_爬虫_笔记
20220402 笔记本走手机热点,重连之后又是另一个ip 20220331 https://mp.weixin.qq.com/s/znXuCB0Fl32TbP_0UaO6SQ 爬虫相关知识快速复习 ...
Python 爬虫框架Scrapy安装汇总
传统方式安装Scrapy(慎用) 练习了基本的操作之后,当然就要找框架来进行爬虫实验啊.于是就在网上找Windows 64安装Scrapy的方法,查到的都是非常繁琐的安装方式,由于Scrapy有很多个 ...
Python:爬虫框架Scrapy的安装与基本使用
一.简单实例,了解基本. 1.安装Scrapy框架这里如果直接pip3 install scrapy可能会出错. 所以你可以先安装lxml:pip3 install lxml(已安装请忽略). 安装 ...
Python:从零搭建Redis-Scrapy分布式爬虫
Scrapy-Redis分布式策略: 假设有四台电脑:Windows 10.Mac OS X.Ubuntu 16.04.CentOS 7.2,任意一台电脑都可以作为 Master端或 Slaver端 ...
Python案例：使用XPath的爬虫
案例:使用XPath的爬虫现在我们用XPath来做一个简单的爬虫,我们尝试爬取某个贴吧里的所有帖子,并且将该这个帖子里每个楼层发布的图片下载到本地. # tieba_xpath.py#!/usr/b ...
Python案例：使用正则表达式的爬虫
案例:使用正则表达式的爬虫现在拥有了正则表达式这把神兵利器,我们就可以进行对爬取到的全部网页源代码进行筛选了. 下面我们一起尝试一下爬取内涵段子网站: http://www.neihan8.com/ ...
关于Python爬虫原理和数据抓取1.1
为什么要做爬虫? 首先请问:都说现在是"大数据时代",那数据从何而来? 企业产生的用户数据:百度指数.阿里指数.TBI腾讯浏览指数.新浪微博指数数据平台购买数据:数据堂.国云数据 ...
python爬虫之Scrapy框架的post请求和核心组件的工作流程
python爬虫之Scrapy框架的post请求和核心组件的工作流程一 Scrapy的post请求的实现在爬虫文件中的爬虫类继承了Spider父类中的start_urls,该方法就可以对star ...
Rstudio 实现爬虫文本分词个性化词云设计--我爱中国我爱党
Rstudio 爬虫文本分词个性化词云设计目录 1.环境准备,加载依赖 2.rvest 爬虫,数据爬取 3.jiebaR用于分词,词频统计 4.wordcloud2 结果可视化 ========= ...

python16_day37【爬虫2】

python16_day37【爬虫2】相关推荐

最新文章

热门文章