scrapy学习03--异步aiohttp练习

# 学习aiohttp# 第一步 安装
#pip install aiohttp   -i https://pypi.douban.com/simple
#pip install cchardet   -i https://pypi.douban.com/simple# 客户端
import aiohttp
import asyncioasync def fetch(session,url):async with session.get(url) as response:return await response.text()# 基本
# async def main():
#     async with aiohttp.ClientSession() as session:
#         html=await fetch(session,"http://httpbin.org/headers")
#         print(html)'''
{"headers": {"Accept": "*/*", "Accept-Encoding": "gzip, deflate", "Host": "httpbin.org", "User-Agent": "Python/3.9 aiohttp/3.7.3", "X-Amzn-Trace-Id": "Root=1-602b6a92-01f81ee520af7312137b8421"}
}
'''#读取网页内容
# async def main():
#     async with aiohttp.ClientSession() as session:
#         async with session.get('http://httpbin.org/get') as resp:
#             print(resp.status) # 状态码
#             print(await resp.text(encoding='utf-8'))'''
200
{"args": {}, "headers": {"Accept": "*/*", "Accept-Encoding": "gzip, deflate", "Host": "httpbin.org", "User-Agent": "Python/3.9 aiohttp/3.7.3", "X-Amzn-Trace-Id": "Root=1-602b6c17-66a4d09013f941f320068d7f"}, "origin": "171.44.106.55", "url": "http://httpbin.org/get"
}
'''# 非文本内容格式 只需把text方法改成read即可#请求的自定义# 自定义Headers
# async def main():
#     async with aiohttp.ClientSession() as session:
#         url="http://httpbin.org/"
#         headers = {#             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
#                           "AppleWebKit/537.36 (KHTML, like Gecko)"
#                           " Chrome/78.0.3904.108 Safari/537.36"
#         }
#         await session.post(url, headers=headers)# 自定义cookie
async def main():url = 'http://httpbin.org/cookies'cookies = {'cookies_are': 'working'}async with aiohttp.ClientSession(cookies=cookies) as session:async with session.get(url) as resp:assert await resp.json() == {"cookies": {"cookies_are": "working"}}asyncio.run(main())# 一个函数用来发起请求，另外一个函数用来下载网页

# 同步爬虫
#爬取豆瓣电影
from datetime import datetimeimport requests
from bs4 import BeautifulSoup
from lxml import etree#请求头
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit""/537.36 (KHTML, like Gecko) ""Chrome/72.0.3626.121 Safari/537.36"}
def get_movie_url():req_url='https://movie.douban.com/chart'response=requests.get(url=req_url,headers=headers)html=etree.HTML(response.text)movies_url=html.xpath('//*[@id="content"]/div/div[1]/div/div/table/tr/td/a/@href')return movies_urldef get_movie_content(movie_url):response=requests.get(movie_url,headers=headers)result=etree.HTML(response.text)movie=dict()name=result.xpath('//*[@id="content"]/h1/span[1]/text()')author =result.xpath('//*[@id="info"]/span[1]/span[2]//text()')movie["name"] = namemovie["author"] = authorreturn movieif __name__ == '__main__':start = datetime.now()movie_url_list = get_movie_url()movies = dict()for url in movie_url_list:movies[url] = get_movie_content(url)print(movies)print("同步用时为：{}".format(datetime.now() - start))'''
{'https://movie.douban.com/subject/30458949/': {'name': ['无依之地 Nomadland'], 'author': ['赵婷']}, 'https://movie.douban.com/subject/30257787/': {'name': ['一秒钟'], 'author': ['张艺谋']}, 'https://movie.douban.com/subject/30443686/': {'name': ['穷途鼠的奶酪梦 窮鼠はチーズの夢を見る'], 'author': ['行定勋']}, 'https://movie.douban.com/subject/34869387/': {'name': ['女人的碎片 Pieces of a Woman'], 'author': ['凯内尔·穆德卢佐']}, 'https://movie.douban.com/subject/33408026/': {'name': ['刻在你心底的名字'], 'author': ['柳广辉']}, 'https://movie.douban.com/subject/34894753/': {'name': ['沐浴之王'], 'author': ['易小星']}, 'https://movie.douban.com/subject/35211578/': {'name': ['逃避虽可耻但有用 新春特别篇 逃げるは恥だが役に立つ ガンバレ人類! 新春スペシャル!!'], 'author': ['金子文纪']}, 'https://movie.douban.com/subject/30450313/': {'name': ['前程似锦的女孩 Promising Young Woman'], 'author': ['埃默拉尔德·芬内尔']}, 'https://movie.douban.com/subject/35275115/': {'name': ['2020去死 Death to 2020'], 'author': ['阿尔·坎贝尔', ' / ', 'Alice Mathias']}, 'https://movie.douban.com/subject/34982759/': {'name': ["玫瑰岛的不可思议的历史 L'incredibile storia dell'isola delle rose"], 'author': ['希德尼·希比利亚']}}
同步用时为：0:00:08.478348
'''

# 将同步代码改成异步测试速度
import asyncio
from datetime import datetimeimport aiohttp
from lxml import etree#请求头
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit""/537.36 (KHTML, like Gecko) ""Chrome/72.0.3626.121 Safari/537.36"}async def get_movie_url():req_url='https://movie.douban.com/chart'async with aiohttp.ClientSession(headers=headers) as session:async with session.get(url=req_url,headers=headers) as response:result=await response.text()result=etree.HTML(result)return result.xpath('//*[@id="content"]/div/div[1]/div/div/table/tr/td/a/@href')async def get_movie_content(movie_url):async with aiohttp.ClientSession(headers=headers) as session:async with session.get(url=movie_url,headers=headers) as response:result=await response.text()result=etree.HTML(result)movie = dict()name = result.xpath('//*[@id="content"]/h1/span[1]//text()')author = result.xpath('//*[@id="info"]/span[1]/span[2]//text()')movie["name"] = namemovie["author"] = authorreturn movieif __name__ == '__main__':start = datetime.now()loop = asyncio.get_event_loop()movie_url_list = loop.run_until_complete(get_movie_url()) # run_until_complete 来运行 loop ，等到 future 完成，run_until_complete 也就返回了。tasks = [get_movie_content(url) for url in movie_url_list]movies = loop.run_until_complete(asyncio.gather(*tasks))print(movies)print("异步用时为：{}".format(datetime.now()- start))'''
[{'name': ['无依之地 Nomadland'], 'author': ['赵婷']}, {'name': ['一秒钟'], 'author': ['张艺谋']}, {'name': ['穷途鼠的奶酪梦 窮鼠はチーズの夢を見る'], 'author': ['行定勋']}, {'name': ['女人的碎片 Pieces of a Woman'], 'author': ['凯内尔·穆德卢佐']}, {'name': ['刻在你心底的名字'], 'author': ['柳广辉']}, {'name': ['沐浴之王'], 'author': ['易小星']}, {'name': ['逃避虽可耻但有用 新春特别篇 逃げるは恥だが役に立つ ガンバレ人類! 新春スペシャル!!'], 'author': ['金子文纪']}, {'name': ['前程似锦的女孩 Promising Young Woman'], 'author': ['埃默拉尔德·芬内尔']}, {'name': ['2020去死 Death to 2020'], 'author': ['阿尔·坎贝尔', ' / ', 'Alice Mathias']}, {'name': ["玫瑰岛的不可思议的历史 L'incredibile storia dell'isola delle rose"], 'author': ['希德尼·希比利亚']}]
异步用时为：0:00:01.431173
'''

# 异步理解
'''
参考博客  https://www.cnblogs.com/xinghun85/p/9937741.html
'''
import asyncio
from datetime import datetimeimport requests# 例子async def test2(i):r=await other_test(i)print("1",i,r)async def other_test(i):r = requests.get(i)print("2",i)await asyncio.sleep(4)print("3",datetime.now()-start)return rurl = ["https://segmentfault.com/p/1210000013564725","https://www.jianshu.com/p/83badc8028bd","https://www.baidu.com/"]
loop = asyncio.get_event_loop()
task = [asyncio.ensure_future(test2(i)) for i in url]
start =datetime.now()
loop.run_until_complete(asyncio.wait(task))
endtime = datetime.now()-start
print(endtime)
loop.close()

# aiohttp 爬虫实战
# 参考博客 ：https://blog.csdn.net/qq_36772866/article/details/105355445?ops_request_misc=%257B%2522request%255Fid%2522%253A%2522161346654316780271553327%2522%252C%2522scm%2522%253A%252220140713.130102334.pc%255Fblog.%2522%257D&request_id=161346654316780271553327&biz_id=0&utm_medium=distribute.pc_search_result.none-task-blog-2~blog~first_rank_v1~rank_blog_v1-3-105355445.pc_v1_rank_blog_v1&utm_term=%E7%88%AC%E8%99%AB&spm=1018.2226.3001.4450#  第一步 分析 页面  查看小说信息所在的div# 导包
import asyncio
import csv
import aiohttp
from bs4 import BeautifulSoup#爬取网站内容
async def getData(url,headers):# 创建会话对象async with aiohttp.ClientSession() as session:# 发送get请求，设置请求头async with session.get(url,headers=headers) as response:#返回响应内容return await response.text()#保存数据
def saveData(result):for i in result:soup=BeautifulSoup(i,'lxml')find_div=soup.find_all('div',class_='book-mid-info')for d in find_div:# 小说名name = d.find('h4').getText()# 作者author = d.find('a', class_='name').getText()# 更新时间update = d.find('p', class_='update').getText()# 写入 csvcsvFile = open(r'C:\Users\zxy\Desktop\data.csv', 'w', encoding='utf-8-sig', newline='')writer = csv.writer(csvFile)writer.writerow([name, author, update])csvFile.close()# 创建异步任务并保存数据
def run():for i in range(25):#构建不同的 url 传入 getData，最后由 asyncio 模块执行task=asyncio.ensure_future(getData(url.format(i+1),headers))#将所有请求都加入到列表 taskstasks.append(task)# 当所有请求执行完成，返回响应结果result = loop.run_until_complete(asyncio.gather(*tasks))saveData(result)print(len(result))if __name__=='__main__':#请求头headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36'}# 任务列表tasks=[]# urlurl='https://www.qidian.com/rank/hotsales?page={}'#loop对象loop=asyncio.get_event_loop()#调用run函数run()

如何在scrapy中使用aiohttp

这里需要修改下载中间件

import requests
# 现在，我们创建一个中间件，在这个中间件里面，使用 requests 请求一个需要延迟 5 秒钟才会返回的网址：
# class TestAiohttp:
#     def get_ip(self):
#         requests.get('http://httpbin.org/delay/5').json()
#
#     def process_request(self,request,spider):
#         print("请求一个5秒才会返回的网址")
#         self.get_ip()#将requests换成aiohttpimport asyncio
import aiohttpclass TestAiohttp:async def get_ip(self):async with aiohttp.ClientSession() as client:resp = await client.get('http://httpbin.org/delay/5')result = await resp.json()print(result)async def process_request(self, request, spider):print('请求一个延迟5秒的网址开始')await asyncio.create_task(self.get_ip())

爬虫部分

from datetime import datetimeimport scrapyclass ExerciseSpider(scrapy.Spider):name = 'exercise'#allowed_domains = ['v.qq.com']# 跟进urlstart_urls = ['http://exercise.com']def start_requests(self):for page in range(1, 10):url=f'http://exercise.kingname.info/exercise_middleware_ip/{page}'yield scrapy.Request(url)def parse(self, response):now=datetime.now()print(f'现在时间是：',now,response.text)'''
在没使用下载中间件的时候，每次请求只需要一秒钟。
我么那在下载中间件内 需要请求一个五秒才会给反馈的地址
如果用普通的requests 每次请求时，请求将会被卡五秒
换成aiohttp后，启动后会启动五个并发(这里设置的最大并发数是5)，所以会同时打印出请求一个延迟5秒的网址开始5次。然后稍稍停 5 秒，这 5 个请求几乎同时完成，于是同时打印出这个延迟网址的返回信息。接下来，后面的请求就是每秒一个。
'''

设置部分settings

# 开启自定义下载中间件
DOWNLOADER_MIDDLEWARES = {'aioTest.middlewares.TestAiohttp': 543
}
# 开启异步
TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'

启动类

import asynciofrom scrapy import cmdlineimport sysif sys.platform == 'win32':asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
# 我这里不加这句会报错cmdline.execute('scrapy crawl exercise'.split())

scrapy学习03--异步aiohttp练习相关推荐

Scrapy:学习笔记(2)——Scrapy项目
Scrapy:学习笔记(2)--Scrapy项目 1.创建项目创建一个Scrapy项目,并将其命名为"demo" scrapy startproject demo cd demo ...
Scrapy学习第四课
python爬虫框架scrapy学习第四课任务:爬取凤凰网导航下所有一级.二级和具体新闻数据执行:爬虫实例结果:爬取结果展示任务:爬取凤凰网导航下所有一级.二级和具体新闻数据凤凰网导航一级 ...
scrapy学习（完全版）
scrapy1.6中文文档 scrapy1.6中文文档 scrapy中文文档 Scrapy框架下载页面解析页面并发深度安装 scrapy学习教程如果安装了anconda,可以在anacon ...
Python scrapy学习之爬取2k唯美壁纸详细过程笔记及讲解
Scrapy 学习爬取图片 Scrapy 爬取图片时候绕了很多圈子,才明白了走了很大的弯路,幸亏绕出来了(大话可能说得有点早~),赶紧记录一下心得体会: 创建爬虫时的参数选择: 一是普通创建爬虫法: ...
Scrapy学习——爬取智联招聘网站案例
Scrapy学习--爬取智联招聘网站案例安装scrapy 下载安装准备分析代码结果安装scrapy 如果直接使用pip安装会在安装Twisted报错,所以我们需要手动安装. 下载安装s ...
创新实训(2)-Scrapy 学习
创新实训(2)-Scrapy 学习参考资料:Scrapy 0.25 文档 1.Scrapy简介 Scrapy是一个为了爬取网站数据,提取结构性数据而编写的应用框架. 可以应用在包括数据挖掘,信息处理 ...
Scrapy 学习笔记（-）
Scrapy Scrapy 是一个为了爬取网站数据,提取结构性数据而编写的应用框架. 其可以应用在数据挖掘,信息处理或存储历史数据等一系列的程序中.其最初是为了页面抓取 (更确切来说, 网络抓取 )所 ...
Scrapy学习篇（九）之文件与图片下载
Media Pipeline Scrapy为下载item中包含的文件(比如在爬取到产品时,同时也想保存对应的图片)提供了一个可重用的 item pipelines . 这些pipeline有些共同的方 ...
scrapy学习资料汇总
学习资料接触 Scrapy,是因为想爬取一些知乎的数据,最开始的时候搜索了一些相关的资料和别人的实现方式. Github 上已经有人或多或少的实现了对知乎数据的爬取,我搜索到的有以下几个仓库: ht ...

scrapy学习03--异步aiohttp练习

如何在scrapy中使用aiohttp

scrapy学习03--异步aiohttp练习相关推荐

最新文章

热门文章