使用协程同时下载多文件，碰到阻塞函数则无法同时下载多文件，结合多线程处理。

# coding=utf-8import asyncio
import os
import requests
from concurrent.futures import ThreadPoolExecutorgPool = ThreadPoolExecutor(5)async def downOne(loop, url):name = os.path.basename(url)print("downOne:start {}".format(name))# loop = asyncio.get_event_loop()# requests.get(url)global gPoolresult = await loop.run_in_executor(gPool, requests.get, url)print("downOne:middle {}".format(name))if result.status_code == 200:with open(name, "wb") as fp:# fp.write(result.content)# r = await loop.run_in_executor(None, fp.write, result.content)r = await loop.run_in_executor(gPool, fp.write, result.content)print("downOne: r = {}".format(r))print("downOne:endxx {}".format(name))return "ok {}".format(name)async def deal(loop):ts = []for i in range(1, 20):name = str(i) + ".mp3"path = "http://mp3-d.ting89.com:9090/有声书/闪婚剩女" + "/" + namets.append(asyncio.ensure_future(downOne(loop, path)))print("deal: middle")result = await asyncio.gather(*ts)print("deal:end")return resultif __name__ == "__main__":loop = asyncio.get_event_loop()try:result = loop.run_until_complete(deal(loop))print("main:result = {}".format(result))finally:loop.close()print("main end")

结果：

deal: middle
downOne:start 1.mp3
downOne:start 2.mp3
downOne:start 3.mp3
downOne:start 4.mp3
downOne:start 5.mp3
downOne:start 6.mp3
downOne:start 7.mp3
downOne:start 8.mp3
downOne:start 9.mp3
downOne:start 10.mp3
downOne:start 11.mp3
downOne:start 12.mp3
downOne:start 13.mp3
downOne:start 14.mp3
downOne:start 15.mp3
downOne:start 16.mp3
downOne:start 17.mp3
downOne:start 18.mp3
downOne:start 19.mp3
downOne:middle 2.mp3
downOne:middle 5.mp3
downOne:middle 1.mp3
downOne:middle 4.mp3
downOne:middle 3.mp3
downOne:middle 7.mp3
downOne:middle 6.mp3
downOne:middle 8.mp3
downOne:middle 9.mp3
downOne:middle 11.mp3
downOne:middle 13.mp3
downOne:middle 10.mp3
downOne:middle 16.mp3
downOne:middle 12.mp3
downOne:middle 15.mp3
downOne: r = 4953446
downOne:endxx 2.mp3
downOne: r = 4837358
downOne:endxx 5.mp3
downOne: r = 5231285
downOne:endxx 1.mp3
downOne: r = 4911755
downOne:endxx 4.mp3
downOne: r = 4826282
downOne:endxx 3.mp3
downOne: r = 4941848
downOne:endxx 7.mp3
downOne: r = 4956686
downOne:endxx 6.mp3
downOne: r = 4995242
downOne:endxx 8.mp3
downOne: r = 4909038
downOne:endxx 9.mp3
downOne: r = 5018113
downOne:endxx 11.mp3
downOne: r = 5015641
downOne:endxx 13.mp3
downOne: r = 5032441
downOne:endxx 10.mp3
downOne: r = 5011252
downOne:endxx 16.mp3
downOne: r = 5028248
downOne:endxx 12.mp3
downOne: r = 5006237
downOne:endxx 15.mp3
downOne:middle 14.mp3
downOne: r = 4998505
downOne:endxx 14.mp3
downOne:middle 17.mp3
downOne: r = 4990877
downOne:endxx 17.mp3
downOne:middle 19.mp3
downOne: r = 5049809
downOne:endxx 19.mp3
downOne:middle 18.mp3
downOne: r = 5057959
downOne:endxx 18.mp3
deal:end
main:result = ['ok 1.mp3', 'ok 2.mp3', 'ok 3.mp3', 'ok 4.mp3', 'ok 5.mp3', 'ok 6.mp3', 'ok 7.mp3', 'ok 8.mp3', 'ok 9.mp3', 'ok 10.mp3', 'ok 11.mp3', 'ok 12.mp3', 'ok 13.mp3', 'ok 14.mp3', 'ok 15.mp3', 'ok 16.mp3', 'ok 17.mp3', 'ok 18.mp3', 'ok 19.mp3']
main end

改进版：

'''
Description:
Author: pdh
Date: 2020-11-05 12:12:22
LastEditors: pdh
LastEditTime: 2020-11-06 14:47:55
FilePath: \down\down.py
'''
# coding=utf-8import os
import requests
from concurrent.futures import ThreadPoolExecutor
import threading
import asynciogLock = threading.Lock()
gPool = ThreadPoolExecutor(30)def pinfo(text):global gLockgLock.acquire()print(text)gLock.release()def allFiles():names = [str(f).zfill(3)+".mp3" for f in range(1, 271)]return namesasync def downOne(name, loop):urlPrefixArr = ("http://mp3-e.ting89.com:9090","http://mp3-2f.ting89.com:9090","http://mp32a.ting89.com:9090","http://mp3-d.ting89.com:9090","http://mp32b.ting89.com:9090","http://mp33a.ting89.com:9090","http://mp3f.ting89.com:9090","http://mp33b.ting89.com:9090","http://mp3-2e.ting89.com:9090","http://mp3-f.ting89.com:9090","http://mp3-2.ting89.com:9090","http://mp3d.ting89.com:9090")global gPoolfor urlPre in urlPrefixArr:for big in range(2020, 2022):for sub in range(45, 90):if os.path.isfile(name):return name, Trueurl = "/".join((urlPre, str(big), str(sub), "红人", name))pinfo("downOne:start {}".format(name))result = await loop.run_in_executor(gPool, requests.get, url)if result.status_code == 200:with open(name, "wb") as f:f.write(result.content)pinfo("downOne:ok {}".format(name))else:pinfo("downOne:fail url={}".format(url))await asyncio.sleep(0.01)return name, Falseasync def downAll(loop):fs = []names = allFiles()for name in names:task = asyncio.ensure_future(downOne(name, loop))fs.append(task)pinfo("downAll:wait")rs = await asyncio.gather(*fs)# pinfo("downAll:waiting {}".format(rs))pinfo("downAll:wait end")return rsdef deal():loop = asyncio.get_event_loop()try:rs = loop.run_until_complete(downAll(loop))count = 0for name, success in rs:if success:count += 1pinfo("name = {}, success= {}".format(name, success))else:print("faile name={}".format(name))pinfo("成功下载: count = {}".format(count))finally:loop.close()if __name__ == "__main__":deal()pinfo("main:middle")while True:line = input("输入")if (line == 'q'):break

结果：

downAll:wait
downOne:start 115.mp3
downOne:start 125.mp3
downOne:start 132.mp3
downOne:start 136.mp3
downOne:start 173.mp3
downOne:start 177.mp3
downOne:start 181.mp3
downOne:start 185.mp3
downOne:start 220.mp3
downOne:start 225.mp3
downOne:start 230.mp3
downOne:start 238.mp3
downOne:start 239.mp3
downOne:start 240.mp3
downOne:start 241.mp3
downOne:start 242.mp3
downOne:start 243.mp3
downOne:start 244.mp3
downOne:start 245.mp3
downOne:start 246.mp3
downOne:start 247.mp3
downOne:start 248.mp3
downOne:start 249.mp3
downOne:ok 220.mp3
downOne:ok 243.mp3
downOne:ok 239.mp3
downOne:ok 244.mp3
downOne:ok 225.mp3
downOne:ok 245.mp3
downOne:ok 248.mp3
downOne:ok 181.mp3
downOne:ok 240.mp3
downOne:ok 173.mp3
downOne:ok 242.mp3
downOne:ok 132.mp3
downOne:ok 177.mp3
downOne:ok 136.mp3
downOne:ok 246.mp3
downOne:ok 230.mp3
downOne:ok 247.mp3
downOne:ok 125.mp3
downOne:ok 115.mp3
downOne:ok 249.mp3
downOne:ok 185.mp3
downAll:wait end
main:middle
输入
PS E:\study\py\down>  cd 'e:\study\py\down'; & 'D:\program\Python\Python37\python.exe' 'c:\Users\andrew\.vscode\extensions\ms-python.python-2020.10.332292344\pythonFiles\lib\python\debugpy\launcher' '64882' '--' 'e:\study\py\down\down.py'
downAll:wait
downAll:wait end
成功下载: count = 270
main:middle
输入
PS E:\study\py\down>  cd 'e:\study\py\down'; & 'D:\program\Python\Python37\python.exe' 'c:\Users\andrew\.vscode\extensions\ms-python.python-2020.10.332292344\pythonFiles\lib\python\debugpy\launcher' '65361' '--' 'e:\study\py\down\down.py'
downAll:wait
downAll:wait end
name = 001.mp3, success= True
name = 002.mp3, success= True
name = 003.mp3, success= True
name = 004.mp3, success= True
name = 005.mp3, success= True
name = 006.mp3, success= True
name = 007.mp3, success= True
name = 008.mp3, success= True
name = 009.mp3, success= True
name = 010.mp3, success= True
name = 011.mp3, success= True
name = 012.mp3, success= True
name = 013.mp3, success= True
name = 014.mp3, success= True
name = 015.mp3, success= True
name = 016.mp3, success= True
name = 017.mp3, success= True
name = 018.mp3, success= True
name = 019.mp3, success= True
name = 020.mp3, success= True
name = 021.mp3, success= True
name = 022.mp3, success= True
name = 023.mp3, success= True
name = 024.mp3, success= True
name = 025.mp3, success= True
name = 026.mp3, success= True
name = 027.mp3, success= True
name = 028.mp3, success= True
name = 029.mp3, success= True
name = 030.mp3, success= True
name = 031.mp3, success= True
name = 032.mp3, success= True
name = 033.mp3, success= True
name = 034.mp3, success= True
name = 035.mp3, success= True
name = 036.mp3, success= True
name = 037.mp3, success= True
name = 038.mp3, success= True
name = 039.mp3, success= True
name = 040.mp3, success= True
name = 041.mp3, success= True
name = 042.mp3, success= True
name = 043.mp3, success= True
name = 044.mp3, success= True
name = 045.mp3, success= True
name = 046.mp3, success= True
name = 047.mp3, success= True
name = 048.mp3, success= True
name = 049.mp3, success= True
name = 050.mp3, success= True
name = 051.mp3, success= True
name = 052.mp3, success= True
name = 053.mp3, success= True
name = 054.mp3, success= True
name = 055.mp3, success= True
name = 056.mp3, success= True
name = 057.mp3, success= True
name = 058.mp3, success= True
name = 059.mp3, success= True
name = 060.mp3, success= True
name = 061.mp3, success= True
name = 062.mp3, success= True
name = 063.mp3, success= True
name = 064.mp3, success= True
name = 065.mp3, success= True
name = 066.mp3, success= True
name = 067.mp3, success= True
name = 068.mp3, success= True
name = 069.mp3, success= True
name = 070.mp3, success= True
name = 071.mp3, success= True
name = 072.mp3, success= True
name = 073.mp3, success= True
name = 074.mp3, success= True
name = 075.mp3, success= True
name = 076.mp3, success= True
name = 077.mp3, success= True
name = 078.mp3, success= True
name = 079.mp3, success= True
name = 080.mp3, success= True
name = 081.mp3, success= True
name = 082.mp3, success= True
name = 083.mp3, success= True
name = 084.mp3, success= True
name = 085.mp3, success= True
name = 086.mp3, success= True
name = 087.mp3, success= True
name = 088.mp3, success= True
name = 089.mp3, success= True
name = 090.mp3, success= True
name = 091.mp3, success= True
name = 092.mp3, success= True
name = 093.mp3, success= True
name = 094.mp3, success= True
name = 095.mp3, success= True
name = 096.mp3, success= True
name = 097.mp3, success= True
name = 098.mp3, success= True
name = 099.mp3, success= True
name = 100.mp3, success= True
name = 101.mp3, success= True
name = 102.mp3, success= True
name = 103.mp3, success= True
name = 104.mp3, success= True
name = 105.mp3, success= True
name = 106.mp3, success= True
name = 107.mp3, success= True
name = 108.mp3, success= True
name = 109.mp3, success= True
name = 110.mp3, success= True
name = 111.mp3, success= True
name = 112.mp3, success= True
name = 113.mp3, success= True
name = 114.mp3, success= True
name = 115.mp3, success= True
name = 116.mp3, success= True
name = 117.mp3, success= True
name = 118.mp3, success= True
name = 119.mp3, success= True
name = 120.mp3, success= True
name = 121.mp3, success= True
name = 122.mp3, success= True
name = 123.mp3, success= True
name = 124.mp3, success= True
name = 125.mp3, success= True
name = 126.mp3, success= True
name = 127.mp3, success= True
name = 128.mp3, success= True
name = 129.mp3, success= True
name = 130.mp3, success= True
name = 131.mp3, success= True
name = 132.mp3, success= True
name = 133.mp3, success= True
name = 134.mp3, success= True
name = 135.mp3, success= True
name = 136.mp3, success= True
name = 137.mp3, success= True
name = 138.mp3, success= True
name = 139.mp3, success= True
name = 140.mp3, success= True
name = 141.mp3, success= True
name = 142.mp3, success= True
name = 143.mp3, success= True
name = 144.mp3, success= True
name = 145.mp3, success= True
name = 146.mp3, success= True
name = 147.mp3, success= True
name = 148.mp3, success= True
name = 149.mp3, success= True
name = 150.mp3, success= True
name = 151.mp3, success= True
name = 152.mp3, success= True
name = 153.mp3, success= True
name = 154.mp3, success= True
name = 155.mp3, success= True
name = 156.mp3, success= True
name = 157.mp3, success= True
name = 158.mp3, success= True
name = 159.mp3, success= True
name = 160.mp3, success= True
name = 161.mp3, success= True
name = 162.mp3, success= True
name = 163.mp3, success= True
name = 164.mp3, success= True
name = 165.mp3, success= True
name = 166.mp3, success= True
name = 167.mp3, success= True
name = 168.mp3, success= True
name = 169.mp3, success= True
name = 170.mp3, success= True
name = 171.mp3, success= True
name = 172.mp3, success= True
name = 173.mp3, success= True
name = 174.mp3, success= True
name = 175.mp3, success= True
name = 176.mp3, success= True
name = 177.mp3, success= True
name = 178.mp3, success= True
name = 179.mp3, success= True
name = 180.mp3, success= True
name = 181.mp3, success= True
name = 182.mp3, success= True
name = 183.mp3, success= True
name = 184.mp3, success= True
name = 185.mp3, success= True
name = 186.mp3, success= True
name = 187.mp3, success= True
name = 188.mp3, success= True
name = 189.mp3, success= True
name = 190.mp3, success= True
name = 191.mp3, success= True
name = 192.mp3, success= True
name = 193.mp3, success= True
name = 194.mp3, success= True
name = 195.mp3, success= True
name = 196.mp3, success= True
name = 197.mp3, success= True
name = 198.mp3, success= True
name = 199.mp3, success= True
name = 200.mp3, success= True
name = 201.mp3, success= True
name = 202.mp3, success= True
name = 203.mp3, success= True
name = 204.mp3, success= True
name = 205.mp3, success= True
name = 206.mp3, success= True
name = 207.mp3, success= True
name = 208.mp3, success= True
name = 209.mp3, success= True
name = 210.mp3, success= True
name = 211.mp3, success= True
name = 212.mp3, success= True
name = 213.mp3, success= True
name = 214.mp3, success= True
name = 215.mp3, success= True
name = 216.mp3, success= True
name = 217.mp3, success= True
name = 218.mp3, success= True
name = 219.mp3, success= True
name = 220.mp3, success= True
name = 221.mp3, success= True
name = 222.mp3, success= True
name = 223.mp3, success= True
name = 224.mp3, success= True
name = 225.mp3, success= True
name = 226.mp3, success= True
name = 227.mp3, success= True
name = 228.mp3, success= True
name = 229.mp3, success= True
name = 230.mp3, success= True
name = 231.mp3, success= True
name = 232.mp3, success= True
name = 233.mp3, success= True
name = 234.mp3, success= True
name = 235.mp3, success= True
name = 236.mp3, success= True
name = 237.mp3, success= True
name = 238.mp3, success= True
name = 239.mp3, success= True
name = 240.mp3, success= True
name = 241.mp3, success= True
name = 242.mp3, success= True
name = 243.mp3, success= True
name = 244.mp3, success= True
name = 245.mp3, success= True
name = 246.mp3, success= True
name = 247.mp3, success= True
name = 248.mp3, success= True
name = 249.mp3, success= True
name = 250.mp3, success= True
name = 251.mp3, success= True
name = 252.mp3, success= True
name = 253.mp3, success= True
name = 254.mp3, success= True
name = 255.mp3, success= True
name = 256.mp3, success= True
name = 257.mp3, success= True
name = 258.mp3, success= True
name = 259.mp3, success= True
name = 260.mp3, success= True
name = 261.mp3, success= True
name = 262.mp3, success= True
name = 263.mp3, success= True
name = 264.mp3, success= True
name = 265.mp3, success= True
name = 266.mp3, success= True
name = 267.mp3, success= True
name = 268.mp3, success= True
name = 269.mp3, success= True
name = 270.mp3, success= True
成功下载: count = 270
main:middle
输入

python 协程加多线程下载asyncio、ThreadPoolExecutor相关推荐

  1. python协程与多线程比较_python-协程、多线程、多进程性能比较

    基本概念 进程:进程是一个具有独立功能的程序关于某个数据集合的一次运行活动.进程是操作系统动态执行的基本单元. 线程:一个进程中包含若干线程,当然至少有一个线程,线程可以利用进程所拥有的资源.线程是独 ...

  2. python协程处理海量文件_Python使用asyncio和run-In-Executor线程池处理多个文件的同时下载,python,协程,加,runinexecutor...

    设置默认线程池 from concurrent.futures import ThreadPoolExecutor exe = ThreadPoolExecutor(2) asyncio.get_ev ...

  3. 5分钟完全掌握Python协程

    1. 协程相关的概念 1.1 进程和线程 进程(Process)是应用程序启动的实例,拥有代码.数据和文件和独立的内存空间,是操作系统最小资源管理单元.每个进程下面有一个或者多个线程(Thread), ...

  4. Python 协程 asyncio 极简入门与爬虫实战

    在了解了 Python 并发编程的多线程和多进程之后,我们来了解一下基于 asyncio 的异步IO编程--协程 01 协程简介 协程(Coroutine)又称微线程.纤程,协程不是进程或线程,其执行 ...

  5. python协程asyncio使用

    协程 协程 (corountine):又称微线程. asyncio,在单线程利用CPU和IO同时执行的原理,实现函数异步执行. 实现协程就是要多个任务的循环,await是挂起命令.每到一个地方awai ...

  6. python 协程 php,python3.x,协程_python协程练习部分代码的理解？，python3.x,协程,asyncio - phpStudy...

    python协程练习部分代码的理解? import asyncio import threading async def wget(host): print('wget {}'.format(host ...

  7. python多线程调用携程,Python 协程，Python携程

    Python 协程,Python携程 协程 进程:操作系统中存在 线程:操作系统中存在 协程:是微线程 模块(greenlet) 协程不是一个真实存在的东西,是由程序员创造出来的 协程,是对一个线程分 ...

  8. Python学习，gevent协程，多线程，多进程demo

    协程,线程,进程,多线程,多进程,线程池,本渣渣是彻底蒙蔽了,不过干就是了,二话不说写(抄)代码就是了,抄多了就明了了,说错了,写多了就会了! 关于gevent Python通过yield提供了对协程 ...

  9. python 协程_Python 协程与 Go 协程的区别（一）

    ? "Python猫" ,一个值得加星标的公众号 花下猫语:年关将近,不知各位过得怎样?我最近有些忙,收获也挺多,以后有机会分享下.吃饭时间,追了两部剧<了不起的麦瑟尔夫人& ...

  10. python 协程可以嵌套协程吗_Python线程、协程探究（2）——揭开协程的神秘面纱...

    一.上集回顾 在上一篇中我们主要研究了python的多线程困境,发现多核情况下由于GIL的存在,python的多线程程序无法发挥多线程该有的并行威力.在文章的结尾,我们提出如下需求: 既然python ...

最新文章

  1. javascript window.open
  2. 十年一遇的奇葩故障--Windows网络编程接口故障：telnet显示无法加载或初始化请求的服务提供程序...
  3. 百度地图开发实例文章（一）
  4. 企业如何选择一个合适的可视化工具
  5. TDirectory.IsRelativePath是否相对路径
  6. 三步教会你在solidworks中画铝型材
  7. 抽象代数 04.06可解群和幂零群
  8. Excel对比条形图
  9. 听说 Python 生成二维码很简单，说的好像我Java 很复杂一样。
  10. xp计算机启动检测硬盘,winxp系统如何设置开机自检修复硬盘
  11. 金山wps的面试经历
  12. Maximo 容器化改造试验
  13. python定义一个数组,如何在Python中创建一个整数数组？
  14. 参考文献格式详细解释和引用（常见）
  15. 全栈设计师必须拥有的Mac软件推荐
  16. Failed to load response dataNo data found for resource with given identifier
  17. ISO认证体系有哪些
  18. 车牌识别C语言源码 第一版
  19. 如何找到一个好的学习率
  20. 常见加密算法的Python实现

热门文章

  1. MY-IKuai-1
  2. 盘点人工智能十大经典应用领域、图解技术原理
  3. PYMOL | 制作动画
  4. obj文件格式学习（自用）
  5. 手机端通过Es文件浏览器访问windows服务器用File Zilla Sever软件 分享的文件
  6. 《阿特拉斯耸耸肩》节选：互害与甩锅的反敏捷组织
  7. Jenkins集成GitHub
  8. AndroidAPI
  9. Flink 清理 Checkpoint的原理和机制
  10. python爬虫学习-scrapy爬取链家房源信息并存储（翻页）