1.文件提前使用爬虫整理好,放到了demo文件夹下的1.txt文件中,文件格式如:

http://www.banktunnel.eu/tumblr.com/tabea-lara.2_files/tumblr_n4kxopXkJ41sq93cpo10_1280.jpg
http://www.pptok.com/wp-content/uploads/2012/08/xunguang-9.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_3163ee0990499b7.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_25d8b5e402d0391.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFNWIA9vCAAmNhS2C8eAAAPyvQApsb4ACY2d098.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOGIIc0VAAom_BUlna4AAPyvQCLDjYACicU104.png
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOqIWH2cAAXWIl1DvA4AAPyvQDbhzkABdY6485.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFPGIfiurAAKxaF37_vcAAPyvQGII2YAArGA075.jpg
http://www.hr.com.cn/attachments/2015/06/b16af76b691f47609f3397e8cdd9c1e2.jpg
http://www.hr.com.cn/attachments/2015/06/226fc8acee0d478c9588ded1d04616b1.jpg
http://www.hr.com.cn/attachments/2015/06/c36e39b67d4c410cb794f553a271377b.jpg

2.新建DownloadFile类

public class DownloadFile
    {
        public DownloadFile()
        {           
        }
        public DownloadFile(string fileName, string fileFolder,string saveFileName)
        {
            FileName = fileName;
            FileFolder = fileFolder;
            SaveFileName = saveFileName;
        }      
        public string FileName;
        public string FileFolder;
        public string SaveFileName;
    }

3.Main函数创建下载用文件夹,读取需下载文件名称到List<DownloadFile>中。调用下载方法执行下载。

private static void Main(string[] args)
        {
            List<string> folders = new List<string>()
            {
                "demo","其他文件夹","其他文件夹1","xxxx"
            };
            Parallel.ForEach(folders, folder => CreateFolder(folder));
            List<DownloadFile> downloadFiles = new List<DownloadFile>();
            Parallel.ForEach(folders, folder =>
            {
                downloadFiles.AddRange(ReadFileUrl(folder));
            });
            Logger.Log(Level.Info, "开始任务执行");
            List<Task> tList = new List<Task>();
            downloadFiles.ForEach(p =>
            {
                tList.Add(
                    TransferWebFileHelper.DownloadingDataFromServerAsync(p)
                );
            });
            Task.WaitAll(tList.ToArray());           
            Logger.Log(Level.Info, "任务执行结束");
        }

/// <summary>
        /// 创建下载用文件夹
        /// </summary>
        /// <param name="folderName"></param>
        private static void CreateFolder(string folderName)
        {
            string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
            if (!Directory.Exists(saveFolder))
            {
                Directory.CreateDirectory(saveFolder);
            }
        }

/// <summary>
        /// 载入待下载文件到集合
        /// </summary>
        /// <param name="folderName"></param>
        /// <returns></returns>
        static List<DownloadFile> ReadFileUrl(string folderName)
        {
            List<DownloadFile> list = new List<DownloadFile>();
            DirectoryInfo dir;
            List<string> contents = new List<string>();
            string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName);
            if (Directory.Exists(path))
            {
                dir = new DirectoryInfo(path);
                FileInfo[] files = dir.GetFiles();
                contents = File.ReadAllLines(files[0].FullName, Encoding.Default).ToList();
                string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
                contents.ForEach(fileName =>
                {
                    //TODO 完全限定文件名必须少于 260 个字符,并且目录名必须少于 248 个字符。
                    string tempFileName = fileName.Split('?')[0].Replace("\\", "").Replace(":", "").Replace("*", "").Replace("\"", "").Replace("<", "").Replace(">", "").Replace("|", "");
                    int index = tempFileName.LastIndexOf("/");
                    tempFileName = tempFileName.Substring(index + 1, tempFileName.Length - index - 1);
                    int tempFileNameLength = tempFileName.Length;
                    if ((260 - saveFolder.Length - 2) < tempFileNameLength)
                    {
                        int difference = tempFileNameLength - (260 - saveFolder.Length - 2);//差
                        tempFileName = tempFileName.Substring(difference, tempFileName.Length - difference);
                    }
                    tempFileName = GetRandomStr() + "_" + tempFileName;
                    string saveFileName = Path.Combine(saveFolder, tempFileName);
                    var model = new DownloadFile(fileName, folderName, saveFileName);
                    list.Add(model);
                });
            }
            return list;
        }

4.下载文件类

public static class TransferWebFileHelper
    {
        /// <summary>
        /// 新建文件,如果它不存在的话
        /// </summary>
        /// <param name="fileName"></param>
        private static void CreateFile(string fileName)
        {
            if (!File.Exists(fileName))
            {
                FileStream fileStream = File.Create(fileName);
                fileStream.Close();
                fileStream.Dispose();
            }
        }

/// <summary>
        /// 下载用方法
        /// </summary>
        /// <param name="downloadFile"></param>
        /// <returns></returns>
        public static async Task DownloadingDataFromServerAsync(DownloadFile downloadFile)
        {
            Uri uri = new Uri(downloadFile.FileName);
            string saveFileName = downloadFile.SaveFileName;
            CreateFile(saveFileName);

using (WebClient client = new WebClient())
            {
                try
                {
                    await client.DownloadFileTaskAsync(uri, saveFileName);
                }
                catch (WebException ex)
                {
                    Console.WriteLine($"从 {uri} 下载 {saveFileName}失败");
                    Logger.Log(Level.Info, $"从 {uri} 下载 {saveFileName}失败");
                    Logger.LogException(ex);
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"从 {uri} 下载 {saveFileName}失败");
                    Logger.Log(Level.Info, $"从 {uri} 下载 {saveFileName}失败");
                    Logger.LogException(ex);
                }
            }
        }
    }

5.其他辅助方法

private static string[] randomArray = new[]
        {
            "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V",
            "W", "X", "Y", "Z"
        };
        private static string GetRandomStr()
        {
            Random ran = new Random();
            int i = ran.Next(0, 26);
            Thread.Sleep(1);//尽量让生成的字符每一个都不相同
            return randomArray[i];
        }

转载于:https://www.cnblogs.com/jeff151013/p/11157967.html

Net WebClient 异步批量下载文件相关推荐

  1. C#异步批量下载文件

    实现原理:采用WebClient进行批量下载任务,简单的模拟迅雷下载效果! 废话不多说,先看掩饰效果: 具体实现步骤如下: 1.新建项目:WinBatchDownload 2.先建一个Windows窗 ...

  2. python批量下载文件教程_超简单超详细python小文件、大文件、批量下载教程

    按照不同的情况,python下载文件可以分为三种: 小文件下载 大文件下载 批量下载 python 小文件下载 流程:使用request.get请求链接,返回的内容放置到变量r中,然后将r写入到你想放 ...

  3. java批量下载文件为zip包

    批量下载文件为zip包的工具类 package com.meeno.trainsys.util;import javax.servlet.http.HttpServletRequest; import ...

  4. python批量下载文件-Python实现批量下载文件

    Python实现批量下载文件 #!/usr/bin/env python # -*- coding:utf-8 -*- from gevent import monkey monkey.patch_a ...

  5. python批量下载网页文件-Python实现批量下载文件

    Python实现批量下载文件 #!/usr/bin/env python # -*- coding:utf-8 -*- from gevent import monkey monkey.patch_a ...

  6. mysql 线程池 下载_java线程池实现批量下载文件

    本文实例为大家分享了java线程池实现批量下载文件的具体代码,供大家参考,具体内容如下 1 创建线程池 package com.cheng.webb.thread; import java.util. ...

  7. php批量下载TXT中的URL,需求是这样,再不改变目录结构的前提下,通过URL,批量下载文件,我已经写好了一点儿了...

    //需求是这样,再不改变目录结构的前提下,通过URL,批量下载文件,我已经写好了一点儿了 //但是有点bug,只能下载最后一个文件 //邱老师解决一下 $file=fopen("q.txt& ...

  8. 如何用python批量下载数据_Python实现批量下载文件

    Python实现批量下载文件 #!/usr/bin/env python # -*- coding:utf-8 -*- from gevent import monkey monkey.patch_a ...

  9. python 下载文件-Python实现批量下载文件

    Python实现批量下载文件 #!/usr/bin/env python # -*- coding:utf-8 -*- from gevent import monkey monkey.patch_a ...

最新文章

  1. 因为这两天比较忙,所以没有及时把要发表的东西写来.废话少说:我前面把两个简单的滚动说了下.接下来介绍第三种集合循环滚动....
  2. oracle遍历表做查询,oracle 语句之对数据库的表名就行模糊查询,对查询结果进行遍历,依次获取每个表名结果中的每个字段(存储过程)...
  3. 用Hadoop1.0.3实现KMeans算法
  4. 开发做甲方还是乙方_建筑行业的职场小白求职时是选择做甲方还是乙方?
  5. MPU和CPU有什么区别?
  6. springMVC注解中@RequestMapping中常用参数value params 以及@RequestParam 详解
  7. 机器人庄园作文_十年后的家乡作文精选8篇
  8. 带栩字的优美古诗句_带栩字的名字_以栩字起名-尚名网
  9. 【Shiro第五篇】SpringBoot + Shiro实现用户授权功能
  10. html5广告的版式设计,页面版式设计
  11. 一种新型的EC编码,LRC码,基于RS码的改进,特点介于RS和副本之间。
  12. 字跳三轮面试完,我想去读个硕士了T_T
  13. 使用vuepress搭建一个完全免费的个人网站
  14. 低照度图像 颜色恢复 matlab,低照度图像修复方法总结
  15. TPM、TCM分别是什么?
  16. 戴口罩人脸识别、高精度人脸识别解决方案
  17. Matlab — 常见矩阵生成及矩阵运算
  18. 牛客SQL 大厂面试真题 某滴打车 6套代码及解析
  19. 2022-2027年中国图书零售连锁市场竞争态势及行业投资前景预测报告
  20. 汇顶科技外包java_【汇顶科技有限公司Java面试】2020春季校招后端Java-看准网

热门文章

  1. Android移动开发之【Android实战项目】textview实现文字逐字显示效果
  2. Python程序设计题解【蓝桥杯官网题库】 DAY4【补】-基础练习
  3. 使用SIFT特征提取和K-Means方法对图片进行分类
  4. 发那科机器人注油_如何给发那科机器人做三年保养?干货!
  5. unity延迟执行下一行代码_Python代码在Linux环境下执行错误异常
  6. java开发面试复试_java开发实习生面试经历
  7. 企业网站建设量身定做的三项基本要素
  8. 网站功能页面构建有何技巧?
  9. 网站发布外链如何防止后期被删除?
  10. antd 动态添加表单_react Ant Design 动态生成表单,并带验证