C#网站图片下载器，网络爬虫，仅供学习参考使用，不可用于商业活动

程序一共包含3个文件，主入口文件Program.cs，Html下载解析文件HtmlDown.cs，文件缓存文件FileCache.cs，其中Program.cs处理常用逻辑，HtmlDown.cs负责下载Html并解析其中标签选取你想要下载的文件，FileCache.cs用来记录已经下载的Url文件，防止重复下载，重复抓取，网址自己修改www.xxx.com,解析方式需要根据下载网页自己修改在，在GetJpeg函数和GetHref函数中

代码下载https://download.csdn.net/download/g313105910/16715818

Program文件

using RestSharp;
using RestSharp.Extensions;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;namespace PictureSpider
{public class Program{public static bool IsExit = false;public static void Main(string[] args){Console.WriteLine("程序已启动");Task task = new Task(SpidersTask);task.Start();while (true){if (Console.Read() == 101){IsExit = true;Console.WriteLine("等待程序保存");}}}public static void SpidersTask(){try{List<Storage> storages = new List<Storage>();FileCache.Load();if (FileCache.Datas.Count == 0){Storage storage = new Storage();storage.Key = "main.html";storage.Value = "https://www.xxx.com";storages.Add(storage);}else{//加载最后一条记录var lastData = FileCache.Datas.Last();Storage storage = new Storage();storage.Key = lastData.Key;storage.Value = lastData.Value;storages.Add(storage);//删除最后一条，让系统从最后一条开始执行FileCache.Datas.Remove(storage.Key);}while (true){if (storages.Count > 0){Console.WriteLine("按e加回城退出程序！");if (IsExit == false){HtmlDown htmlDown = new HtmlDown();var first = storages.First();if (!FileCache.CheckIsExist(first.Key)){Console.WriteLine($"storages.Count={storages.Count.ToString()},first.Key={first.Key},first.Value={first.Value}");var list = htmlDown.Start("out/" + first.Key + "/", first.Key, first.Value);FileCache.Add(first.Key, first.Value);storages.RemoveAt(0);for (int i = 0; i < list.Count; i++){storages.Add(list[i]);}}else{Console.WriteLine($"KeyExist storages.Count={storages.Count.ToString()}.Key={first.Key},list.Value={first.Value}");storages.RemoveAt(0);}}else{Console.WriteLine($"IsExit=={IsExit.ToString()}");break;}}else{Console.WriteLine($"storages.Count=0");break;}}FileCache.Save();Console.WriteLine("程序退出");}catch (WebException webEx){Console.WriteLine(webEx.Message.ToString());}}}
}

HtmlDown文件

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;namespace PictureSpider
{public class HtmlDown{public List<Storage> Start(string path,string name,string url){WebClient webClient = new WebClient();webClient.Credentials = CredentialCache.DefaultCredentials;webClient.Headers.Add("Accept", "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*");webClient.Headers.Add("Accept-Language", "zh-CN");webClient.Headers.Add("Host", "");webClient.Headers.Add("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1)");webClient.Headers.Add("Content-Type", "application/x-www-form-urlencoded");webClient.Credentials = CredentialCache.DefaultCredentials;//获取或设置用于向Internet资源的请求进行身份验证的网络凭据Byte[] pageData = webClient.DownloadData(url); //从指定网站下载数据string pageHtml = Encoding.Default.GetString(pageData);  //如果获取网站页面采用的是GB2312，则使用这句    FileCache.DirectoryCheak(path);//获取网页内容并存储using (System.IO.StreamWriter file = new System.IO.StreamWriter(path+name+ @"-html.txt", true)){file.WriteLine(pageHtml);}//获取JpegList<string> urls = GetJpeg(pageHtml);using (System.IO.StreamWriter file = new System.IO.StreamWriter(path + name + @"-url.txt", true)){for (int i = 0; i < urls.Count(); i++){file.WriteLine(urls[i]);}}List<Storage> urlList = GetHref(pageHtml);return urlList;}private List<Storage> GetHref(string pageHtml){List<Storage> urlList = new List<Storage>();var urlSplitThree = GetValue(pageHtml, "a href", " ");for (int i = 0; i < urlSplitThree.Count(); i++){int index = urlSplitThree[i].IndexOf(".html");if (index == (urlSplitThree[i].Length - 5 - 1) && index != -1 && urlSplitThree[i].Length <= 30){Storage storage = new Storage();storage.Key = urlSplitThree[i].Replace("a href=", string.Empty);storage.Value = "https://www.xxx.com/" + storage.Key;urlList.Add(storage);}}return urlList;}private List<string> GetJpeg(string pageHtml){//找到'进行分割var urlSplit = pageHtml.Split('\'');List<string> urls = new List<string>();for (int i = 0; i < urlSplit.Length; i++){int index = urlSplit[i].IndexOf(".jpeg");if (index == (urlSplit[i].Length - 5) && index != -1){string pictrueUrl = "https:" + urlSplit[i];urls.Add(pictrueUrl);FileCache.AddUrls(pictrueUrl,pictrueUrl);}}//找到"进行分割var urlSplitTwo = pageHtml.Split('\"');for (int i = 0; i < urlSplitTwo.Length; i++){int index = urlSplitTwo[i].IndexOf(".jpeg");if (index == (urlSplitTwo[i].Length - 5) && index != -1){string pictrueUrl = "https:" + urlSplitTwo[i];urls.Add(pictrueUrl);FileCache.AddUrls(pictrueUrl, pictrueUrl);}}return urls;}private void Down(string url){try{//byte[] Bytes = webClient.DownloadData(pictrueUrl);//using (MemoryStream ms = new MemoryStream(Bytes))//{//    Image outputImg = Image.FromStream(ms);//    outputImg.Save($"{i.ToString()}.jpeg");//}}catch (Exception ex){Console.WriteLine(ex.ToString());}}/// <summary>/// 获得字符串中开始和结束字符串/// </summary>/// <param name="str">字符串</param>/// <param name="start">开始</param>/// <param name="end">结束</param>/// <returns></returns>private List<string> GetValue(string str, string start, string end){return Regex.Matches(str, "[.\\s\\S]*?(?<=(" + start + "))[.\\s\\S]*?(?<=(" + end + "))", RegexOptions.Multiline | RegexOptions.Singleline).Cast<Match>().Select(t => t.Value.LastIndexOf(start) == -1 ? t.Value : t.Value.Substring(t.Value.LastIndexOf(start))).ToList();}}
}

FileCache文件

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;namespace PictureSpider
{public class Storage{public string Key { get; set; }public string Value { get; set; }}public static class FileCache{public static Dictionary<string, string> Datas = new Dictionary<string, string>();public static Dictionary<string, string> Urls = new Dictionary<string, string>();/// <summary>/// 效验文件夹，没有就创建/// </summary>public static void DirectoryCheak(string path){if (false == System.IO.Directory.Exists(path)){System.IO.Directory.CreateDirectory(path);}}public static void AddUrls(string key, string value){string data;//没有才可以添加if (!Urls.TryGetValue(key, out data)){Urls.Add(key, value);}}public static void Add(string key,string value){string data;if (Datas.TryGetValue(key, out data)){Datas[key] = value;}else{Datas.Add(key, value);}}public static bool CheckIsExist(string key){//更新数据string data;//需要有数据才可以数据if (Datas.TryGetValue(key, out data)){return true;}else{return false;}}public static void Load(){try{DirectoryCheak("cache");// 从文件中读取并显示每行string line = "";using (StreamReader sr = new StreamReader("cache/htmllist.txt")){while ((line = sr.ReadLine()) != null){try{var str = line.Split('\t');Add(str[0], str[1]);}catch (Exception ex){Console.WriteLine("line=" + line + ",ex=" + ex.ToString());}}}}catch(Exception ex){Console.WriteLine(ex.ToString());}}public static void CheckFile(string filename){if (System.IO.File.Exists(Path.GetFullPath(filename))){File.Delete(Path.GetFullPath(filename));}}public static void Save(){DirectoryCheak("cache");CheckFile("cache/htmllist.txt");{StreamWriter sw = new StreamWriter("cache/htmllist.txt");foreach (var item in Datas){sw.WriteLine(item.Key + "\t" + item.Value);}sw.Flush();sw.Close();}{StreamWriter sw = new StreamWriter("cache/htmlurl.txt", true);foreach (var url in Urls){sw.WriteLine(url.Value);}sw.Flush();sw.Close();}}}
}

C#网站图片下载器，网络爬虫，仅供学习参考使用，不可用于商业活动相关推荐

人力资源学python有意义吗-python爬虫抖音个人资料仅供学习参考切勿用于商业...
本文仅供学习参考切勿用于商业本次爬取使用fiddler+模拟器(下载抖音APP)+pycharm 1. 下载最新版本的fiddler(自行百度下载),以及相关配置 1.1.依次点击,菜单栏-Too ...
DNS欺骗攻击——实现受害者访问任何网站都转向到攻击者指定的网站（仅供学习参考）
DNS欺骗攻击--实现受害者访问任何网站都转向到攻击者指定的网站 (仅供学习参考) 情景要求: 同一局域网内受害者IP:192.168.26.130 攻击者系统Kali Linux,IP地址:192 ...
15kw充电桩电源模块，三相电源模块电路图，仅供学习参考
15kw充电桩电源模块,三相电源模块电路图,仅供学习参考,只包含电路图和PCB文件,无软件代码 id=623312177000&
截止20210605的微博部分API汇总——仅供学习参考使用
文章目录免责声明背景 API列表搜索主评论API REQUEST RESPOND 搜索次评论API REQUEST RESPOND 转发评论API REQUEST RESPOND 上传图片API ...
python 毕业设计源码博客_blogs: 仅供学习参考使用，Python Django毕业设计——个人博客系统...
大学毕业设计个人Blog博客系统基于"python3.6.5"和"Django2"开发的的个人博客系统. 层级结构 cd bbs; tree . ├── bb ...
canal mysql重置_canal: 首先装完阿里的canal，然后数据库同步，仅供学习参考
背景需要将数据库增量内容同步到另外的数据库,两者保持一致.包含增删改,数据库创建.删除,数据库名称修改,内容清空,表结构修改,索引创建.修改.删除. 方案采用阿里巴巴Canal,用于监听mysql ...
python自动生成采集规则_【仅供学习参考】Python多线程池采集小说，超简单！
[Python] 纯文本查看复制代码#采集小说lingdiankanshu.co import requests from lxml import etree from multiprocessin ...
欢迎随时回家——python制作一个火车票抢票版本（附完整代码，仅供学习参考）
嗨害大家好鸭~我是小熊猫
Python控制手机摄像头拍照（自己的），并把照片自动发送到邮箱（仅供学习参考）
让我们直接开始吧!

C#网站图片下载器，网络爬虫，仅供学习参考使用，不可用于商业活动

C#网站图片下载器，网络爬虫，仅供学习参考使用，不可用于商业活动相关推荐

最新文章

热门文章