自己用C#写一个采集器、蜘蛛(zz)
效果图：
代码如下：
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.Web;
using System.IO;
using System.Collections;
using System.Text.RegularExpressions;
namespace chinaz
{
class Program
{
static void Main(string[] args)
{
string cookie = null;
using (StreamReader sr = new StreamReader("cookie.txt"))
{
cookie = sr.ReadToEnd();
sr.Close();
}
//string tmp = SRWebClient.GetPage("http://bbs.chinaz.com/Members.html?page=1&sort=CreateDate&desc=true&keyword=", Encoding.UTF8, cookie);
int a = int.Parse(Console.ReadLine());
int b = int.Parse(Console.ReadLine());
string url = Console.ReadLine();
Hashtable hash = new Hashtable();
Encoding encoding = Encoding.GetEncoding(Console.ReadLine());
for (int i = a; i <= b; i++)
{
string html = SRWebClient.GetPage(string.Format(url, i), encoding, cookie);
//Console.WriteLine(html);
if (html != null && html.Length > 1000)
{
Match m = Regex.Match(html, @"\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*", RegexOptions.Compiled | RegexOptions.IgnoreCase);
while (m != null && m.Value != null && m.Value.Trim() != string.Empty)
{
if (!hash.Contains(m.Value))
{
Console.WriteLine(m.Value);
using (StreamWriter sw = new StreamWriter("mail.txt", true))
{
sw.WriteLine(m.Value);
sw.Close();
}
hash.Add(m.Value, string.Empty);
}
m = m.NextMatch();
}
}
}
Console.Write("完成");
Console.ReadLine();
}
}
public class SRWebClient
{
public CookieCollection cookie;
public SRWebClient()
{
cookie = null;
}
#region 从包含多个 Cookie 的字符串读取到 CookieCollection 集合中
private static void AddCookieWithCookieHead(ref CookieCollection cookieCol, string cookieHead, string defaultDomain)
{
if (cookieCol == null) cookieCol = new CookieCollection();
if (cookieHead == null) return;
string[] ary = cookieHead.Split(';');
for (int i = 0; i < ary.Length; i++)
{
Cookie ck = GetCookieFromString(ary[i].Trim(), defaultDomain);
if (ck != null)
{
cookieCol.Add(ck);
}
}
}
#endregion
#region 读取某一个 Cookie 字符串到 Cookie 变量中
private static Cookie GetCookieFromString(string cookieString, string defaultDomain)
{
string[] ary = cookieString.Split(',');
Hashtable hs = new Hashtable();
for (int i = 0; i < ary.Length; i++)
{
string s = ary[i].Trim();
int index = s.IndexOf("=");
if (index > 0)
{
hs.Add(s.Substring(0, index), s.Substring(index + 1));
}
}
Cookie ck = new Cookie();
foreach (object Key in hs.Keys)
{
if (Key.ToString() == "path") ck.Path = hs[Key].ToString();
else if (Key.ToString() == "expires")
{
//ck.Expires=DateTime.Parse(hs[Key].ToString();
}
else if (Key.ToString() == "domain") ck.Domain = hs[Key].ToString();
else
{
ck.Name = Key.ToString();
ck.Value = hs[Key].ToString();
}
}
if (ck.Name == "") return null;
if (ck.Domain == "") ck.Domain = defaultDomain;
return ck;
}
#endregion
/**/
/// <TgData>
/// <Alias>下载Web源代码</Alias>
/// </TgData>
public string DownloadHtml(string URL, bool CreateCookie)
{
try
{
HttpWebRequest request = HttpWebRequest.Create(URL) as HttpWebRequest;
if (cookie != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookie);
}
request.AllowAutoRedirect = false;
//request.MaximumAutomaticRedirections = 3;
request.Timeout = 20000;
HttpWebResponse res = (HttpWebResponse)request.GetResponse();
string r = "";
System.IO.StreamReader S1 = new System.IO.StreamReader(res.GetResponseStream(), System.Text.Encoding.Default);
try
{
r = S1.ReadToEnd();
if (CreateCookie)
cookie = res.Cookies;
}
catch (Exception er)
{
//Log l = new Log();
//l.writelog("下载Web错误", er.ToString());
}
finally
{
res.Close();
S1.Close();
}
return r;
}
catch
{
}
return string.Empty;
}
/**/
/// <TgData>
/// <Alias>下载文件</Alias>
/// </TgData>
public long DownloadFile(string FileURL, string FileSavePath, bool CreateCookie)
{
long Filelength = 0;
HttpWebRequest req = HttpWebRequest.Create(FileURL) as HttpWebRequest;
if (cookie != null)
{
req.CookieContainer = new CookieContainer();
req.CookieContainer.Add(cookie);
}
req.AllowAutoRedirect = true;
HttpWebResponse res = req.GetResponse() as HttpWebResponse;
if (CreateCookie)
cookie = res.Cookies;
System.IO.Stream stream = res.GetResponseStream();
try
{
Filelength = res.ContentLength;
byte[] b = new byte[512];
int nReadSize = 0;
nReadSize = stream.Read(b, 0, 512);
System.IO.FileStream fs = System.IO.File.Create(FileSavePath);
try
{
while (nReadSize > 0)
{
fs.Write(b, 0, nReadSize);
nReadSize = stream.Read(b, 0, 512);
}
}
finally
{
fs.Close();
}
}
catch (Exception er)
{
//Log l = new Log();
//l.writelog("下载文件错误", er.ToString());
}
finally
{
res.Close();
stream.Close();
}
return Filelength;
}
/**/
/// <TgData>
/// <Alias>提交数据</Alias>
/// </TgData>
public string Request(string RequestPageURL, RequestData Data, bool CreateCookie)
{
StreamReader reader = null;
HttpWebResponse response = null;
HttpWebRequest request = null;
try
{
string StrUrl = RequestPageURL;
request = HttpWebRequest.Create(StrUrl) as HttpWebRequest;
string postdata = Data.GetData();
request.Referer = RequestPageURL;
request.AllowAutoRedirect = false;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
request.Timeout = 20000;
if (cookie != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookie);
}
Uri u = new Uri(StrUrl);
if (postdata.Length > 0) //包含要提交的数据 就使用Post方式
{
request.ContentType = "application/x-www-form-urlencoded"; //作为表单请求
request.Method = "POST"; //方式就是Post
//把提交的数据换成字节数组
Byte[] B = System.Text.Encoding.UTF8.GetBytes(postdata);
request.ContentLength = B.Length;
System.IO.Stream SW = request.GetRequestStream(); //开始提交数据
SW.Write(B, 0, B.Length);
SW.Close();
}
response = request.GetResponse() as HttpWebResponse;
if (CreateCookie)
//cookie = response.Cookies;
AddCookieWithCookieHead(ref cookie, response.Headers["Set-Cookie"], request.RequestUri.Host);
reader = new StreamReader(response.GetResponseStream(), Encoding.Default);
return reader.ReadToEnd();
}
catch (Exception ex)
{
string x = ex.StackTrace;
}
finally
{
if (response != null)
response.Close();
}
return string.Empty;
}
public bool PostDownload(RequestData Data, out string file)
{
file = null;
StreamReader reader = null;
HttpWebResponse response = null;
HttpWebRequest request = null;
try
{
string StrUrl = "http://www.imobile.com.cn/wapdiyringdownload.php";
request = HttpWebRequest.Create(StrUrl) as HttpWebRequest;
string postdata = Data.GetData();
request.Referer = StrUrl;
request.AllowAutoRedirect = false;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
request.Timeout = 20000;
if (cookie != null)
{
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(cookie);
}
Uri u = new Uri(StrUrl);
if (postdata.Length > 0) //包含要提交的数据 就使用Post方式
{
request.ContentType = "application/x-www-form-urlencoded"; //作为表单请求
request.Method = "POST"; //方式就是Post
//把提交的数据换成字节数组
Byte[] B = System.Text.Encoding.UTF8.GetBytes(postdata);
request.ContentLength = B.Length;
System.IO.Stream SW = request.GetRequestStream(); //开始提交数据
SW.Write(B, 0, B.Length);
SW.Close();
}
response = request.GetResponse() as HttpWebResponse;
string des = response.Headers["Content-Disposition"].Trim();
file = des.Substring(des.IndexOf("filename=") + 9);
file = new Random().Next(100).ToString() + "/" + file;
System.IO.Stream stream = response.GetResponseStream();
try
{
int Filelength = (int)response.ContentLength;
byte[] b = new byte[512];
int nReadSize = 0;
nReadSize = stream.Read(b, 0, 512);
System.IO.FileStream fs = System.IO.File.Create("f:/mobileMusic/" + file);
try
{
while (nReadSize > 0)
{
fs.Write(b, 0, nReadSize);
nReadSize = stream.Read(b, 0, 512);
}
}
finally
{
fs.Close();
}
}
catch (Exception er)
{
//Log l = new Log();
//l.writelog("下载文件错误", er.ToString());
}
finally
{
response.Close();
stream.Close();
}
}
catch (Exception ex)
{
string x = ex.StackTrace;
}
finally
{
if (response != null)
response.Close();
}
return true;
}
#region GetPage
/// <summary>
/// 获取源代码
/// </summary>
/// <param name="url"></param>
/// <param name="coding"></param>
/// <param name="TryCount"></param>
/// <returns></returns>
public static string GetPage(string url, Encoding encoding, int TryCount)
{
for (int i = 0; i < TryCount; i++)
{
string result = GetPage(url, encoding, null);
if (result != null && result != string.Empty)
return result;
}
return string.Empty;
}
/// <summary>
/// 获取源代码
/// </summary>
/// <param name="url"></param>
/// <param name="coding"></param>
/// <returns></returns>
public static string GetPage(string url, Encoding encoding, string cookie)
{
HttpWebRequest request = null;
HttpWebResponse response = null;
StreamReader reader = null;
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2;)";
request.Timeout = 20000;
request.AllowAutoRedirect = false;
if (cookie != null)
request.Headers["Cookie"] = cookie;
response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
{
reader = new StreamReader(response.GetResponseStream(), encoding);
string html = reader.ReadToEnd();
return html;
}
}
catch
{
}
finally
{
if (response != null)
{
response.Close();
response = null;
}
if (reader != null)
reader.Close();
if (request != null)
request = null;
}
return string.Empty;
}
#endregion
}
public class RequestData
{
Hashtable hash = new Hashtable();
public RequestData()
{
}
public string GetData()
{
string r = "";
foreach (string key in hash.Keys)
{
if (r.Length > 0) r += "&";
r += key + "=" + hash[key];
}
return r;
}
public void AddField(string Field, string Value)
{
hash[Field] = Value;
}
}
}
转载于:https://www.cnblogs.com/stu-acer/archive/2008/10/11/1308712.html
自己用C#写一个采集器、蜘蛛(zz)相关推荐
- python从零写一个采集器:获取网页信息
博客链接 https://uublog.com/article/20170216/python-extarct-html-info/ 前言 获取内容,比较纠结是用BeautifulSoup还是直接用正 ...
- 使用IntentService给自己的Android应用写一个文件下载器。
接着上一篇的http://www.cnblogs.com/zhengxt/p/3657833.html,当我们想给自己的APP写一个文件下载器时,可以用重写IntentService来实现. 使用In ...
- 用Pyqt5写一个视频播放器
用Pyqt5写一个视频播放器 编写方式: Qt Designer设计,pyuic5转化为.py文件. 播放器效果: 代码链接:https://github.com/GRF-Sunomikp31/Qt_ ...
- 从0开始写一个播放器系列-开篇
从0开始写一个播放器系列-开篇 阅读本系列博客所需要具备的知识: js, ts , canvas Api , DOM, webpack, 不懂也没有太大的关系, 我会捎带着写出来, 剩下的自行百度 第 ...
- python写exploit采集器
前言: 根据天文地理与风水学,我掐指一算的看到了一篇不错的文章,文章里面写到整理exploit 我顿时心理想写一个exploit采集器,那么说时迟那时快.就开始写了 代码: 思路:http://exp ...
- python写一个文件下载器_python使用tcp实现一个简单的下载器
上一篇中介绍了tcp的流程,本篇通过写一个简单的文件下载器程序来巩固之前学的知识. 文件下载器的流程如下: 客户端: 输入目标服务器的ip和port 输入要下载文件的名称 从服务器下载文件保存到本地 ...
- 用python写一个下载器
最近有人反馈部分用户使用迅雷X会出现低于下载速度600K的限速情况,为了解决这个问题,今天我们来写一个不限速的下载器. 把代码里http://xxxxxxx/xxxxxxxx/xxxxxx/xxxxx ...
- 用qt帮我写一个播放器，要求实时解码h264
好的,我来帮你写一个基于 Qt 的播放器,它可以实时解码 H.264 视频流. 首先,你需要安装 Qt 开发环境和相关工具,然后打开 Qt Creator 创建一个新项目.在新项目中,你需要添加一些代 ...
- python写一个文件下载器_Python3使用TCP编写一个简易的文件下载器
原标题:Python3使用TCP编写一个简易的文件下载器 利用Python3来实现TCP协议,和UDP类似.UDP应用于及时通信,而TCP协议用来传送文件.命令等操作,因为这些数据不允许丢失,否则会造 ...
最新文章
- Properties持久的属性集
- 物联网背后的网络安全风险
- php 文字水印如何居中,php文字水印和php图片水印实现代码(二种加水印方法)
- django权限二(多级菜单的设计以及展示)
- 驱动框架5——基于驱动框架写led驱动
- soapui返回值类型都有哪些_滚珠丝杠的常用类型都有哪些？
- DCMTK：Receiving Images from PACS using DCMSCU
- 【云周刊】第120期：麒麟来了！PUE逼近1.0，阿里展示液冷黑科技
- 职员)2015-11-09 星期一 日志
- GA-T1400协议--注册注销
- 整整1600套Axure原型图设计源文件UI UX交互设计案例
- 随机微分方程与 Ito Lemma 的关系
- 开源H5棋牌 cocos creator微信棋牌小游戏 幼麟棋牌服务端分析笔记
- 变频器的技术应用：接线与参数设置
- cruzer php sandisk 闪迪u盘量产工具_SanDisk Cruzer Micro(U盘量产工具) V1.0 电脑版
- 开发游戏引擎需要具备什么
- Google浏览器设置不自动更新：关闭谷歌浏览器自动更新方法（总是自动更新提示失败）
- date函数详细用法
- p系统ndows10的功能更新1903,Win10 1903到底有什么不同？Win10 1903更新内容汇总
- 电缆计算机作用,铠装计算机电缆DJYV用途