discuz论坛的抓取

discuz论坛通用URL地址：

URL+?mod=my&q=关键字（关键字的编码gb2312）

例如： 'http://bbs.anzhi.com/search.php?mod=my&q=htc+%CA%D6%BB%FA';

而真实的sId的值是存储在head里面的location中的。

例如:location=http://search.bbs.hiapk.com/f/search?q=%E6%89%8B%E6%9C%BA&sId=8520930&ts=1355197250&mySign=5785b7cb&searchLevel=3&menu=1&rfh=1&qs=txt.tsort.a&orderField=posted&orderType=desc

           string url = string.Format("http://bbs.hiapk.com/search.php?mod=my&q={0}", crawlerModel.Keyword);HTMLContent= GetHteml(url);

private string GetHteml(string url){HttpWebRequest request = null;HttpWebResponse response = null;string gethost = string.Empty;CookieContainer cc = new CookieContainer();string Cookiesstr = string.Empty;try{request = (HttpWebRequest)WebRequest.Create(url);request.Method = "GET";request.KeepAlive = true;request.Headers.Add("Cookie:" + Cookiesstr);request.CookieContainer = cc;request.AllowAutoRedirect = false;response = (HttpWebResponse)request.GetResponse();//设置cookieCookiesstr = request.CookieContainer.GetCookieHeader(request.RequestUri);//取再次跳转链接//StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("gb2312"));//string ss = sr.ReadToEnd();gethost =response.Headers["location"];request.Abort();response.Close();              }catch (Exception){//第一次GET出错return "";}try{request = (HttpWebRequest)WebRequest.Create(gethost);request.Method = "GET";request.KeepAlive = true;request.Headers.Add("Cookie:" + Cookiesstr);request.CookieContainer = cc;request.AllowAutoRedirect = false;response = (HttpWebResponse)request.GetResponse();//设置cookieCookiesstr = request.CookieContainer.GetCookieHeader(request.RequestUri);//取再次跳转链接StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("utf-8"));string ss = sr.ReadToEnd();string pattern = @"<a.+?>按时间排序</a>";if (!Regex.IsMatch(ss, pattern)) return "";gethost = "http://search.bbs.anzhi.com"+GetURL(Regex.Match(ss, pattern).Value.Replace("&","&"));request.Abort();sr.Close();response.Close();}catch (Exception){//第一次GET出错return "";}try{request = (HttpWebRequest)WebRequest.Create(gethost);request.Method = "GET";request.KeepAlive = true;request.Headers.Add("Cookie:" + Cookiesstr);request.CookieContainer = cc;request.AllowAutoRedirect = false;response = (HttpWebResponse)request.GetResponse();//设置cookieCookiesstr = request.CookieContainer.GetCookieHeader(request.RequestUri);//取再次跳转链接StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("utf-8"));string ss = sr.ReadToEnd();request.Abort();sr.Close();response.Close();return ss;}catch (Exception){//第一次GET出错return "";}}

       private string GetURL(string urlHtml){MatchCollection matchList;Regex regex = new Regex(@"f="".*?""");matchList = regex.Matches(urlHtml);if (matchList.Count > 0){return matchList[0].Value.Substring(3, matchList[0].Value.Length - 4);}return "";}

小例子：

using System;
using System.Collections.Generic;
using IWOMWebCrawlerDbLayer.DAL;
using IWOMWebCrawlerDbLayer.Model;
using System.Text;
using System.Text.RegularExpressions;
using System.Collections;
using System.Threading;
using IWOMWebCrawlerDbLayer.Common;
using HtmlAgilityPack;
using System.Net;
using System.IO;namespace IWOMWebCrawlerApp.Crawler
{public class tousue_teizi : AbstractSearchEngine{public tousue_teizi(){this.SearchID = 1600;this.SearchName = "投诉易--帖子";}private int pageId=0;/// <summary>/// 根据任务生成抓取要素/// </summary>protected override void initCrawlerModel(IwomTask taskItem){crawlerModel = new CrawlerModel();crawlerModel.Keyword = CommonFunction.AssembledKeyword(taskItem.KeyWord, IWOMWebCrawlerDbLayer.Common.KeyWordUrlEncode.Normal, false);//关键词crawlerModel.PageSize = crawlerModel.PageSize > 10 ? 10 : taskItem.GetItems;//每页大小crawlerModel.Postion = taskItem.Task_Postion;}/// <summary>/// 根据任务要素构造抓取的url/// </summary>protected override string createUrl(int pageIndex){if (pageIndex != 0) return "";string url = string.Format("http://www.tousue.com/search.php?mod=my&q={0}", crawlerModel.Keyword);return url;}/// <summary>/// 每抓取一页都间隔的时间/// </summary>protected override void PageSleep(){Thread.Sleep(500);}/// <summary>/// 页面的编码/// </summary>protected override Encoding getPageEncoding(){return Encoding.GetEncoding("utf-8");}/// <summary>/// 根据内容判断是否被封禁了/// </summary>protected override bool checkContentIsForbat(string HTMLContent){return true;}/// <summary>/// 根据内容判断是否是最后一页了/// </summary>protected override bool checkContentIsLastPage(string HTMLContent){return false;}/// <summary>/// 根据网页信息得到文章集合/// </summary>protected override List<CrawlerResult> GetArticleByHtml(string HTMLContent, int task_ID){string url = string.Format("http://www.tousue.com/search.php?mod=my&q={0}", crawlerModel.Keyword);HTMLContent = GetHteml(url,pageId);string mainId = "result-items";XPathModel model = new XPathModel();model.listXPath = "./ul/li";model.titleXPath = "./h3/a";model.urlXPath = "./h3/a";model.timeXPath = "./p[3]";List<CrawlerResult> arrayList = CommonFunction.GetListByXPath(HTMLContent, task_ID, mainId, model, TitleMethod, CommonFunction.UrlMethod, TimeMethod);return arrayList;}/// <summary>/// 获得解析的URL/// </summary>private string GetURL(string urlHtml){MatchCollection matchList;Regex regex = new Regex(@"f="".*?""");matchList = regex.Matches(urlHtml);if (matchList.Count > 0){return matchList[0].Value.Substring(3, matchList[0].Value.Length - 4);}return "";}private string GetHteml(string url,int pageId){HttpWebRequest request = null;HttpWebResponse response = null;string gethost = string.Empty;CookieContainer cc = new CookieContainer();string Cookiesstr = string.Empty;try{request = (HttpWebRequest)WebRequest.Create(url);request.Method = "GET";request.KeepAlive = true;request.Headers.Add("Cookie:" + Cookiesstr);request.CookieContainer = cc;request.AllowAutoRedirect = false;response = (HttpWebResponse)request.GetResponse();//设置cookieCookiesstr = request.CookieContainer.GetCookieHeader(request.RequestUri);//取再次跳转链接//StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("gb2312"));//string ss = sr.ReadToEnd();gethost = response.Headers["location"];request.Abort();response.Close();}catch (Exception){//第一次GET出错return "";}try{request = (HttpWebRequest)WebRequest.Create(gethost);request.Method = "GET";request.KeepAlive = true;request.Headers.Add("Cookie:" + Cookiesstr);request.CookieContainer = cc;request.AllowAutoRedirect = false;response = (HttpWebResponse)request.GetResponse();//设置cookieCookiesstr = request.CookieContainer.GetCookieHeader(request.RequestUri);//取再次跳转链接StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("utf-8"));string ss = sr.ReadToEnd();string pattern = @"<a.+?>按时间排序</a>";if (!Regex.IsMatch(ss, pattern)) return "";gethost = "http://search.discuz.qq.com" + GetURL(Regex.Match(ss, pattern).Value.Replace("&", "&")) + "&page=" + pageId.ToString();request.Abort();sr.Close();response.Close();}catch (Exception){//第一次GET出错return "";}try{request = (HttpWebRequest)WebRequest.Create(gethost);request.Method = "GET";request.KeepAlive = true;request.Headers.Add("Cookie:" + Cookiesstr);request.CookieContainer = cc;request.AllowAutoRedirect = false;response = (HttpWebResponse)request.GetResponse();//设置cookieCookiesstr = request.CookieContainer.GetCookieHeader(request.RequestUri);//取再次跳转链接StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding("utf-8"));string ss = sr.ReadToEnd();request.Abort();sr.Close();response.Close();return ss;}catch (Exception){//第一次GET出错return "";}}private string TitleMethod(string content){string pattern = @"(?<=[\d]回答).+";if (Regex.IsMatch(content, pattern))return Regex.Match(content, pattern).Value;return content;}private DateTime TimeMethod(string content){string pattern = @"[\d]{4}-[\d]{1,2}-[\d]{1,2}";string temp = "";DateTime time = new DateTime();time = CommonFunction.GetTimeByChinese(content);if (time != DateTime.MinValue)return time;if (Regex.IsMatch(content, pattern))temp = Regex.Match(content, pattern).Value;DateTime.TryParse(temp, out time);return time;}/// <summary>/// 生成测试任务的方法/// </summary>protected override string initTestUrl(){this.HaseCreateTime = true;this.HasePageSize = 10;this.HaseSiteName = true;this.HaseSummary = true;pageId = 2;HaseAuthor = true;crawlerModel.Keyword = CommonFunction.AssembledKeyword("海尔", IWOMWebCrawlerDbLayer.Common.KeyWordUrlEncode.Normal, false);string url = string.Format("http://www.tousue.com/search.php?mod=my&q={0}", crawlerModel.Keyword);return url;}}
}

discuz论坛的抓取相关推荐

收集的材料关于数据库和抓取器方面的
同事的博客里面有很多不错的东西值得我这个水平不高的人学习顺便给他涨个人气抓取器: C#.NET GB2312编码转化为中文 c#验证码下载正则表达式替换---lamda表达式(UniC ...
python爬取论坛付费内容_Python爬虫抓取论坛关键字过程解析
前言: 之前学习了用python爬虫的基本知识,现在计划用爬虫去做一些实际的数据统计功能.由于前段时间演员的诞生带火了几个年轻的实力派演员,想用爬虫程序搜索某论坛中对于某些演员的讨论热度,并按照日期统 ...
disallow: /api.php,dz论坛如何禁止搜索引擎抓取任何内容？
这其实不只是针对于discuz这个程序建的网站,针对所有的网站都有效.网站的根目录有一个robots.txt文件.这个就相当于一个协议.它告诉搜索引擎,你可以抓取我网站里的哪些内容.所以,想要禁止搜索 ...
爬虫 - 抓取52论坛帖子列表
1. 前言这两周稍微得了点空闲,又对爬虫有相当兴趣,PythonPycharm都是现成的,说干就干. 从需求出发,起初是想做个爬图的程序,下点动漫美图什么的,非常实用.网站和图片URL都抓好了,结果 ...
Discuz 论坛模块全部帖子和评论爬取
Discuz 论坛模块全部帖子和评论爬取 Discuz 是一款由PHP编写的开源论坛 Discuz 官方论坛: https://www.discuz.net/forum.php 要爬取的页面地址: D ...
python爬取论坛付费内容_Python进阶量化交易专栏场外篇20-爬虫抓取股票论坛帖子...
欢迎大家订阅<教你用 Python 进阶量化交易>专栏!为了能够提供给大家更轻松的学习过程,笔者在专栏内容之外已陆续推出一些手记来辅助同学们学习本专栏内容,目前推出的扩展篇链接如下: 为了 ...
python 爬取财经新闻股票_Python进阶量化交易专栏场外篇20-爬虫抓取股票论坛帖子...
欢迎大家订阅<教你用 Python 进阶量化交易>专栏!为了能够提供给大家更轻松的学习过程,笔者在专栏内容之外已陆续推出一些手记来辅助同学们学习本专栏内容,目前推出的扩展篇链接如下: 为了 ...
爬虫实战，抓取论坛帖子内容
本文主要内容:以最短的时间写一个最简单的爬虫,可以抓取论坛的帖子标题和帖子内容. 本文受众:没写过爬虫的萌新. 入门 0.准备工作需要准备的东西: Python.scrapy.一个IDE或者随便什么 ...
[python 爬虫]Python爬虫抓取虎扑论坛帖子图片
自从可以实现抓取文字了,自然要尝试更多的类型,比如图片.我是一个有逛虎扑论坛习惯的人,经常会发现有些帖子的图片挺好看的想保存下来,但是如果人为保存的话,一个帖子至少都有二三十张,这将是一个庞大的工作量 ...

discuz论坛的抓取

discuz论坛的抓取相关推荐

最新文章

热门文章