import urllib2class Spider:"""内涵段子爬虫类"""def loadPage(self, page):"""@brief 定义一个url请求网页的方法@param page 需要请求的第几页@returns 返回的页面html"""url = "http://www.neihan8.com/article/list_5_" + str(page)
+ ".html"#User-Agent头user_agent = 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT
6.1; Trident/5.0'headers = {'User-Agent': user_agent}req = urllib2.Request(url, headers = headers)response = urllib2.urlopen(req)html = response.read()print html#return html

if __name__ == '__main__':"""======================内涵段子小爬虫======================"""print '请按下回车开始'raw_input()#定义一个Spider对象mySpider = Spider()mySpider.loadpage(1)

def loadPage(self, page):"""@brief 定义一个url请求网页的方法@param page 需要请求的第几页@returns 返回的页面html"""url = "http://www.neihan8.com/article/list_5_" + str(page)
+ ".html"#User-Agent头user_agent = 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT
6.1; Trident/5.0'headers = {'User-Agent': user_agent}req = urllib2.Request(url, headers = headers)response = urllib2.urlopen(req)html = response.read()gbk_html = html.decode('gbk').encode('utf-8')# print gbk_htmlreturn gbk_html　

import re

def loadPage(self, page):"""@brief 定义一个url请求网页的方法@param page 需要请求的第几页@returns 返回的页面html"""url = "http://www.neihan8.com/article/list_5_" + str(page)
+ ".html"#User-Agent头user_agent = 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT
6.1; Trident/5.0'headers = {'User-Agent': user_agent}req = urllib2.Request(url, headers = headers)response = urllib2.urlopen(req)html = response.read()gbk_html = html.decode('gbk').encode('utf-8')#找到所有的段子内容<div class = "f18 mb20"></div>#re.S 如果没有re.S 则是只匹配一行有没有符合规则的字符串，如果没有则下一行重新匹配# 如果加上re.S 则是将所有的字符串将一个整体进行匹配pattern = re.compile(r'<div.*?class="f18 mb20">(.*?)</di
v>', re.S)item_list = pattern.findall(gbk_html)return item_listdef printOnePage(self, item_list, page):"""@brief 处理得到的段子列表@param item_list 得到的段子列表@param page 处理第几页"""print "******* 第 %d 页 爬取完毕...*******" %pagefor item in item_list:print "================"print item

def printOnePage(self, item_list, page):"""@brief 处理得到的段子列表@param item_list 得到的段子列表@param page 处理第几页"""print "******* 第 %d 页 爬取完毕...*******" %pagefor item in item_list:print "================"item = item.replace("<p>", "").replace("</p>", "").repl
ace("<br />", "")print item

def writeToFile(self, text):
'''@brief 将数据追加写进文件中@param text 文件内容
'''myFile = open("./duanzi.txt", 'a') #追加形式打开文件myFile.write(text)myFile.write("---------------------------------------------
--------")myFile.close()

def printOnePage(self, item_list, page):
'''@brief 处理得到的段子列表@param item_list 得到的段子列表@param page 处理第几页
'''print "******* 第 %d 页 爬取完毕...*******" %pagefor item in item_list:# print "================"item = item.replace("<p>", "").replace("</p>", "").repl
ace("<br />", "")# print itemself.writeToFile(item)

def doWork(self):
'''让爬虫开始工作
'''while self.enable:try:item_list = self.loadPage(self.page)except urllib2.URLError, e:print e.reasoncontinue#对得到的段子item_list处理self.printOnePage(item_list, self.page)self.page += 1 #此页处理完毕，处理下一页print "按回车继续..."print "输入 quit 退出"command = raw_input()if (command == "quit"):self.enable = Falsebreak

import sys
from urllib import request, parse
import reimport chardetclass Spider:def __init__(self):# 初始化起始页位置self.page = 1# 爬取开关, 如果为True继续爬取, 反之停止self.switch = Truedef loadPage(self):"""下载页面"""print("正在下载数据.....")url = "http://www.neihan8.com/article/list_5_"+str(self.page)+".html"# 字符串拼接urlheaders = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"}# 构建请求, 返回req对象req = request.Request(url, headers=headers)# 发送请求, 返回response相应response = request.urlopen(req)# html为每页的源码, 格式为字符串html = response.read()encode_type = chardet.detect(html)# print("编码格式为:--->"+str(encode_type))html = html.decode(encode_type['encoding'])# print(html)# 创建一个正则表达式规则对象, 匹配每页里段子内容, re.S表示为匹配去啊怒字符串对象pattern = re.compile('<div class="f18 mb20">(.*?)</div>', re.S)# 将正则匹配对象应用到html源码字符串内, 返回这个页面内的所有段子的列表content_list = pattern.findall(html)self.dealPage(content_list)def dealPage(self, content_list):"""处理每页的段子"""print("正在处理数据.....")for content in content_list:# 将集合里的每个段子挨个处理, 替换掉无用数据
content = content.replace("<p>", "").replace("<br />", "").replace("&ldquo;", "").replace("&rdquo;", "").replace("                        　　", "")# print(content)
            self.writePage(content)print("正在保存数据.....")def writePage(self, content):"""把每条段子逐个写入到文件中"""with open("段子.txt", "a") as f:f.write(content)def startWork(self):"""控制爬虫爬行"""# 爬虫开关while self.switch:self.loadPage()command = input("如果继续爬取, 请按回车(退出输入quit)")if command == "quit":print("OKKKKKKKKKKKKKK")self.switch = Falseself.page += 1print("谢谢使用!!!")if __name__ == "__main__":neihan = Spider()neihan.startWork()

View Code