#coding=utf-8
import url_manager, html_downloader, html_parser, html_outputerclass SpiderMain(object):def __init__(self):self.urls = url_manager.UrlManager()self.downloader = html_downloader.HtmlDownloader()self.parser = html_parser.HtmlParser()self.outputer = html_outputer.HtmlOutputer()def craw(self, root_url):self.urls.add_new_url(root_url)count = 1while self.urls.has_new_url():try:new_url = self.urls.get_new_url()print "craw %d : %s" %(count,new_url)html_cont = self.downloader.download(new_url)new_urls, new_data = self.parser.parse(new_url,html_cont)self.urls.add_new_urls(new_urls)self.outputer.collect_data(new_data)if count == 100:breakcount = count + 1except:print "craw failed"self.outputer.output_html()if __name__=="__main__":root_url = "http://www.1000rtrt.com/"obj_spider = SpiderMain()obj_spider.craw(root_url)

#coding=utf-8class UrlManager(object):def __init__(self):self.new_urls = set()self.old_urls = set()def add_new_url(self, url):if url is None:returnif url not in self.new_urls and url not in self.old_urls:self.new_urls.add(url)def add_new_urls(self, urls):if urls is None or len(urls) == 0:returnfor url in urls:self.add_new_url(url)def has_new_url(self):return len(self.new_urls) != 0def get_new_url(self):new_url = self.new_urls.pop()self.old_urls.add(new_url)return new_url

#coding=utf-8
import urllib2class HtmlDownloader(object):def download(self,url):if url is None:return Noneuser_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'headers = { 'User-Agent' : user_agent }request = urllib2.Request(url,headers = headers)response = urllib2.urlopen(request)if response.getcode() != 200:return Nonereturn response.read()

#coding=utf-8
from bs4 import BeautifulSoup
import re
import urlparse
import urllib2class HtmlParser(object):def _get_new_urls(self, page_url, soup):new_urls = set()links = soup.find_all('a', href=re.compile(r"News/\d+/\d+\.htm"))for link in links:new_url = link['href']new_full_url = urlparse.urljoin(page_url,new_url)new_urls.add(new_full_url)return new_urlsdef _get_new_data(self, page_url, soup):res_data = {}res_data['url'] = page_urlres_data['pic'] = []nodes = soup.find_all('img')for node in nodes:if node.attrs.has_key('src'):src = node.get('src')if src[0:4]=='http':res_data['pic'].append(src)return res_datadef parse(self, page_url, html_cont):if page_url is None or html_cont is None:returnsoup = BeautifulSoup(html_cont,'html.parser',from_encoding='utf-8')new_urls = self._get_new_urls(page_url, soup)new_data = self._get_new_data(page_url, soup)return new_urls,new_data

#coding=utf-8
import urllib
import os
class HtmlOutputer(object):def __init__(self):self.datas = []def collect_data(self,data):if data is None:returnself.datas.append(data)path = data['url'].replace(r'/','').replace(':','')os.mkdir(path)x=0for pic in data['pic']:urllib.urlretrieve(pic,os.path.join(path,'%s.jpg'%x))x+=1def output_html(self):fout = open('output.txt','w')for data in self.datas:fout.write("%s\n" %data['url'])for pic in data['pic']:fout.write("%s\n"%pic)