3、网页解析器 beautifulsoup语法基础

#根据HTML网页字符串创建BeautifulSoup对象
soup = BeautifulSoup(html_doc,                   #HTML文档字符串'html.parser',            #HTML解析器from_encoding='utf8'       #HTML文档的编码)#方法：find_all(name,attrs,string) 查找所有标签为a的节点
soup.find_all('a')
#查找所有标签为a，链接符合/view/123.htm形式的节点
soup.find_all('a',href='/view/123.htm')
#查找所有标签为div class为abc 文字为python的节点
soup.find_all('div',class_='abc',string='Python')#得到节点： <a href='1.html'>Python</a>
#获取查找到的节点的标签名称
node.name
#获取查找到的a节点的href属性
node['href']
#获取查找到的a节点的链接文字
node.get_text()

4、网页解析器 beautifulsoup语法练习

#-*- coding:utf-8 -*-
from bs4 import BeautifulSoup
import re
html_doc="""百度新闻页面源码，篇幅原因源码不写进来了"""soup = BeautifulSoup(html_doc, 'html.parser',from_encoding='utf-8')print '获取所有的链接'
links = soup.find_all('a')
for link in links:#print link.name,link['href'],link.get_text()print link.name, link.get('href'), link.get_text()print '获取http://news.baidu.com的链接'
link_node = soup.find('a',href='http://qijunjie.baijia.baidu.com/article/799732')
print link_node.name, link_node.get('href'), link_node.get_text()print '正则匹配'
link_node = soup.find('a',href=re.compile(r"qijunjie"))
print link_node.name, link_node.get('href'), link_node.get_text()print '获取a打开新页面的链接'
links= soup.find_all('a',target='_blank')
for link in links:print link.name, link.get_text()

5、网络爬虫示例

   5.1调度器  spider_main.py

#-*- coding:utf-8 -*-
from baike_spider import html_downloader,html_outputer,html_parser,url_managerclass SpiderMain(object):def __init__(self):self.urls = url_manager.UrlManage()self.downloader = html_downloader.HtmlDownloader()self.parser = html_parser.HtmlParser()self.outputer = html_outputer.HtmlOutputer()def craw(self,root_url):self.urls.add_new_url(root_url)count = 1while self.urls.has_new_url():try:new_url = self.urls.get_new_url()print 'craw %d : %s'%(count,new_url)html_cont = self.downloader.download(new_url)new_urls,new_data = self.parser.parse(new_url,html_cont)self.urls.add_new_urls(new_urls)self.outputer.collect_data(new_data)if count == 1000:breakcount = count + 1except:print 'craw failed'self.outputer.output_html()if __name__=="__main__":root_url = "http://baike.baidu.com/item/乌金木?sefr=cr"obj_spider = SpiderMain()obj_spider.craw(root_url)

   5.2 url管理器  url_manager.py

#-*- coding:utf-8 -*-
class UrlManage(object):def __init__(self):self.new_urls = set()self.old_urls = set()def add_new_url(self, url):if url is None:returnif url not in self.new_urls and url not in self.old_urls:self.new_urls.add(url)def has_new_url(self):return len(self.new_urls) != 0def get_new_url(self):new_url = self.new_urls.pop()self.old_urls.add(new_url)return new_urldef add_new_urls(self, urls):if urls is None or len(urls) == 0:returnfor url in urls:self.add_new_url(url)

   5.3 下载器  html_downloader.py

#-*- coding:utf-8 -*-
import urllib2class HtmlDownloader(object):def download(self, url):if url is None:return Noneresponse = urllib2.urlopen(url)if response.getcode() != 200:return Nonereturn response.read()

   5.4 解析器  html_parser.py

#-*- coding:utf-8 -*-
import re
import urlparsefrom bs4 import BeautifulSoupclass HtmlParser(object):def parse(self, page_url, html_cont):if page_url is None or html_cont is None:returnsoup = BeautifulSoup(html_cont,'html.parser',from_encoding='utf-8')new_urls = self._get_new_urls(page_url,soup)new_data = self._get_new_data(page_url,soup)return new_urls,new_datadef _get_new_urls(self, page_url, soup):new_urls = set()links = soup.find_all('a',href=re.compile(r"/view/\d+\.htm"))for link in links:new_url = link['href']new_full_url = urlparse.urljoin(page_url,new_url)new_urls.add(new_full_url)return new_urlsdef _get_new_data(self, page_url, soup):res_data = {}res_data['url'] = page_url#<dd class="lemmaWgt-lemmaTitle-title"><h1>乌金木</h1>title_node = soup.find('dd',class_="lemmaWgt-lemmaTitle-title").find("h1")res_data['title'] = title_node.get_text()#<div class="lemma-summary" label-module="lemmaSummary">summary_node = soup.find('div',class_="lemma-summary")res_data['summary'] = summary_node.get_text()return res_data

   5.5 输出器  html_outputer.py

#-*- coding:utf-8 -*-
class HtmlOutputer(object):def __init__(self):self.datas = []def collect_data(self, data):if data is None:returnself.datas.append(data)def output_html(self):fout = open('out.html','w')fout.write("<html>")fout.write("<head>")fout.write("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />")fout.write("</head>")fout.write("<body>")fout.write("<table>")for data in self.datas:fout.write("<tr>")fout.write("<td>%s</td>" % data['url'])fout.write("<td>%s</td>" % data['title'].encode('utf-8'))fout.write("<td>%s</td>" % data['summary'].encode('utf-8'))fout.write("</tr>")fout.write("</table>")fout.write("</body>")fout.write("</html>")