下表列出了主要的解析器,以及它们的优缺点:

推荐使用lxml作为解析器,因为效率更高. 在Python2.7.3之前的版本和Python3中3.2.2之前的版本,必须安装lxml或html5lib, 因为那些Python版本的标准库中内置的HTML解析方法不够稳定.

from bs4 import BeautifulSoupsoup = BeautifulSoup(open("index.html"))
print(soup.prettify())
soup.title
# <title>The Dormouse's story</title>soup.title.name
# u'title'soup.title.string
# u'The Dormouse's story'soup.title.parent.name
# u'head'soup.p
# <p class="title"><b>The Dormouse's story</b></p>soup.p['class']
# u'title'soup.a
# <a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>soup.find_all('a')
# [<a class="sister" href="http://example.com/elsie" id="link1">Elsie</a>,
#  <a class="sister" href="http://example.com/lacie" id="link2">Lacie</a>,
#  <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>]soup.find(id="link3")
# <a class="sister" href="http://example.com/tillie" id="link3">Tillie</a>
从文档中找到所有<a>标签的链接:
for link in soup.find_all('a'):print(link.get('href'))# http://example.com/elsie# http://example.com/lacie# http://example.com/tillie
从文档中获取所有文字内容:
print(soup.get_text())
# The Dormouse's story
#

#!/usr/bin/env python
#coding=utf-8
#
#   Copyright 2017 liuxinxing
#from bs4 import BeautifulSoup
import urllib2import datetime
import time
import PyRSS2Gen
import re
import sys
reload(sys)
sys.setdefaultencoding('utf-8')class RssSpider():def __init__(self):self.myrss = PyRSS2Gen.RSS2(title='OSChina',link='http://my.oschina.net',description=str(datetime.date.today()),pubDate=datetime.datetime.now(),lastBuildDate = datetime.datetime.now(),items=[])self.xmlpath=r'./oschina.xml'self.baseurl="http://www.oschina.net/blog"#if os.path.isfile(self.xmlpath):#os.remove(self.xmlpath)def useragent(self,url):i_headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36","Referer": 'http://baidu.com/'}req = urllib2.Request(url, headers=i_headers)html = urllib2.urlopen(req).read()return htmldef enterpage(self,url):pattern = re.compile(r'\d{4}\S\d{2}\S\d{2}\s\d{2}\S\d{2}')rsp=self.useragent(url)# print rspsoup=BeautifulSoup(rsp, "html.parser")# print souptimespan=soup.find('div',{'class':'blog-content'})# print timespantimespan=str(timespan).strip().replace('\n','').decode('utf-8')# match=re.search(r'\d{4}\S\d{2}\S\d{2}\s\d{2}\S\d{2}',timespan)# timestr=str(datetime.date.today())# if match:#     timestr=match.group()#print timestrititle=soup.title.stringprint ititlediv=soup.find('div',{'class':'BlogContent'})# print type(div)doc = div.get_text()# print type(doc)return ititle,docdef getcontent(self):rsp=self.useragent(self.baseurl)# print rspsoup=BeautifulSoup(rsp, "html.parser")# print soupul=soup.find('div',{'id':'topsOfRecommend'})# print ulfor div in ul.findAll('div',{'class':'box-aw'}):# div=li.find('div')# print divif div is not None:alink=div.find('a')if alink is not None:link=alink.get('href')print linkif self.isbloglink(link):title,doc =self.enterpage(link)self.savefile(title,doc)def isbloglink(self,link):express = r".*/blog/.*"mo = re.search(express, link)if mo:return Trueelse:return Falsedef savefile(self,title,doc):doc = doc.decode('utf-8')with open("./data/"+title+".txt",'w') as f:f.write(doc)if __name__=='__main__':rssSpider=RssSpider()rssSpider.getcontent()# rssSpider.enterpage("https://my.oschina.net/diluga/blog/1501203")