import html.parser as parser
import urllib.request

 1 class MyHtmlParser(parser.HTMLParser):
 2     def __init__(self, lst = None):
 3         super().__init__()   #这里容易漏掉导致出错
 4         if not lst:
 5             self.urls = []
 6         else:
 7             self.urls = lst
 8
 9     def handle_starttag(self, tag, attrs):
10         for attr, value in attrs:
11             if "http" in value and ".js" not in value \
12             and value not in self.urls:
13                 self.urls.append(value)
14
15     def handle_startendtag(self, tag, attrs):
16         for attr, value in attrs:
17             if "http" in value and ".js" not in value \
18             and value not in self.urls:
19                 self.urls.append(value)

1 def ParseUrlsInText(text, lst):
2     pars = MyHtmlParser(lst)
3     try:
4         pars.feed(text)
5     #添加异常处理，可能会遇到各种异常
6     except parser.HTMLParseError as ex:
7         print("parse failed.",ex)  

 1 def VisitUrlsInPage(pageUrl):
 2     url_list = [pageUrl]
 3     for url in url_list:
 4         try:
 5             fh = urllib.request.urlopen(url)
 6             data = fh.read()
 7             ParseUrlsInText(str(data), url_list)
 8             #这里是为了快速结束，去掉就变成小爬虫了
 9             if len(url_list) >= 200:
10                 break
11         except urllib.request.URLError:
12             print("Failed.")
13             continue
14     print("length: ", len(url_list))
15     for url in url_list:
16         print(url)
17
18
19 if __name__ == '__main__':
20     VisitUrlsInPage("http://www.cnblogs.com/jason-yang/")