零基础自学用Python 3开发网络爬虫-这篇博客写的不错，通俗易懂，文笔也很好
知乎上关于如何使用Python来编爬虫的解答1
知乎上关于如何使用Python来编爬虫的解答2正则表达式30分钟入门

 1 def get_top10article(self):
 2         top10_url = 'http://bbs.nju.edu.cn/bbstop10'
 3         bbs_url = 'http://bbs.nju.edu.cn/'
 4
 5         req = urllib2.Request(top10_url, headers = self.headers)
 6         response = urllib2.urlopen(req)
 7         top10_page = response.read()
 8         #print top10_page
 9
10         #unicode_top10_page = top10_page.decode('utf-8')
11         pattern_str = '<tr.*?bgcolor=.*?><td>(.*?)<td><a.*?href=(.*?)>(.*?)</a><td><a.*?href="(.*?)">(.*?)\n</a><td><a.*?href=(.*?)>(.*?)</a><td>(.*?)\n'
12         pattern = re.compile(pattern_str)
13         #pattern = re.compile(r'<tr.*?bgcolor=.*?><td>(.*?)<td><a.*?href=(.*?)>(.*?)</a><td><a.*?href="(.*?)">(.*?)</a><td><a.*?href=(.*?)>(.*?)</a>')
14         top10_retrive_infos = pattern.findall(top10_page)
15         for info in top10_retrive_infos:
16             article = Article(info[0], bbs_url + info[1], info[2], bbs_url + info[3], info[4], bbs_url + info[5], info[6])
17             self.top10.append(article)
18             #print info

 1 def get_article(self, url):
 2         # url + '&start=-1' 显示本主题全部帖子
 3         all_article_url = url + '&start=-1'
 4         req = urllib2.Request(all_article_url, headers = self.headers)
 5         response = urllib2.urlopen(req)
 6         article_content = response.read()
 7
 8         # use regular experssion to find out all the reply article content
 9         pattern_str = '<textarea.*?id=.*?class=hide>(.*?)--\n.*?</textarea>'
10         pattern = re.compile(pattern_str, re.S)
11         all_replies_content = pattern.findall(article_content)
12
13         f = open('all_replies_content.txt', 'w')
14
15         result_content = []
16         for reply in all_replies_content:
17             f.write(reply)
18             result_content.append(reply)
19             #print reply
20         return result_content

 1 # -*- coding: cp936 -*-
 2 import urllib2
 3 import urllib
 4 import re
 5 # 自定义帖子类，包括十大排名、板块链接、板块名、帖子链接、帖子标题、作者链接和作者 7个字段
 6 class Article:
 7     def __init__(self, rank, board_link, board, article_link, title, author_link, author):
 8         self.rank = rank
 9         self.board_link = board_link
10         self.board = board
11         self.article_link = article_link
12         self.title = title
13         self.author_link = author_link
14         self.author = author
15
16 class Lily_Top10_Spider:
17     def __init__(self):
18         self.top10 = []
19         self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
20         self.headers = {'User-Agent' : self.user_agent}
21
22
23     # 获取十大信息，添加到列表中并返回列表
24     def get_top10article(self):
25         top10_url = 'http://bbs.nju.edu.cn/bbstop10'
26         bbs_url = 'http://bbs.nju.edu.cn/'
27
28         req = urllib2.Request(top10_url, headers = self.headers)
29         response = urllib2.urlopen(req)
30         top10_page = response.read()
31         #print top10_page
32
33         #unicode_top10_page = top10_page.decode('utf-8')
34         pattern_str = '<tr.*?bgcolor=.*?><td>(.*?)<td><a.*?href=(.*?)>(.*?)</a><td><a.*?href="(.*?)">(.*?)\n</a><td><a.*?href=(.*?)>(.*?)</a><td>(.*?)\n'
35         pattern = re.compile(pattern_str)
36         #pattern = re.compile(r'<tr.*?bgcolor=.*?><td>(.*?)<td><a.*?href=(.*?)>(.*?)</a><td><a.*?href="(.*?)">(.*?)</a><td><a.*?href=(.*?)>(.*?)</a>')
37         top10_retrive_infos = pattern.findall(top10_page)
38         for info in top10_retrive_infos:
39             article = Article(info[0], bbs_url + info[1], info[2], bbs_url + info[3], info[4], bbs_url + info[5], info[6])
40             self.top10.append(article)
41             #print info
42
43
44         for a in self.top10:
45             print a.title, ' ', a.author, ' ', a.board, ' ', a.article_link
46
47     def get_article(self, url):
48         # url + '&start=-1' 显示本主题全部帖子
49         all_article_url = url + '&start=-1'
50         req = urllib2.Request(all_article_url, headers = self.headers)
51         response = urllib2.urlopen(req)
52         article_content = response.read()
53         #print article_content
54
55
56         # use regular experssion to find out all the reply article content
57         pattern_str = '<textarea.*?id=.*?class=hide>(.*?)--\n.*?</textarea>'
58         pattern = re.compile(pattern_str, re.S)
59         all_replies_content = pattern.findall(article_content)
60
61         f = open('all_replies_content.txt', 'w')
62         #print all_replies
63
64         result_content = []
65         for reply in all_replies_content:
66             f.write(reply)
67             result_content.append(reply)
68             #print reply
69         return result_content
70         #return self.top10
71
72
73 ls = Lily_Top10_Spider()
74 ls.get_top10article()
75
76 print '#1 article content:'
77 article_content = ls.get_article(ls.top10[9].article_link)
78 for s in article_content:
79     print s
80 print 'print end.'
81             

参考文献：HOWTO Fetch Internet Resources Using urllib2Python爬虫入门教程