1 # -*- coding: cp936 -*-

2 import urllib2

3 import re

4 from pyquery import PyQuery as pq

5 from lxml import etree

6

7 #mailpattern = re.compile('[^\._:>\\-][\w\.-]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+')

8 mailpattern = re.compile('[A-Za-z0-9_]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+')

9

10 htmlcount = 0 #to count the urls

11 maxcount = 3000 # the max count

12 allUrls = set()

13 allMails = set()

14 UrlsQlist = []

15 UrlsQdict = {}

16 url = "http://www.163.com"

17 fmails = open("E:/py/crawler/mailresult.txt","a")

18 furls = open("E:/py/crawler/urlresult.txt","a")

19

20

21

22

23 def geturls(data):#the function to get the urls in the html

24 urls = set()

25 if data:

26 d = pq(data)

27 label_a = d.find('a')#用pyquery库去找到 a 标签.

28 if label_a:

29 label_a_href = d('a').map(lambda i,e:pq(e)('a').attr('href'))

30 for u in label_a_href:

31 if u[0:10]!="javascript" :

32 if u[0:4] == "http":

33 urls.add(u)

34 else:

35 urls.add(url + u)

36 #for u in urls:

37 #print u

38 return urls

39 else:

40 return None

41

42 def gethtml(url):

43 try:

44 fp = urllib2.urlopen(url)

45 except:

46 print "urllib2.urlopen error"

47 return None

48 else:

49 mybytes =fp.read()

50 fp.close()

51 return mybytes

52

53 def savemails(data): # the function to save the emails

54 if data:

55 mailResult = mailpattern.findall(data)

56 mailResultset = set(mailResult)

57 if mailResultset:

58 allMails.update(mailResultset)

59

60 def savehtml(pagecontent,count):

61 if pagecontent != None:

62 f = open("E:/py/crawler/html/"+str(count)+".html","w")

63 f.write(pagecontent)

64 f.close()

65 else:

66 f = open("E:/py/crawler/html/"+str(count)+"error"+".html","w")

67 f.write("this page empty")

68 f.close()

69

70 def BFS(firstUrl):

71 global htmlcount

72 global maxcount

73 allUrls.add(firstUrl)

74 UrlsQlist = list(allUrls)

75 while htmlcount < maxcount : #数量小于最大值

76 tempUrl = UrlsQlist.pop(0)# the queue

77 myWebStr = gethtml(tempUrl)

78 savehtml(myWebStr,htmlcount)

79 savemails(myWebStr)

80 firstUrls_set = geturls(myWebStr)#初始页面的处理

81 if firstUrls_set != None:

82 allUrls.update(firstUrls_set) #记录全部 url

83 for u in firstUrls_set:

84 if u not in UrlsQlist:

85 UrlsQlist.append(u)

86 htmlcount = htmlcount + 1

87

88

89 BFS(url)

90 for u in allMails:

91 try:

92 fmails.write(u)

93 fmails.write('\n')

94 except:

95 continue

96 for u in allUrls:

97 try:

98 furls.write(u)

99 furls.write('\n')

100 except:

101 continue

102 fmails.close()

103 furls.close()

2013.5.13 update

本来想在加个多线程。。。。结果看了 好多资料 无处下手,再研究研究 ，日后再改

加了点 url规范化。代码整理如下：

1 import urllib2

2 import re

3 from pyquery import PyQuery as pq

4 from lxml import etree

5 import urlparse

6 import time

7

8 allUrls = set()

9 allMails = set()

10 urlsDownlist = []

11

12 class mailCrawler:

13 def __init__(self,mailExpression,start_url,maxcount):

14 ''' mailExpressoin 邮箱的正则表达式;

15 start_url开始邮箱；

16 maxcount最大数量'''

17 self.mailpattern = re.compile(mailExpression)

18 self.maxcount = maxcount

19 self.htmlcount = 0

20 self.UrlsQlist = []#url queue 实现广度优先

21 self.url = start_url

22

23

24 def url_normal(self,url):

25 '''url 规范化 '''

26 scheme,netloc,path,query = urlparse.urlsplit(url)[:4]

27 netloc = netloc.lower()

28

29 url.encode("utf-8")

30

31 if path:

32 path = re.sub('/{2,}','/',path)#去除url中的重复/

33 path = re.sub(r'\.$','',path)#去除url中结尾多余的点

34 path = re.sub('/$','',path)#去除url中结尾多余的/

35 path = re.sub('\s','',path)#取出url中的空格

36 if query:

37 return '%s://%s%s?%s' % (scheme,netloc,path or '/',query)

38 else:

39 return '%s://%s%s' % (scheme,netloc,path)

40

41 def geturls(self,data):

42 '''解析html中的url'''

43 urls = set()

44 if data:

45 d = pq(data)

46 label_a = d.find('a')#用pyquery库去找到 a 标签.

47 if label_a:

48 label_a_href = d('a').map(lambda i,e:pq(e)('a').attr('href'))

49 for u in label_a_href:

50 if u[0:10]!="javascript" and u[0:6]!="mailto" :

51 if u[0:4] == "http":

52 normal_url = self.url_normal(u)

53 urls.add(normal_url)

54 else:

55 normal_url = self.url_normal(self.url + u)

56 urls.add(normal_url)

57 return urls

58 else:

59 return None

60

61 def gethtml(self,url):

62 '''下载html 5s超时'''

63 try:

64 fp = urllib2.urlopen(url,None,5)

65 except:

66 print "urllib2.urlopen error or timeout"

67 return None

68 else:

69 mybytes =fp.read()

70 fp.close()

71 return mybytes

72

73 def savemails(self,data):

74 '''将抓取到的url存放到 allmails中 ，set去重复'''

75 global allMails

76 if data:

77 mailResult = self.mailpattern.findall(data)

78 mailResultset = set(mailResult)

79 if mailResultset:

80 allMails.update(mailResultset)

81

82 def savehtml(self,pagecontent,htmlcount,url):

83 '''保存html文件 '''

84 if pagecontent != None:

85 f = open("E:/py/crawler/html/"+str(htmlcount)+".html","w")

86 f.write(pagecontent)

87 f.close()

88 else:

89 f = open("E:/py/crawler/html/"+str(htmlcount)+"error"+".html","w")

90 try:

91 f.write(url)

92 except:

93 f.write("encode error")

94 f.close()

95

96 def BFS(self):

97 '''用队列实现广度优先，爬取url '''

98 global allUrls

99 global urlsDownlist

100 allUrls.add(self.url)

101 self.UrlsQlist = list(allUrls)

102 while self.htmlcount < self.maxcount : #数量小于最大值

103 tempUrl = self.UrlsQlist.pop(0)# the queue

104 print tempUrl

105 urlsDownlist.append(tempUrl)

106 myWebStr = self.gethtml(tempUrl)

107 self.savehtml(myWebStr,self.htmlcount,tempUrl)

108 self.savemails(myWebStr)

109 firstUrls_set = self.geturls(myWebStr)#初始页面的处理

110 if firstUrls_set != None:

111 for u in firstUrls_set:

112 if u not in allUrls:

113 allUrls.add(u)

114 self.UrlsQlist.append(u)

115 self.htmlcount = self.htmlcount + 1

116

117

118 def main():

119 reg = r'[A-Za-z0-9_]+@(?:[A-Za-z0-9]+\.)+[A-Za-z]+'

120 url = "http://www.baidu.com"

121 count = 100

122 fmails = open("E:/py/crawler/mailresult.txt","a")

123 furls = open("E:/py/crawler/urlresult.txt","a")

124 fdownUrls = open("E:/py/crawler/urlDownresult.txt","a")

125 newcrawler = mailCrawler(reg,url,count)

126 newcrawler.BFS()

127 for u in allMails:

128 try:

129 fmails.write(u)

130 fmails.write('\n')

131 except:

132 continue

133 for u in allUrls:

134 try:

135 furls.write(u)

136 furls.write('\n')

137 except:

138 continue

139 for u in urlsDownlist:

140 try:

141 fdownUrls.write(u)

142 fdownUrls.write('\n')

143 except:

144 continue

145 fmails.close()

146 furls.close()

147 fdownUrls.close()

148

149 if __name__ == '__main__':

150 main()

分享到：

2013-05-18 03:59

浏览 4513

评论