1 #-*- coding:utf-8 -*-

2 importtime3 importjson4 importlxml.html5 from lxml importetree6 from HTMLParser import HTMLParser #使用beautifulsoup也可以

7

8 #http://www.crummy.com/software/BeautifulSoup/

9 #http://blog.csdn.net/my2010sam/article/details/14526223

10

11 try:12 from urllib.request importurlopen,Request13 except:14 from urllib2 importurlopen, Request15

16

17 area={"11":"北京","12":"天津","13":"河北","14":"山西","15":"内蒙古","21":"辽宁","22":"吉林","23":"黑龙江","31":"上海",18 "32":"江苏","33":"浙江","34":"安徽","35":"福建","36":"江西","37":"山东","41":"河南","42":"湖北","43":"湖南",19 "44":"广东","45":"广西","46":"海南","50":"重庆","51":"四川","52":"贵州","53":"云南","54":"西藏","61":"陕西",20 "62":"甘肃","63":"青海","64":"宁夏","65":"新疆","71":"台湾","81":"香港","82":"澳门","91":"国外"}21 ages =[0]*11

22 provinceCnt=[0]*91

23 RECORD_BY_EACH_PAGE = [10,15,30,50,5000]24 currentYear=time.localtime()[0]#get year

25 URL_BY_PAGESIZE='http://bzflh.szjs.gov.cn/TylhW/lhmcAction.do?pageSize=%s&method=queryYgbLhmcInfo&waittype=2'

26

27 #http://XXX.cn?pageSize=XXX&page=XXX,waittype=2 公租房，waittype=1 安居房

28 URL_BY_PAGE_PAGESIZE ='http://bzflh.szjs.gov.cn/TylhW/lhmcAction.do?pageSize=%s&method=queryYgbLhmcInfo&waittype=%s&page=%s'

29

30 #Social_Housing_Items=[URL_BY_PAGE_PAGESIZE_GongZuFang,URL_BY_PAGE_PAGESIZE_AnJuFang]

31

32 defgetHomePage(url,pagesize):33 try:34 request =Request(url)35 lines=urlopen(request,timeout=10).read()36 if len(lines)<20:37 return None #no data

38 exceptException as e:39 printe40 else:41 if pagesize!=10 and pagesize!=15 and pagesize!=30 and pagesize!=50 and pagesize !=5000:42 pagesize = 15 #default as 15 record each page

43 lines=lines.decode('utf-8')44 splitLines=lines.split('\r\n')45 for line insplitLines:46 #if "pageSize" in line:

47 #print line[:50]

48 if "pagebanner" inline:49 totalPage= line[:50].split('>')[1].split(' ')[0]50 totalPage=totalPage.split(',')51 if len(totalPage)>1:52 pages=(int(totalPage[0])*1000+int(totalPage[1]))/pagesize53 returnpages54

55 defgetRawData(url):56 try:57 request =Request(url)58 lines=urlopen(request,timeout=10).read()59 if len(lines)<20:60 return None #no data

61 exceptException as e:62 printe63 else:64 return lines.decode('utf-8')65

66 defgetIdentityInfo(code):67 """

68 :param code: identity code showing province and date69 :return: province,date70 """

71 provinceCode=code[:2]72 cityCode = code[2:6]73 date=code[6:10]74 returnprovinceCode,date75

76 classDataparser(HTMLParser):77 def __init__(self):78 HTMLParser.__init__(self)79 self.tr=False80 self.td =081 self.data =False82 defhandle_starttag(self,tag,attrs):83 """

84 参数tag是标签名,比如td,tr',attrs为标签所有属性(name,value)列表,这里是[('class','para')]85 :param tag:86 :param attrs:87 :return:88 """

89 if tag=='tr':90 self.tr=True91 if tag =='td'and self.tr==True:92 self.data =True93 for name,value inattrs:94 print "name and value are",name,value95 defhandle_endtag(self,tag):96 if tag=='td':97 self.data =False98 #print "a end tag:",tag,self.getpos()

99

100 defhandle_data(self,data):101 if self.data and len(data)==18 and '\r\n' not indata:102 #print data #ID card NO

103 provinceCode,date=getIdentityInfo(data)104 ageRange=currentYear -int(date)105 if ageRange>=100:106 print 'test',ageRange107 #ages[ageRange/10] +=1

108 #temp=area[provinceCode].decode('utf-8')

109 PC=int(provinceCode)110 provinceCnt[PC]+=1

111

112 if __name__ =='__main__':113 #计算总共页数，每页可以自己限定

114 for type in range(2):115 pages=getHomePage(URL_BY_PAGE_PAGESIZE%(RECORD_BY_EACH_PAGE[0],type+1,1),RECORD_BY_EACH_PAGE[4])116 parse=Dataparser()117 while pages>=1:118 #for page in range(pages):

119 lines=getRawData(URL_BY_PAGE_PAGESIZE%(RECORD_BY_EACH_PAGE[4],type+1,pages))120 parse.feed(lines)121 #parse.close()

122 pages-=1

123 parse.close()124 if type==0:125 print "深圳安居房申请人全国分布情况统计:"

126 for i inprovinceCnt:127 if i>0: #只打印有数据的省份

128 pIndex=str(provinceCnt.index(i))129 printarea[pIndex],i130 provinceCnt =[0]*91

131 elif type==1:132 print "深圳公租房申请人全国分布情况统计:"

133 for i inprovinceCnt:134 if i>0: #只打印有数据的省份

135 pIndex=str(provinceCnt.index(i))136 printarea[pIndex],i137 provinceCnt =[0]*91