'''Created on 2017-10-9

@author: wbhuangzhiqiang'''

importsysimportreimportcsvimporturllib.requestfrom bs4 importBeautifulSoup#成功打开页面时返回页面对象，否则打印错误信息，退出程序

defget_bsobj(url):

page=urllib.request.urlopen(url)if page.getcode() == 200:

html=page.read()

bsobj= BeautifulSoup(html, "html5lib")returnbsobjelse:print ("页面错误")

sys.exit()defget_house_info_list(url):

house_info_list=[]

bsobj=get_bsobj(url)if notbsobj:returnNone#获取页数

globalhouse_info_page

house_page= bsobj.find("a", {"gahref":"results_totalpage"})

house_info_page=int(house_page.get_text())#print(house_info_page)

house_list = bsobj.find_all("div", {"class":"info"})for house inhouse_list:#title = house.find("div", {"class": "prop-title"}).get_text().split("|")

#获取信息数据(例：加怡名城 | 2室1厅 | 62.48平米 | 西 | 精装)，通过“|”符号分割字符串

info = house.find("span", {"class": "info-col row1-text"}).get_text().split("|")#print("==========1====")

info2 = house.find("span", {"class": "info-col row2-text"}).get_text().split("|")#print("==========2====")

#print(info2)

#print("==========2====")

#print(info2)

#print("==========2====")

minor = house.find("span", {"class": "info-col price-item minor"}).get_text().strip()#小区(例：加怡名城)，strip()去除字符串两边的空格，encode，将字符串编码成 utf-8 格式

block = info2[1].strip()+info2[2].strip()+info2[0].strip()if len(info2)>3:

naidai= info2[3].strip()else:

naidai='未知'

#房型

house_type =info[0].strip()#面积

size =info[1].strip()

price_sz= house.find("span", {"class": "total-price strong-num"}).get_text()

price_dw= house.find("span", {"class": "unit"}).get_text()

price=price_sz+price_dw#print(price)

house_info_list.append({'房型':house_type,'面积':size,'价格':price,'房屋位置':block,'年代':naidai,'单价':minor})#print('**********************')

#print(house_info_list)

#print(len(house_info_list))

returnhouse_info_list#读取前100个页面的房屋信息，将信息保存到 house.csv 文件中

defhouse_mess(url):

house_info_list=[]

get_house_info_list(url)if house_info_page>20:for i in range(0,21):

new_url= url +'/d'+str(i)

house_info_list.extend(get_house_info_list(new_url))#print(new_url)

#print(house_info_list)

#print("****************house_info_list*********************")

#print(house_info_list)

ifhouse_info_list:#将数据保存到 house.csv 文件中

with open("./house.csv", "w",newline='') as f:#writer 对象

writer =csv.writer(f)

fieldnames=house_info_list[0].keys()

writer.writerow(fieldnames)for house_info inhouse_info_list:#print('&&&&&&&&&&&&&&&&&&&&&&&')

#print(house_info)

writer.writerow(house_info.values())#house_mess('http://sh.lianjia.com/ershoufang/minhang')