'''
Created on 2017-10-9@author: wbhuangzhiqiang
'''
import csv
import urllib.request
from bs4 import BeautifulSoup
url='https://www.lianjia.com'
#获取html
# 获取 html 页面
html = urllib.request.urlopen(url).read()# 获取 BeautifulSoup 对象，用 html5lib 解析（也可用 lxml 或其它方式解析，html5lib 容错性较好，所以此处选用 html5lib ）
bsobj = BeautifulSoup(html, "html5lib")
# 得到 class="cityList clear" 的 div 下所有 a 标签
city_tags = bsobj.find("div",{"class":"cityList clear"}).findChildren("a")
print(city_tags)# 将每一条数据抽离，保存在 citys.csv 文件中
with open("./citys.csv", "w") as f:writ = csv.writer(f)for city_tag in city_tags:# 获取 <a> 标签的 href 链接city_url = city_tag.get("href")# 获取 <a> 标签的文字，如：天津city_name = city_tag.get_text()writ.writerow((city_name, city_url))print (city_name, city_url)

'''
Created on 2017-10-9@author: wbhuangzhiqiang
'''
import sys
import re
import csv
import urllib.request
from bs4 import BeautifulSoup# 成功打开页面时返回页面对象，否则打印错误信息，退出程序
def  get_bsobj(url):page = urllib.request.urlopen(url)if page.getcode() == 200:html = page.read()bsobj = BeautifulSoup(html, "html5lib")return bsobjelse:print ("页面错误")sys.exit()def get_house_info_list(url):house_info_list = []bsobj = get_bsobj(url)if not bsobj:return None#获取页数global house_info_pagehouse_page = bsobj.find("a", {"gahref":"results_totalpage"})house_info_page=int(house_page.get_text())#print(house_info_page)house_list = bsobj.find_all("div", {"class":"info"})for  house in house_list:#title = house.find("div", {"class": "prop-title"}).get_text().split("|")# 获取信息数据（例：加怡名城 | 2室1厅 | 62.48平米 | 西 | 精装），通过“|”符号分割字符串info = house.find("span", {"class": "info-col row1-text"}).get_text().split("|")#print("==========1====")info2 = house.find("span", {"class": "info-col row2-text"}).get_text().split("|")#print("==========2====") #print(info2)#print("==========2====") #print(info2)#print("==========2====")    minor = house.find("span", {"class": "info-col price-item minor"}).get_text().strip()# 小区（例：加怡名城），strip()去除字符串两边的空格，encode，将字符串编码成 utf-8 格式block = info2[1].strip()+info2[2].strip()+info2[0].strip()if len(info2)>3:naidai = info2[3].strip()else:naidai='未知'#房型house_type =info[0].strip()#面积size =info[1].strip()price_sz = house.find("span", {"class": "total-price strong-num"}).get_text()price_dw = house.find("span", {"class": "unit"}).get_text()price =price_sz+price_dw#print(price)    house_info_list.append({'房型':house_type,'面积':size,'价格':price,'房屋位置':block,'年代':naidai,'单价':minor})#print('**********************')#print(house_info_list)#print(len(house_info_list))return  house_info_list
# 读取前100个页面的房屋信息，将信息保存到 house.csv 文件中
def  house_mess(url):house_info_list =[]get_house_info_list(url)if house_info_page>20:for  i in range(0,21):new_url = url +'/d'+str(i)house_info_list.extend(get_house_info_list(new_url))#print(new_url)#print(house_info_list)#print("****************house_info_list*********************")#print(house_info_list)if house_info_list:# 将数据保存到 house.csv 文件中with open("./house.csv", "w",newline='') as f:# writer 对象writer = csv.writer(f)fieldnames=house_info_list[0].keys()writer.writerow(fieldnames)for house_info in house_info_list:#print('&&&&&&&&&&&&&&&&&&&&&&&')#print(house_info)
writer.writerow(house_info.values())
#house_mess('http://sh.lianjia.com/ershoufang/minhang')

'''
Created on 2017-10-9@author: wbhuangzhiqiang
'''
#coding=gbk
import csv
import sys
import urllib.request
from bs4 import BeautifulSoup
from house_info import house_mess
def  get_city_dict():city_dict = {}with open('./citys.csv', 'r') as  f:reader =csv.reader(f)for  city in reader:if len(city)>0:city_dict[city[0]] = city[1]return city_dict
city_dict = get_city_dict()
#print(city_dict)
# 打印所有的城市名def get_district_dict(url):district_dict = {}html = urllib.request.urlopen(url).read()bsobj = BeautifulSoup(html, "html5lib")roles = bsobj.find("div", {"class":"level1"}).findChildren("a")for role in roles:# 对应区域的 urldistrict_url = role.get("href")# 对应区域的名称district_name = role.get_text()# 保存在字典中district_dict[district_name] = district_urlreturn district_dictdef   run():city_dict = get_city_dict()for city in city_dict.keys():print(city,end=' ')print() key_city= input("请输入城市  ")# 根据用户输入的城市名，得到城市 urlcity_url = city_dict.get(key_city)# 根据用户输入的城市名，得到城市 urlif city_url:print (key_city, city_url)else:print( "输入错误")# 退出
        sys.exit()ershoufang_city_url = city_url + "/ershoufang"print(ershoufang_city_url)district_dict = get_district_dict(ershoufang_city_url)# 打印区域名for district in district_dict.keys():print (district,end=' ')print()input_district = input("请输入地区：")district_url = district_dict.get(input_district)# 输入错误，退出程序if not district_url:print ("输入错误")sys.exit()# 如果都输入正确house_info_url = city_url + district_urlhouse_mess(house_info_url)if __name__ == "__main__":run()