# import libraray
import requests
import os
import pandas as pd
import re
from lxml import etree
from bs4 import BeautifulSoup# source url
url2 = "http://app.hfpi.gov.cn/fangjia/ws/Detail2.aspx?Id=5421"
headers = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"}# 每次都更新下参数
def update_data_dict(soup):data_dict['__VIEWSTATE'] = soup.find(id='__VIEWSTATE')['value']data_dict['__EVENTTARGET'] = 'AspNetPager1'data_dict['__EVENTARGUMENT'] += 1#保存下来爬取的当页的数据
def save_data(soup):x = soup('table',width='97%')xx = x[2]y = xx('tr')total = []for item in y[1:-1]:content1 = item('td')empty = ''for i in content1:if i.find('div') == None:continueelse:infopiece = i.find('div').text.strip() + ' 'empty += infopiecetotal.append(empty)with open('result0.txt','a') as f:for senc in total:f.write(senc + '\n')  # 接着去下一页继续爬
def get_next_page_data():html2 = requests.post(url, data=data_dict).textsoup2 = BeautifulSoup(html2, 'lxml')update_data_dict(soup2)save_data(soup2)## 现在开始爬取
html_jsy = requests.get(url2).text
soup_jsy = BeautifulSoup(html_jsy,'lxml')
save_data(soup_jsy)
update_data_dict(soup_jsy)
# 这里总页数我就设置20，如果后面没有了，也不会继续访问
for i in range(20):get_next_page_data_2()

import pandas as pddf = pd.read_csv('result-jsy.txt', sep=' ',header=None)#change column name
df.columns = ['BuildingNo','RmNo','Type','Area','ShareArea','RealArea','Price','totalPrice','1','2','3','4']# 这次开盘只有16还有18, 先把16 18给选出来，然后合并
df_16 = df[df.BuildingNo =='G16']
df_18 = df[df.BuildingNo == 'G18']
p_list = [df_16,df_18]
df_temp = pd.concat(p_list)

df_temp[df_temp.Price <= 14500.0]

df_select1[df_select1.RmNo > 500]