#coding:utf-8import requests
from lxml import html
import csv
import re
from fontTools.ttLib import TTFont
import base64
import MySQLdb
import time
from download import get_user_agentdef download(url,headers=None,cookies=None,proxies=None,num_retries=3):  #支持user-agent和proxy#proxies = {"http": "http://10.10.1.10:3128", "https": "http://10.10.1.10:1080",}response=requests.get(url,headers=headers,cookies=cookies,proxies=proxies)if response.status_code and 500<=response.status_code<600:  # 出现服务器端错误时重试三次if num_retries > 0:response = download(url,user_agent,proxies,num_retries-1)return responsedef convertNumber(html_page):base_fonts = ['uni9FA4', 'uni9F92', 'uni9A4B', 'uni9EA3', 'uni993C', 'uni958F', 'uni9FA5', 'uni9476', 'uni9F64','uni9E3A']base_fonts2 = ['&#x' + x[3:].lower() + ';' for x in base_fonts]  # 构造成 鸺 的形式pattern = '(' + '|'.join(base_fonts2) + ')'font_base64 = re.findall("base64,(AA.*AAAA)", response.text)[0]  # 找到base64编码的字体格式文件font = base64.b64decode(font_base64)with open('58font2.ttf', 'wb') as tf:tf.write(font)onlinefont = TTFont('58font2.ttf')convert_dict = onlinefont['cmap'].tables[0].ttFont.tables['cmap'].tables[0].cmap  # convert_dict数据如下：{40611: 'glyph00004', 40804: 'glyph00009', 40869: 'glyph00010', 39499: 'glyph00003'new_page = re.sub(pattern, lambda x: getNumber(x.group(),convert_dict), html_page)return new_pagedef getNumber(g,convert_dict):key = int(g[3:7], 16)  # '&#x9ea3',截取后四位十六进制数字，转换为十进制数，即为上面字典convert_dict中的键number = int(convert_dict[key][-2:]) - 1  # glyph00009代表数字8， glyph00008代表数字7，依次类推return str(number)def getLocation(title):desc = title.split()if u'寓' in desc[0]or u'社区' in desc[0]:location = desc[0].strip(u'【整租】【合租】')+desc[2]else:location=desc[1]return locationdef getAveragePrice(price):price_range = price.split('-')if len(price_range)==2:average_price = (int(price_range[0])+int(price_range[1]))/2else:average_price=pricereturn average_priceif __name__ == '__main__':conn = MySQLdb.connect(host='localhost',user='root',passwd='',db='renthouse',charset='utf8',port=3306) #不要写成utf-8cursor = conn.cursor()cursor.execute("""CREATE TABLE IF NOT EXISTS 58house(id INT PRIMARY KEY AUTO_INCREMENT NOT NULL,title VARCHAR(128) NOT NULL,location VARCHAR(128) NOT NULL,room VARCHAR(128) NOT NULL ,price VARCHAR(64) NOT NULL ,average_price VARCHAR(64) NOT NULL ,url VARCHAR(768) NOT NULL);""")conn.commit()seed_url = "https://nj.58.com/pinpaigongyu/pn/{page}/"page = 0flag=Truewhile flag:try:user_agent = get_user_agent()headers={"User_Agent":user_agent}page = page+1current_url=seed_url.format(page=page)print current_urlresponse = download(url=current_url,headers=headers)new_page = convertNumber(response.text)tree = html.fromstring(new_page)li_tags = tree.xpath('//ul[@class="list"]/li')#print li_tagsif (not li_tags) or page>600:print pageflag=Falsefor li_tag in li_tags:title = li_tag.xpath('.//div[@class="des strongbox"]/h2/text()')[0].strip()room = li_tag.xpath('.//p[@class="room"]/text()')[0].replace('\r\n', '').replace(r' ', '')price = li_tag.xpath('.//div[@class="money"]//b/text()')[0].strip().replace('\r\n', '').replace(r' ', '')url = li_tag.xpath('./a[@href]')[0].attrib['href'].strip()location = getLocation(title)average_price = getAveragePrice(price)cursor.execute("""INSERT INTO 58house(title,location,room,price,average_price,url) VALUES('%s','%s','%s','%s','%s','%s');"""%(title,location,room,price,average_price,url))conn.commit()except Exception as e:print "download page %s error: %s"%(page,e)time.sleep(5) # 停顿时间小于5s时会要求图形验证码conn.close()