安居客爬虫项目，爬取房源，保存mysql数据库，详细代码如下！！！

import time
import requests
import random
import pymysql
from lxml import etreeclass AnJuKe():# 初始化def __init__(self, url):self.connect = pymysql.connect(host = 'localhost',db = 'pachong',user = 'root',password = '12345')self.cursor = self.connect.cursor()#创建游标self.tree = self.get_tree(url)self.result_city()# 判断是否为空def is_empty(self,data):if data:data = data[0]else:data = '无信息'return data# 得到treedef get_tree(self, url):# 代理ipproxies_list = [{'http': 'http://117.191.11.111:8080'},{'http': 'http://118.25.104.254:1080'},{'http': 'http://203.195.168.154:3128'},{'http': 'http://117.191.11.75:80'},{'http': 'http://117.191.11.72:80'},]proxies = random.choice(proxies_list)headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36'}response = requests.get(url, headers = headers, proxies = proxies).text# print(response)tree = etree.HTML(response)return tree# 获取城市详情def result_city(self):# 城市列表city_list_class =  self.tree.xpath('//div[@class="letter_city"]/ul/li[position()>12 and position()<18]/div/a/@href')# print(city_list_class)for city_url in city_list_class:# print(city_url)city_url1 = city_url.split('.')[0]# print(city_url1)for i in range(1,51):# print(1)url = '%s.zu.anjuke.com/fangyuan/p%s/' % (city_url1,i)# 调用get函数tree = self.get_tree(url)room_list_class = tree.xpath(r'//div[@class="zu-itemmod"]')# 所有N-S循环for room_list in room_list_class:# print(1)# 主题title = room_list.xpath(r'./div[@class="zu-info"]/h3/a/b/text()')title = self.is_empty(title)title = self.zhuanma(title)# print('1',title)# 图片image = room_list.xpath(r'./a[@class="img"]//img[@class="thumbnail"]/@lazy_src')image = self.is_empty(image)# print('2',image)# 几室bedroom_num = room_list.xpath(r'./div[@class="zu-info"]/p[1]/b[1]/text()')bedroom_num = self.is_empty(bedroom_num)bedroom_num = self.zhuanma(bedroom_num)# print('3',bedroom_num)# 几厅living_room_num = room_list.xpath(r'./div[@class="zu-info"]/p[1]/b[2]/text()')living_room_num = self.is_empty(living_room_num)living_room_num = self.zhuanma(living_room_num)# print('0',living_room_num)# 面积area = room_list.xpath(r'./div[@class="zu-info"]/p[1]/b[3]/text()')area = self.is_empty(area)area = self.zhuanma(area)# print('4',area)# 楼层情况floor = room_list.xpath('./div[@class="zu-info"]/p/text()')[4].split('共')[0].replace('(','')# floor = self.is_empty(floor)# print('5',floor)# 总楼层floors = room_list.xpath('./div[@class="zu-info"]/p/text()')[4].split('共')[1].replace(')','')# floors = self.is_empty(floors)# print('6',floors)# 经纪人agent = room_list.xpath('./div[@class="zu-info"]/p/text()')[5]# agent = self.is_empty(agent)# print('7',agent)# 小区名neighborhood = room_list.xpath('./div[@class="zu-info"]/address[@class="details-item"]/a/text()')[0]# neighborhood = self.is_empty(neighborhood)# print('8',neighborhood)# 地址address =  room_list.xpath('./div[@class="zu-info"]/address[@class="details-item"]/text()')[1]addres = address.strip()# address = self.is_empty(address)# print('9',address)# 租房方式rent_way =  room_list.xpath('./div[@class="zu-info"]/p[@class="details-item bot-tag"]/span[1]/text()')rent_way = self.is_empty(rent_way)# print('10',rent_way)# 朝向face_direction =  room_list.xpath('./div[@class="zu-info"]/p[@class="details-item bot-tag"]/span[2]/text()')face_direction = self.is_empty(face_direction)# print('11',face_direction)#地铁subline =  room_list.xpath('./div[@class="zu-info"]/p[@class="details-item bot-tag"]/span[@class="cls-4"]/text()')subline = self.is_empty(subline)# print('12',subline)# 价格price = room_list.xpath('./div[@class="zu-side"]/p/strong/b/text()')[0]# price = self.is_empty(price)price = self.zhuanma(price)# print('13',price)self.save_mysql(title,image,bedroom_num,living_room_num,area,floor,floors,agent,neighborhood,addres,rent_way,face_direction,subline,price)time.sleep(120)#保存数据库def save_mysql(self,title,image,bedroom_num,living_room_num,area,floor,floors,agent,neighborhood,addres,rent_way,face_direction,subline,price):sql = 'insert into anjuke(title,image,bedroom_num,living_room_num,area,floor,floors,agent,neighborhood,addres,rent_way,face_direction,subline,price) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'self.cursor.execute(sql, (title,image,bedroom_num,living_room_num,area,floor,floors,agent,neighborhood,addres,rent_way,face_direction,subline,price))self.connect.commit()print('数据插入成功')# except:#     print('数据插入失败')# 自创字符转码def zhuanma(self,mm):str1 = ''dicts = {'驋': '1', '餼': '2', '龤': '3', '麣': '4', '鑶': '5', '齤': '6', '鸺': '7', '閏': '8', '龥': '9', '龒': '0','.':'.'}for i in mm:if i in dicts:ss = dicts[i]str1 += sselse:str1 += ireturn str1if __name__ == '__main__':url = 'https://www.anjuke.com/sy-city.html'AnJuKe(url)

安居客爬虫项目，爬取房源，保存mysql数据库，详细代码如下！！！相关推荐

python爬虫金融数据_python爬虫项目-爬取雪球网金融数据（关注、持续更新）
(一)python金融数据爬虫项目爬取目标:雪球网(起始url:https://xueqiu.com/hq#exchange=cn&firstname=1&secondname=1_ ...
Python爬虫项目--爬取某宝男装信息
本次爬取用到的知识点有: 1. selenium 2. pymysql 3 pyquery ''' 文章:Python爬虫项目--爬取某宝男装信息作者:Star_Zhao 源自:https://w ...
python爬虫实现爬取网页主页信息（html代码）
python爬虫实现爬取网页主页信息(html代码) 1.爬取网站源码 urllib整体介绍: urllib是一个包,收集几个模块来处理网址 urllib.request打开和浏览url中内容 url ...
高德地图交通态势爬取并存入MySQL数据库
高德地图交通态势爬取并存入mysql数据库高德地图交通态势开放api链接: 高德交通态势高德地图交通态势使用流程: 第一步,申请"Web服务API接口"密钥(Key): 第二步 ...
Scrapy-redis 分布式爬虫-成都安居客二手房数据爬取
Joint-spider 项目地址:GitHub Scrapy-Redis 架构: 成都贝壳,安居客房源信息爬虫基于 python 分布式房源数据爬取系统,为房价数据挖掘及可视化提供数据支持.采用 ...
爬虫项目--爬取安居客二手房信息
爬虫实战(爬取安居客二手房信息-成都天府新区) 环境:python3.6 pycharm bs4库解析方式:bs4 需求:爬取二手房信息字段(titile,house_type,build_time ...
实战项目一、安居客(北京) 二手房抓取房源信息
一.首先明确爬取的数据为安居客(北京)的二手房源的数据信息,主要有房源链接地址,房源价格,房源单价,房源规模,房源大小,房源建造年份,房源地址. https://beijing.anjuke.com/ ...
python爬虫爬取房源_Python爬虫项目--爬取自如网房源信息
本次爬取自如网房源信息所用到的知识点: 1. requests get请求 2. lxml解析html 3. Xpath 4. MongoDB存储正文 1.分析目标站点 1. url: http:/ ...
爬虫实战-北京链家，安居客二手房的爬取
链家mobie北京二手房5w多信息抓取,存MongoDB后进行数据可视化 import scrapy from scrapy_redis.spiders import RedisCrawlSpider ...
安居客二手房信息爬取
本文实现爬取安居客二手房房源信息,并保存到本地csv文本中爬取网址:https://tianjin.anjuke.com/sale/hexi/(天津河西区二手房房源) 思路 1.构造请求地址:通过分 ...

安居客爬虫项目，爬取房源，保存mysql数据库，详细代码如下！！！

安居客爬虫项目，爬取房源，保存mysql数据库，详细代码如下！！！相关推荐

最新文章

热门文章