Python爬虫抓取去哪儿网景点信息告诉你国庆哪儿最堵

摘要

本文主要介绍了使用Python抓取去哪儿网站的景点信息并使用BeautifulSoup解析内容获取景点名称、票销售量、景点星级、热度等数据，然后使用xlrd、xlwt、xlutils等库来处理Excel，写入到Excel中，最后使用matplotlib可视化数据，并用百度的heatmap.js来生成热力图。
首先，上张效果图：

如果想了解更多Python的伙伴或者小白中有任何困难不懂的可以加入我们python交流学习QQ群：250933691，多多交流问题，互帮互助，群里有不错的学习教程和开发工具。资源分享

下面就详细来介绍如何一步步实现。

准备省份名单

访问是按照省份来进行搜索的，所以我们需要准备一份全国各省份的名单，这里，我已经准备好了这份名单

北京市，天津市，上海市，重庆市，河北省，山西省，辽宁省，吉林省，黑龙江省，江苏省，浙江省，安徽省，福建省，江西省，山东省，河南省，湖北省，湖南省，广东省，海南省，四川省，贵州省，云南省，陕西省，甘肃省，青海省，台湾省，内蒙古自治区，广西壮族自治区，西藏自治区，宁夏回族自治区，新疆维吾尔自治区，香港，澳门

将这些数据保存为TXT，一行

然后使用Python加载：

def ProvinceInfo(province_path):tlist = []with open(province_path, 'r', encoding='utf-8') as f:lines = f.readlines()for line in lines:tlist = line.split('，')return tlist

构建URL
这里URL是根据城市名称信息来生成的

  site_name = quote(province_name)  # 处理汉字问题url1 = 'http://piao.qunar.com/ticket/list.htm?keyword='url2 = '&region=&from=mps_search_suggest&page='url = url1 + site_name + url2

当然上面这个URL还不是最终的URL，因为一个城市搜索后有很多页面，我们需要定位到具体页面才行，这涉及到了如何判断页面数的问题，放在下文。
抓取页面信息函数:

# 获得页面景点信息def GetPageSite(url):try:page = urlopen(url)except AttributeError:logging.info('抓取失败！')return 'ERROR'try:bs_obj = BeautifulSoup(page.read(), 'lxml')# 不存在页面if len(bs_obj.find('div', {'class': 'result_list'}).contents) <= 0:logging.info('当前页面没有信息！')return 'NoPage'else:page_site_info = bs_obj.find('div', {'class': 'result_list'}).childrenexcept AttributeError:logging.info('访问被禁止！')return Nonereturn page_site_info

获得页面数目

# 获取页面数目def GetPageNumber(url):try:page = urlopen(url)except AttributeError:logging.info('抓取失败！')return 'ERROR'try:bs_obj = BeautifulSoup(page.read(), 'lxml')# 不存在页面if len(bs_obj.find('div', {'class': 'result_list'}).contents) <= 0:logging.info('当前页面没有信息！')return 'NoPage'else:page_site_info = bs_obj.find('div', {'class': 'pager'}).get_text()except AttributeError:logging.info('访问被禁止！')return None# 提取页面数page_num = re.findall(r'\d+\.?\d*', page_site_info.split('...')[-1])return int(page_num[0])

对取得的数据进行解析取得感兴趣的数据

# 格式化获取信息def GetItem(site_info):site_items = {}  # 储存景点信息site_info1 = site_info.attrssite_items['name'] = site_info1['data-sight-name']  # 名称site_items['position'] = site_info1['data-point']  # 经纬度site_items['address'] = site_info1['data-districts'] + ' ' + site_info1['data-address']  # 地理位置site_items['sale number'] = site_info1['data-sale-count']  # 销售量site_level = site_info.find('span', {'class': 'level'})if site_level:site_level = site_level.get_text()site_hot = site_info.find('span', {'class': 'product_star_level'})if site_hot:site_hot = site_info.find('span', {'class': 'product_star_level'}).em.get_text()site_hot = site_hot.split(' ')[1]site_price = site_info.find('span', {'class': 'sight_item_price'})if site_price:site_price = site_info.find('span', {'class': 'sight_item_price'}).em.get_text()site_items['level'] = site_levelsite_items['site_hot'] = site_hotsite_items['site_price'] = site_pricereturn site_items

获取一个省的全部页面数据，用到了前面的函数

# 获取一个省的所有景点def GetProvinceSite(province_name):site_name = quote(province_name)  # 处理汉字问题url1 = 'http://piao.qunar.com/ticket/list.htm?keyword='url2 = '&region=&from=mps_search_suggest&page='url = url1 + site_name + url2NAME = []  # 景点名称POSITION = []  # 坐标ADDRESS = []  # 地址SALE_NUM = []  # 票销量SALE_PRI = []  # 售价STAR = []  # 景点星级SITE_LEVEL = []  # 景点热度i = 0  # 页面page_num = GetPageNumber(url + str(i + 1))  # 页面数logging.info('当前城市 %s 存在 %s 个页面' % (province_name, page_num))flag = True  # 访问非正常退出标志while i < page_num:  # 遍历页面i = i + 1# 随机暂停1--5秒，防止访问过频繁被服务器禁止访问time.sleep(1 + 4 * random.random())# 获取网页信息url_full = url + str(i)site_info = GetPageSite(url_full)# 当访问被禁止的时候等待一段时间再进行访问while site_info is None:wait_time = 60 + 540 * random.random()while wait_time >= 0:time.sleep(1)logging.info('访问被禁止，等待 %s 秒钟后继续访问' % wait_time)wait_time = wait_time - 1# 继续访问site_info = GetPageSite(url_full)if site_info == 'NoPage':  # 访问完成logging.info('当前城市 %s 访问完成，退出访问！' % province_name)breakelif site_info == 'ERROR':  # 访问出错logging.info('当前城市 %s 访问出错,退出访问' % province_name)flag = Falsebreakelse:# 返回对象是否正常if not isinstance(site_info, Iterable):logging.info('当前页面对象不可迭代 ，跳过 %s' % i)continueelse:# 循环获取页面信息for site in site_info:info = GetItem(site)NAME.append(info['name'])POSITION.append(info['position'])ADDRESS.append(info['address'])SALE_NUM.append(info['sale number'])SITE_LEVEL.append(info['site_hot'])SALE_PRI.append(info['site_price'])STAR.append(info['level'])logging.info('当前访问城市 %s,取到第 %s 组数据: %s' % (province_name, i, info['name']))return flag, NAME, POSITION, ADDRESS, SALE_NUM, SALE_PRI, STAR, SITE_LEVEL

最后就是把数据写入到Excel中，这里因为数据量很大，而且是获得了一个城市的数据后再写入一次，而在爬取过程中很可能由于各种原因中断，因而每次读取Excel都会判断当前省份是否已经读取过。

# 创建Exceldef CreateExcel(path, sheets, title):try:logging.info('创建Excel: %s' % path)book = xlwt.Workbook()for sheet_name in sheets:sheet = book.add_sheet(sheet_name, cell_overwrite_ok=True)for index, item in enumerate(title):sheet.write(0, index, item, set_style('Times New Roman', 220, True))book.save(path)except IOError:return '创建Excel出错！'# 设置Excel样式def set_style(name, height, bold=False):style = xlwt.XFStyle()  # 初始化样式font = xlwt.Font()  # 为样式创建字体font.name = name  # 'Times New Roman'font.bold = boldfont.color_index = 4font.height = height# borders= xlwt.Borders()# borders.left= 6# borders.right= 6# borders.top= 6# borders.bottom= 6style.font = font# style.borders = bordersreturn style# 加载Excel获得副本def LoadExcel(path):logging.info('加载Excel：%s' % path)book = xlrd.open_workbook(path)copy_book = copy(book)return copy_book# 判断内容是否存在def ExistContent(book, sheet_name):sheet = book.get_sheet(sheet_name)if len(sheet.get_rows()) >= 2:return Trueelse:return False# 写入Excel并保存def WriteToTxcel(book, sheet_name, content, path):logging.info('%s 数据写入到 (%s-%s)' % (sheet_name, os.path.basename(path), sheet_name))sheet = book.get_sheet(sheet_name)for index, item in enumerate(content):for sub_index, sub_item in enumerate(item):sheet.write(sub_index + 1, index, sub_item)book.save(path)

数据分析、可视化

完成了前面几个步骤之后，我们就已经做好了爬取数据的工作了，现在就是需要可视化数据了，这里，设计的主要内容有：读取Excel数据，然后对每一个sheet（一个省份）读取数据，并去处重复数据，最后按照自己的要求可视化，当然，这里地图可视化部分使用了百度的heatmap.js工具，首先需要把景点的经纬度等信息生成json格式。

def GenerateJson(ExcelPath, JsonPath, SalePath, TransPos=False):try:if os.path.exists(JsonPath):os.remove(JsonPath)if os.path.exists(SalePath):os.remove(SalePath)sale_file = open(SalePath, 'a', encoding='utf-8')json_file = open(JsonPath, 'a', encoding='utf-8')book = xlrd.open_workbook(ExcelPath)except IOError as e:return esheets = book.sheet_names()sumSale = {}  # 总销售量for sheet_name in sheets:sheet = book.sheet_by_name(sheet_name)row_0 = sheet.row_values(0, 0, sheet.ncols - 1)  # 标题栏数据# 获得热度栏数据for indx, head in enumerate(row_0):if head == '销售量':index = indxbreaklevel = sheet.col_values(index, 1, sheet.nrows - 1)# 获得景点名称数据for indx, head in enumerate(row_0):if head == '名称':index = indxbreaksite_name = sheet.col_values(index, 1, sheet.nrows - 1)if not TransPos:for indx, head in enumerate(row_0):if head == '经纬度':index = indxbreakpos = sheet.col_values(index, 1, sheet.nrows - 1)temp_sale = 0  # 临时保存销售量for i, p in enumerate(pos):if int(level[i]) > 0:lng = p.split(',')[0]lat = p.split(',')[1]lev = level[i]temp_sale += int(lev)sale_temp = sheet_name + site_name[i] + ',' + levjson_temp = '{"lng":' + str(lng) + ',"lat":' + str(lat) + ', "count":' + str(lev) + '}, 'json_file.write(json_temp + '\n')sale_file.write(sale_temp + '\n')sumSale[sheet_name] = temp_saleelse:passjson_file.close()sale_file.close()return sumSale

当然，上面这个函数同时还绘制了景点销量信息的图。不过这里先讨论生成json文本后接下来处理。运行上面的程序会在你指定的路径下生成一个名为LngLat.json的文件，使用文本编辑器打开，然后把内容复制到heatmap.html这个文件的数据部分，这里为了代码不至于太长我删除了大部分数据信息，只保留了一部分，你只需要把下面的代码复制保存为html格式然后在 var points =[]中添加生成的json内容就可以了。最后使用浏览器打开，即可看到下面这样的效果：

<!DOCTYPE html>
<html>
<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /><meta name="viewport" content="initial-scale=1.0, user-scalable=no" /><script type="text/javascript" src="http://gc.kis.v2.scr.kaspersky-labs.com/C8BAC707-C937-574F-9A1F-B6E798DB62A0/main.js" charset="UTF-8"></script><script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=x2ZTlRkWM2FYoQbvGOufPnFK3Fx4vFR1"></script><script type="text/javascript" src="http://api.map.baidu.com/library/Heatmap/2.0/src/Heatmap_min.js"></script><title>热力图功能示例</title><style type="text/css">ul,li{list-style: none;margin:0;padding:0;float:left;}html{height:100%}body{height:100%;margin:0px;padding:0px;font-family:"微软雅黑";}#container{height:500px;width:100%;}#r-result{width:100%;}</style>
</head>
<body>
<div id="container"></div>
<div id="r-result"><input type="button"  onclick="openHeatmap();" value="显示热力图"/><input type="button"  onclick="closeHeatmap();" value="关闭热力图"/>
</div>
</body>
</html>
<script type="text/javascript">var map = new BMap.Map("container");          // 创建地图实例var point = new BMap.Point(105.418261, 35.921984);map.centerAndZoom(point, 5);             // 初始化地图，设置中心点坐标和地图级别map.enableScrollWheelZoom(); // 允许滚轮缩放var points =[
{"lng":116.403347,"lat":39.922148, "count":19962},
{"lng":116.03293,"lat":40.369733, "count":3026},
{"lng":116.276887,"lat":39.999497, "count":3778},
{"lng":116.393097,"lat":39.942341, "count":668},
{"lng":116.314607,"lat":40.01629, "count":1890},
{"lng":116.020213,"lat":40.367229, "count":2190},
{"lng":116.404015,"lat":39.912729, "count":904},
{"lng":116.398287,"lat":39.94015, "count":392}, {"lng":89.215713,"lat":42.94202, "count":96},
{"lng":89.212779,"lat":42.941938, "count":83},
{"lng":90.222236,"lat":42.850153, "count":71},
{"lng":80.931218,"lat":44.004188, "count":82},
{"lng":89.087234,"lat":42.952765, "count":40},
{"lng":86.866582,"lat":47.707518, "count":54},
{"lng":85.741271,"lat":48.36813, "count":4},
{"lng":87.556853,"lat":43.894646, "count":83},
{"lng":89.699515,"lat":42.862384, "count":81},
{"lng":80.903663,"lat":44.286633, "count":53},
{"lng":89.254534,"lat":43.025333, "count":50},
{"lng":86.1271,"lat":41.789203, "count":63},
{"lng":84.537278,"lat":43.314894, "count":81},
{"lng":84.282954,"lat":41.286104, "count":94},
{"lng":77.181601,"lat":37.397422, "count":32},
{"lng":82.666502,"lat":41.611567, "count":64},
{"lng":89.577441,"lat":44.008065, "count":57},
{"lng":83.056664,"lat":41.862089, "count":79},
{"lng":82.639664,"lat":41.588593, "count":53},
{"lng":89.537959,"lat":42.888903, "count":61},
{"lng":89.52734,"lat":42.876443, "count":95},
{"lng":87.11464,"lat":48.310173, "count":86},
{"lng":80.849732,"lat":44.238021, "count":6},
{"lng":89.488521,"lat":42.991858, "count":59},
{"lng":89.550783,"lat":42.882572, "count":92},
{"lng":88.055115,"lat":44.13238, "count":61},
{"lng":77.100143,"lat":39.095865, "count":63},
{"lng":78.992124,"lat":41.103398, "count":42},
{"lng":77.699877,"lat":39.013786, "count":62},
{"lng":81.912557,"lat":43.222123, "count":61},
{"lng":87.526264,"lat":47.75415, "count":33},
{"lng":87.556853,"lat":43.894632, "count":110},
{"lng":87.622686,"lat":43.820354, "count":10}, ]if(!isSupportCanvas()){alert('热力图目前只支持有canvas支持的浏览器,您所使用的浏览器不能使用热力图功能~')}
//详细的参数,可以查看heatmap.js的文档 https://github.com/pa7/heatmap.js/blob/master/README.md
//参数说明如下:
/* visible 热力图是否显示,默认为true* opacity 热力的透明度,1-100* radius 势力图的每个点的半径大小   * gradient  {JSON} 热力图的渐变区间 . gradient如下所示*    {.2:'rgb(0, 255, 255)',.5:'rgb(0, 110, 255)',.8:'rgb(100, 0, 255)'}其中 key 表示插值的位置, 0~1. value 为颜色值. */
heatmapOverlay = new BMapLib.HeatmapOverlay({"radius":20});
map.addOverlay(heatmapOverlay);
heatmapOverlay.setDataSet({data:points,max:10000});
//是否显示热力图function openHeatmap(){heatmapOverlay.show();}
function closeHeatmap(){heatmapOverlay.hide();}
closeHeatmap();function setGradient(){/*格式如下所示:{0:'rgb(102, 255, 0)',.5:'rgb(255, 170, 0)',1:'rgb(255, 0, 0)'}*/var gradient = {};var colors = document.querySelectorAll("input[type='color']");colors = [].slice.call(colors,0);colors.forEach(function(ele){gradient[ele.getAttribute("data-key")] = ele.value; });heatmapOverlay.setOptions({"gradient":gradient});}
//判断浏览区是否支持canvasfunction isSupportCanvas(){var elem = document.createElement('canvas');return !!(elem.getContext && elem.getContext('2d'));}
</script>