前言

最近购买了《Python3 爬虫、数据清洗与可视化实战》，刚好适逢暑假，就尝试从携程页面对广州的周边游产品进行爬虫数据捕捉。

因为才学Python不够一个星期，python的命名规范还是不太了解，只能套用之前iOS开发的命名规范，有不足之处请多多指点

一、前期

1.主要用到的库

from bs4 import BeautifulSoup

import time

import re #正则表达式

from selenium import webdriver

from selenium.webdriver.common.by import By

from selenium.webdriver.support.wait import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

from selenium.webdriver.common.action_chains import ActionChains #浏览器操作

import xlrd

import xlwt

from xlutils.copy import copy

import os

BeautifulSoup：用于对标签等数据进行定位和抓取

selenium：用于启动浏览器和对页面进行自动操作

time：暂停等待操作

xlrd、xlwt、xlutils：对数据结果进行Excel读写保存操作

2.核心思路

1，跳进出发点的周边游页面(广州)

2，在首页捕捉推荐的热门目的地和热点景点，进行保存

3，针对目的地地点进行遍历搜索所展示的旅游产品

4，产品数据参数抓取

5，数据保存

6，退出浏览器

二、代码

1.启动浏览器

def setupDriverSetting():

global driver

# url = 'http://m.ctrip.com/restapi/soa2/10290/createclientid?systemcode=09&createtype=3&conte'#获取cookieID

# 手机端

# url = 'https://m.ctrip.com/webapp/vacations/tour/list?tab=64&kwd=%E7%8F%A0%E6%B5%B7&salecity=32&searchtype=tour&sctiy=32'

# 电脑端

url = 'https://weekend.ctrip.com/around/'

# 设置用chrome启动

driver = webdriver.Chrome()

# #设置fireFox请求头参数

# profile = webdriver.FirefoxProfile()

# user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:58.0) Gecko/20100101 Firefox/58.0"

# profile.set_preference("general.useragent.override",user_agent)

#

# driver = webdriver.Firefox(profile)

driver.get(url)

用webdriver启动Chrome或者fireFox，并跳进首页URL

2.选择出发点城市

def select_StartPlace(startPlace):

#点击出发点view

driver.find_element_by_xpath("//*[@id='CitySelect']").click()

#选择出发点

cityList = driver.find_elements_by_xpath("//*[@id='CitySelect']/dd/ul")

for link in cityList:

links = link.find_elements(By.TAG_NAME,"a")

for eachCity in links:

cityStr = eachCity.text

if cityStr == startPlace:

print("找到目标城市:"+eachCity.get_attribute('href'))

driver.get(eachCity.get_attribute('href'))

time.sleep(2)

try:

WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.XPATH, "//*[@id='SearchText']")))

except:

print('出发地页面加载不成功')

break

主要是用find_element_by_xpath寻找目标城市进行选择筛选，然后跳到城市专页

3.搜索目的地

def finAllDestinationPage():

#查找总数组

destType = driver.find_element_by_id("J_sub_circum")#id 决定产品范围(周边游，境外游)

print(destType.text)

destType1 = destType.find_element_by_class_name("side_jmp_dest")

destTypeItem = destType1.get_attribute('innerHTML')

item = BeautifulSoup(destTypeItem,'lxml')

destTypeList = item.find_all('li')

allDestinationListDic = {}

for each in destTypeList:

typeName = each.h4.string

typeList = each.find_all('a')

list = []

for i in typeList:

list.append(i.string)

allDestinationListDic[typeName] = list

return allDestinationListDic

搜索所有可推荐目的地和景点，并用字典保存

4.旅游产品列表页

def jump_destinationPage(startPlace,destination):

#定位搜索栏

try:

WebDriverWait(driver,5).until(EC.presence_of_element_located((By.XPATH,"//*[@id='SearchText']")))

except:

print('查找不到搜索栏')

finally:

print('本地页面加载完毕')

driver.find_element_by_xpath("//input[@id='SearchText']").send_keys(destination)

print("输入目的地："+destination)

driver.find_element_by_xpath("//*[@id='SearchBtn']").click()

print("点击搜索按钮结束")

time.sleep(2)

try:

WebDriverWait(driver,5).until(EC.presence_of_element_located((By.XPATH,"//*[@id='js-dpSearcher']")))

except:

print('产品列表页加载不成功')

finally:

print('产品列表页加载完毕')

#再选一次出发地，以防出错

reSelect_StartPlace(startPlace)

#搜索页数

pageHtml = driver.find_element_by_xpath("//*[@id='_sort']/div/span")

print(pageHtml.text)

pageNumStr = pageHtml.text

pageNumStr = pageNumStr[:-1]

print("获取的num:" + pageNumStr)

#正则表达式 查找页数

pageNumS = re.findall(r'\d+',pageNumStr)

pageNum = int(pageNumS[1])

print(pageNum)

tourProductList = []

for i in range(0,pageNum):

itemList = showCurrentPageAllData()

#收集数据

for j in range(0,len(itemList)):

eachItem = collectCurrentPageEachData(j)

tourProductList.append(eachItem)

#点击下一页

driver.find_element_by_xpath("//input[@id='ipt_page_txt']").clear()

driver.find_element_by_xpath("//input[@id='ipt_page_txt']").send_keys(str(i+2))

driver.find_element_by_xpath("//*[@id='ipt_page_btn']").click()

print("点击下一页结束->"+str(i+2)+"页")

time.sleep(2)

return driver

跳进产品页，并根据标签，抓取总页数，在遍历所有旅游产品后，再跳到下一页进行循环遍历

5.产品数据抓取

def collectCurrentPageEachData(itemNum):

itemList = driver.find_elements_by_class_name("product_box")

str = itemList[itemNum].get_attribute('innerHTML')#转换成字符串

# item = BeautifulSoup(str,"html.parser")#获取item的soup对象

item = BeautifulSoup(str, "lxml") # 获取item的soup对象

# print("+++++++"+item.prettify())

# 解析

#产品名称

titleNameHtml = item.find('h2',class_= 'product_title')

print("-------"+titleNameHtml.get_text())

productName = titleNameHtml.get_text()

#产品链接

productLink = titleNameHtml.a['href']

productLink = productLink[2:]

productLink = "https://"+productLink

print("link:" + productLink)

#产品类型

productType = item.find('em')

print("type:"+productType.get_text())

productTypeStr = productType.get_text()

#产品价格

priceHtml = item.find('span',class_='sr_price')

priceStr = priceHtml.strong.get_text()

#判断是否为数字

if priceStr.isdigit() == True :

priceStr = "%.2f"%float(priceStr)

print("price:"+priceStr)

#产品供应商

productRetail = item.find('p',class_='product_retail')

productRetailStr = productRetail['title']

if "供应商" in productRetailStr:

productRetailStr = productRetailStr[4:]

print("retail:" + productRetailStr)

#产品评分

try :

gradeHtml = item.find('p', class_='grade')

gradeStr = gradeHtml.strong.get_text()

print("grade:" + gradeStr)

except:

print('查找不到评分')

gradeStr = ''

# 产品人数

try:

commentHtml = item.find('div', class_='comment')

commentStr = commentHtml.em.get_text()

commentNumS = re.findall(r'\d+', commentStr)

commentNum = int(commentNumS[0])

print("comment:",commentNum)

except:

print('查找不到出游人数')

commentNum = ''

return {

'名称':productName,

'链接':productLink,

'类型':productTypeStr,

'价格':priceStr,

'供应商':productRetailStr,

'评分':gradeStr,

'人数':commentNum,

}

在产品页面上获取所有可见信息，并返回

6.数据保存

class ExcelFileManager:

def creatExcelFile(fileName,sheetName,headRowList):

# 获取项目所在目录

filePath = os.getcwd() + '/' + fileName + '.xls'

#如果不存在就新增

try:

oldFile = xlrd.open_workbook(filePath)

file = copy(oldFile)

except:

file = xlwt.Workbook()

print("新建文件")

#如果不存在就新增

try:

sheet1 = file.add_sheet(sheetName,cell_overwrite_ok=True)

except:

sheet1 = file.get_sheet(sheetName)

#设置style样式

head_style = xlwt.easyxf('font: name Times New Roman, color-index red, bold on',num_format_str='#,##0.00')

row0 = headRowList

for i in range(0,len(row0)):

sheet1.write(0,i,row0[i],head_style)

print(filePath)

file.save(filePath)

def addDataToExcelFile(fileName,sheetName,dataList):

filePath = os.getcwd()+'/'+fileName+'.xls'

file = xlrd.open_workbook(filePath)

#已存在的行数

newRows = file.sheet_by_name(sheetName).nrows

new_File = copy(file)

sheet = new_File.get_sheet(sheetName)

try:

for i in range(0,len(dataList)):

for j in range(0,len(dataList[i])):

sheet.write(i+newRows,j,dataList[i][j])

except Exception as e:

print(e)

new_File.save(filePath)

Excel文件创建与保存数据，不得不说，python对Excel支持不是很友好，xlrd和xlwt仅支持读和写，不支持增加sheet或者在原有Excel文件上添加数据等操作，需要用到第三方库

三、抓取结果：

1530848043475.jpg