python爬取携程机票并发送过滤后的机票信息到指定邮箱

0x0 前言

在疫情发生之前，在知乎上刷到一个回答，说是用服务器实现一个自动爬取机票并将过滤后的机票信息发送到自己的邮箱中，感觉十分有趣。由于疫情原因无法返校，加上家和学校相隔接近3000公里，再加上阿里云送服务器。于是想起了这个idea。

0x1 实现思路

服务器镜像选择的是centos7.7，个人比较喜欢centos，因为可以直接通过root用户操作。
爬取机票信息，选取了携程作为数据源。
存放机票信息，选择Mariadb。
发送机票信息邮件到我的邮箱。

基于以上选择，在Ubuntu18.04平台上使用python作为开发语言。大概的结构如下图。

0x2 实现过程

阿里云服务器的领取就比较简单了，不再赘述。

爬取机票信息需要爬虫，但爬虫对我来说完全是一个全新的知识，事先找了很多资料，发现很多文章都注重如何防止爬虫被识别。但在我的想法中这个项目每隔一天爬取一次数据就足够了。所以在这两个文章的帮助下,成功的完成了爬取机票信息的过程。Python3爬虫基础实战篇之机票数据采集,Python爬取携程机票代码实例。因为对这方面也不是很熟悉，所以在这里就不涉及分析的过程了。

#coding:utf-8
#getllist.py
import log
import requests
import getjson
import json# 获取艺龙机票信息
def getlylist():data=getjson.getcity() # 获取查询需要的请求数据Departure=str(data[0])    # 出发机场三字码DepartureName=str(data[1])    # 出发城市名Arrival=str(data[2])        # 到达机场三字码ArrivalName=str(data[3])  # 到达城市名DepartureDate=str(data[4])  # 出发日期try:url = "https://www.ly.com/flights/api/getflightlist"headers = {"Cookie":"__ftoken=nH1p4OADy%2BhIyjjDXqd7UuCg97ZyEvHWEuua31Q9G2zMeSBPRpzk3a%2FuWEIc7QlNqnha2ncAIbeEOHQQFPWvlA%3D%3D; __ftrace=efde5834-4ae4-45e4-8f55-8942d7304bf8; _dx_uzZo5y=696e7d7c4d7cb8e724cb154e92fd6d27e1fec611898ed7f00d27f905a63c9f8e672f2a0f; _dx_app_bc4b3ca6ae27747981b43e9f4a6aa769=5ed9ea6bGCjbBOcZ1O2OOVzPEPgKWXPp6E1Dv5g1","User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36 Edg/83.0.478.44",}data = {"Departure":Departure,"Arrival":Arrival,"GetType":"1","QueryType":"1","DepartureDate":DepartureDate,"DepartureName":DepartureName,"ArrivalName":ArrivalName,"IsBaby":0,"paging":{"cid":"8f272df1-5931-48bb-b5bd-8d5181339194","dataflag":"some"},"DepartureFilter":"","ArrivalFilter":"","flat":"465","plat":"465","isFromKylin":1,"refid":""}html = requests.post(url, headers=headers,data=data).textlog.info("Crawling information from ly succeeded")return htmlexcept: log.error("Failed to crawl information from ly")# 获取携程机票信息
def getcxlist():data=getjson.getcity() # 获取查询需要的请求数据Departure=str(data[0])    # 出发机场三字码DepartureName=str(data[1])    # 出发城市名Arrival=str(data[2])        # 到达机场三字码ArrivalName=str(data[3])  # 到达城市名DepartureDate=str(data[4])  # 出发日期try:url = "https://flights.ctrip.com/itinerary/api/12808/products"headers = {"Referer":"https://flights.ctrip.com/itinerary/oneway/"+Departure+"-"+Arrival+"?date="+DepartureDate,"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:77.0) Gecko/20100101 Firefox/77.0","Content-Type": "application/json"}data = {"flightWay":"Oneway","classType":"ALL","hasChild":False,"hasBaby":False,"searchIndex":1,"airportParams":[{"dcity":Departure,"acity":Arrival,"dcityname":DepartureName,"acityname":ArrivalName,"date":DepartureDate}],"token":"f08d143d0e4dc48def301a5a04f2ae7b"}html = requests.post(url, headers=headers,data=json.dumps(data)).textlog.info("Crawling information from cx succeeded")return htmlexcept:log.error("Failed to crawl information from cx")

在centos7上用执行yum install mariadb-server安装mariadb，其他配置可以参考centos7 安装Mariadb。表结构如下。

通过python的smtplib模块发送邮件，可以参考。

#!/usr/bin/python
# -*- coding: UTF-8 -*-
#smtp.py
import data
import log
import getjson
import smtplib
from email.mime.text import MIMEText
from email.header import Header# 构造邮件正文
def getmailtext():myresult = data.getfilterdata()text=''formattext="航空公司:%s 航班号:%s 价格:%s \n起飞时间:%s 降落时间:%s \n出发航站楼:%s 到达航站楼:%s 机型:%s\n----------------\n"for x in myresult:stri=formattext % (x[1],x[2],x[3],x[4],x[5],x[6]+x[7]+"航站楼",x[8]+x[9]+"航站楼",x[10])text+=strireturn text# 发送邮件
def mail():header=getjson.getmailheader()mail_host=header[0]  #设置服务器sender_pass=header[1]   #口令 sender = header[2]receivers = header[3] # 接收邮件，可设置为你的QQ邮箱或者其他邮箱try:message = MIMEText(getmailtext(), 'plain', 'utf-8')message['From'] = Header('<'+sender+'>', 'utf-8')message['To'] =  Header(receivers, 'utf-8')message['Subject'] = "机票信息"smtpObj = smtplib.SMTP_SSL(mail_host, 465)smtpObj.login(sender,sender_pass)smtpObj.sendmail(sender, receivers, message.as_string())log.info("Mail sent successfully")except smtplib.SMTPException:log.error("Unable to send Mail")

剩余模块代码。

data.py

# -*- coding: UTF-8 -*-
# data.py
import getlist
import log
import mysqldao
import getjson
import datetime
import json
import sys
reload(sys)
sys.setdefaultencoding('utf8')# 存储艺龙机票信息
def storelydata():try:html = getlist.getcxlist()html = json.loads(html)["body"]for j in html["FlightInfoSimpleList"]:item={"flyOffTime":j["flyOffTime"],"arrivalTime":j["arrivalTime"],"flightNo":j["flightNo"],"price":j["lecp"],"equipmentName":j['equipmentName'],"arriveAirportShortName":j["arriveAirportShortName"],"boardPoint":j["boardPoint"],"originAirportShortName":j["originAirportShortName"],"offPoint":j["offPoint"],"Departure":j["originAirportCode"],"Arrival":j["arriveAirportCode"],}if mysqldao.getbyflightNo(j["flightNo"]):mysqldao.updatetable(item)else:mysqldao.insertdatabase(item)log.info("Data saved successfully")except:log.error("Failed to save data!!!")# 存储携程机票信息
def storecxdata():try:html = json.loads(getlist.getcxlist()).get('data').get('routeList')#print(html)for j in html:if len(j.get('legs')) == 1:legs = j.get('legs')flight = legs[0].get('flight')characteristic = legs[0].get('characteristic')item={"airlineName":flight["airlineName"],"flyOffTime":flight["departureDate"],"arrivalTime":flight["arrivalDate"],"flightNo":flight["flightNumber"],"price":characteristic["lowestPrice"],"equipmentName":flight['craftTypeName']+'('+flight["craftTypeKindDisplayName"]+')',"arriveAirportShortName":flight["arrivalAirportInfo"]["cityName"]+flight["arrivalAirportInfo"]["airportName"],"boardPoint":flight["arrivalAirportInfo"]["terminal"]["name"],"originAirportShortName":flight["departureAirportInfo"]["cityName"]+flight["departureAirportInfo"]["airportName"],"offPoint":flight["departureAirportInfo"]["terminal"]["name"],"DepartureAirportTlc":flight["departureAirportInfo"]["airportTlc"],"ArrivalAirportTlc":flight["arrivalAirportInfo"]["airportTlc"],}if mysqldao.getbyflightNo(flight["flightNumber"]):mysqldao.updatetable(item)else:mysqldao.insertdatabase(item)log.info("Data saved successfully")except:log.error("Failed to save data!!!")# 从数据库中获取list类型的数据
def getdata():try:data=mysqldao.gettable()log.info("Read data successfully")return dataexcept:log.error("Failed to read data")# 过滤条件
def getfilter(condition):ret=Falsefliter=getjson.getfliter()price=fliter[0]date=datetime.datetime.strptime(fliter[1],'%Y-%m-%d %H:%M:%S')Departure=fliter[2]Arrival=fliter[3]if (date.date()==condition[4].date() and date.time()>condition[4].time()):if (price>condition[3] and Departure==condition[11] and Arrival==condition[12]):ret=Truereturn ret# 获取经过过滤的list类型的数据
def getfilterdata():list = []try:result=mysqldao.gettable()for x in result:if getfilter(x):list.append(x)log.info("Read data successfully")return listexcept:log.error("Failed to read data")

mysqldao.py

#!/usr/bin/python
# -*- coding: UTF-8 -*-
# mysqldao.py
import mysql.connector
import log
import json
import sys
reload(sys)
sys.setdefaultencoding('utf8')
mydb = mysql.connector.connect(host="localhost",user="root",passwd="admin",database="test"
)
mycursor = mydb.cursor()# 构造插入sql语句
def dic2sqlinsert(dic):sql = "insert into airfare (%s) VALUES (%s);"sf = ''index=''for key in dic:value = str(dic[key])ff= str(key)if key =='price':sf += (value + ',')else:sf += ('"'+value + '",')index+=(ff+',')sf = sf.rstrip(',')index = index.rstrip(',')sql2 = sql % (index,sf)return sql2# 构造更新sql语句
def dic2sqlupdate(dic):sql="UPDATE airfare SET %s WHERE flightNo='%s';"sf = ''temp=''flightNo=''for key in dic:value = str(dic[key])ff= str(key)if key=='flightNo':flightNo=valuecontinueelif key =='price':sf = (value + ',')else:sf = ('"'+value + '",')temp+=ff+'='+sftemp=temp.rstrip(',')sql=sql % (temp,flightNo)return sql# 执行sql语句
def executesql(sql):mycursor.execute(sql)# 数据表内容有更新，必须使用到该函数
def reflashdatabase():mydb.commit()# 增加行
def insertdatabase(data):sql=dic2sqlinsert(data)executesql(sql) #mycursor.execute(ret)reflashdatabase()# 查询表
def gettable():executesql("SELECT * FROM airfare")myresult = mycursor.fetchall() return myresult# 通过航班号查询
def getbyflightNo(flightNo):sql="SELECT * FROM airfare where flightNo='%s'"sql=sql % flightNoexecutesql(sql)myresult = mycursor.fetchall()return myresult# 通过航班号更新
def updatetable(data):sql=dic2sqlupdate(data)executesql(sql)reflashdatabase()

getjson.py

#coding:utf-8
import json
import sys
reload(sys)
sys.setdefaultencoding('utf8')# 获取发送邮件的用户和接受的用户
def getmailheader():with open('/root/project/program/json/email.json', 'r') as f:   #根据自身的路径填入绝对路径header = json.load(f)tup = (header["mail_host"],header["sender_pass"],header["sender"],header["receivers"])return tup# 获得需要爬取的机票的数据
def getcity():with open('/root/project/program/json/crawl.json', 'r') as f:crawl = json.load(f)tup = (crawl["Departure"],crawl["DepartureName"],crawl["Arrival"],crawl["ArrivalName"],crawl["DepartureDate"])return tup# 获得过滤的条件
def getfliter():with open('/root/project/program/json/fliter.json', 'r') as f:fliter= json.load(f)tup = (fliter["price"],fliter["flyOffTime"],fliter["Departure"],fliter["Arrival"])return tup

log.py

#coding:utf-8
import logging
import sys
reload(sys)
sys.setdefaultencoding('utf8')
LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"
logging.basicConfig(filename='/root/project/program/log/info.log', level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) #写入绝对路径# 记录运行信息
def info(text):logging.info(text)# 记录错误信息
def error(text):logging.error(text)

文件存放结构

json文件夹中存放相应的.json文件，log文件夹中存放日志信息，program文件夹中放入python脚本。

选择Linux Crontab来触发查询和发送邮件。执行contab -e写入

30 6 * * * /usr/bin/python /root/project/program/mail.py #每天的早上6:30发送邮件
10 6 * * * /usr/bin/python /root/project/program/get.py  #每天的早上6:10爬取数据

因为contab不会继承环境变量所以要写入绝对路径

mail.py

#coding:utf-8
import data
data.storecxdata()

get.py

#coding:utf-8
import smtp
smtp.mail()

测试结果。

0x3 总结

通过这个项目了解到了爬虫的实现原理，感觉还是很不错的。这个项目的代码写的还是有点乱的，文件的存放也不是很规范。按需食用。