抓取了99%的数据还剩一点点瑕疵要处理掉.

在脚本的运行日志里不和谐的一段:

05-29 18:14 transchedule INFO train number 5611 processed

05-29 18:14 transchedule ERROR 2:2

05-29 18:14 transchedule INFO start process http://www.tielu.org/TrainList/TrainList-5.html

05-29 18:14 transchedule INFO train number 5612 processed

05-29 18:14 transchedule ERROR 7 黄:2

5611和5612的椑木镇和黄磏,这两个生僻字直接导致了我的脚本少了14条途经车站的数据....数据该升级啦

铁路网的数据: 这边是极品时刻表的:

原来他们的数据源是一样的.....到他们家论坛提个BUG去.

把5611和5612存到本地,改掉html的gb2312编码,补好数据,完工啦.

========数据补充脚本========

#! /usr/bin/env python

#coding=utf8

#import urllib2

from HTMLParser import HTMLParser

from pyquery import PyQuery as pq

import sqlite3,urllib2,logging,sys

from datetime import datetime

from decimal import Decimal

# 日志初始化

logFileName='./%s.log'%(datetime.now().strftime("%Y%m%d%H%M%S"))

logging.basicConfig(level=logging.INFO,

format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',

datefmt='%m-%d %H:%M',

filename=logFileName,

filemode='w')

logger = logging.getLogger('transchedule')

hdlr = logging.StreamHandler(sys.stdout)

formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')

hdlr.setFormatter(formatter)

logger.addHandler(hdlr)

logger.setLevel(logging.DEBUG)

global witchTrain # 所属当前页面列车的序列号

witchTrain = 0

# 初始化数据库

conn = sqlite3.connect('trainsInfo.sqlite')

c = conn.cursor()

def getScheduleInfo(i,e):

global witchTrain # 所属当前页面列车的序列号

td = pq(e)('td')

if td.eq(0).text() in ('No.',""):

witchTrain += 1

return

# 解析异常处理

if len(td) == 2:

logger.error("%s:%s"%(td.text().encode('gb18030'),len(td)))

return

# 停车时间

stopTime = datetime.strptime("00:00",'%H:%M')

try:

stopTime = datetime.strptime(td.eq(5).text(),'%H:%M')

except Exception:

#print 'stop time parse error:%s:%s'%(td.eq(5).text(),td.eq(0).text())

None

# 开车时间

startTime = datetime.strptime("00:00",'%H:%M')

try:

startTime = datetime.strptime(td.eq(6).text(),'%H:%M')

except Exception:

#print 'start time parse error:%s:%s'%(td.eq(6).text(),td.eq(0).text())

None

# 里程

range = int(td.eq(7).text()[:-2])

# 硬座

hardSeatPrice=0.0

if td.eq(8) and len(td.eq(8).text()) > 1:

hardSeatPrice=td.eq(8).text()[:-1]

# 硬卧中铺

hardBerthPrice=0.0

if td.eq(9) and len(td.eq(9).text()) > 1 and td.eq(9).text()[:-1] <> "-":

hardBerthPrice=td.eq(9).text()[:-1]

# 软座

softSeatPrice=0.0

if td.eq(10) and len(td.eq(10).text()) > 1:

softSeatPrice=td.eq(10).text()[:-1]

# 软卧下铺

softBerthPrice='0'

if td.eq(11) and len(td.eq(11).text()) > 1:

softBerth=td.eq(11).text()[:-1]

return [[witchTrain,

int(td.eq(0).text()),td.eq(1).text(),

td.eq(4).text(),stopTime,

startTime,range,

hardSeatPrice,hardBerthPrice,

softSeatPrice,softBerthPrice,]]

def insertTrainSchedule(trainInfo,scheduleData,cursor):

scheduleData[0]=trainInfo

try:

cursor.execute('insert into trains_schedule values(?,?,?,?,?,?,?,?,?,?,?)',scheduleData)

except Exception,e:

logger.error("%s %s"%(e,scheduleData[0]))

# 获取页面内的车次

def get561x(pageFilePath):

# 列车途经站点

trainSchedule = pq(filename=pageFilePath)('body center

div.ResultContent div.ResultContentLeft div.ResultContentLeftContent\

div.ResultTrainCodeContent table').eq(2)('tr')

trainSchedules = trainSchedule.map(getScheduleInfo)

global witchTrain

witchTrain = 0

for oneTrainSchedule in trainSchedules:

insertTrainSchedule(pageFilePath[:-5],oneTrainSchedule,c)

conn.commit()

logger.info("start get data...")

try:

for oneHtmlFile in ['5611.html','5612.html',]:

get561x(oneHtmlFile)

except Exception,e:

logger.error(e)

logger.info("finish...")

导出sqlite数据文件:4,880,384 bytes