前言：在上一篇文章中，我们介绍了在http://PM2.5.in这个网站采集空气质量的数据，本篇文章是对其产生的一些问题的另一种解决方案，提供更加权威的数据采集。

技术框架：selenium、json、etree

这里的selenium是一种自动化测试的工具，它可以帮助我们模拟浏览器打开网页并获取网页数据，本文之所以选择这种方式进行，是因为以requests方式直接请求无法获取到正确的数据，这个网页的数据是动态加载，需要用户执行点击操作才会被请求

我们还是按照常规套路来分析下这个网站，打开F12,看下这个网站的数据请求

可以发现这个网站的数据的请求接口，但当我们直接用requests去请求这个接口，会发现无法获取正确的数据，原因是这个网站采用了MmEwMD这个值进行了反爬虫，这个是一个比较常见的反爬虫措施，他这个值是在发起请求时动态生成的，最简单的解决这个问题的办法就是采用selenium之类的模拟浏览器方法进行请求，这样的话，发出的请求也会自动带上这个参数

请求的代码如下图所示

driverPath = 'browser\\chromedriver.exe'

options = webdriver.ChromeOptions()

options.add_experimental_option("excludeSwitches", ["enable-automation"])

options.add_experimental_option('useAutomationExtension', False)

# options.add_argument(('--proxy-server=http://' + ip))

browser = webdriver.Chrome(options=options, executable_path=driverPath)

browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {

"source": """

Object.defineProperty(navigator, 'webdriver', {

get: () => undefined

})

"""

})

browser.get(self.url)

html = browser.page_source

browser.quit()

# print(html)

reponse = etree.HTML(html)

data = reponse.xpath('//body/text()')[0]

json_data = json.loads(data)

我们通过调用谷歌浏览器直接请求对应的页面，获取到数据后，关闭浏览器，通过etree解析网页结果，通过观察发现，我们获取到的数据是json数组，因此我们使用json解析数据，然后将对应的数据存储到数据库

result_list = json_data['data']['hour']

print(result_list)

for result in result_list:

item = dict()

item['affect'] = result['AFFECTINFO']

item['action'] = result['SUGGEST']

if('AQIPRIMPOLLUTE' in result):

item['primary_pollutant'] = result['AQIPRIMPOLLUTE']

else:

item['primary_pollutant'] = '无'

item['AQI'] = result['AQI']

item['PM2.5/1h'] = result['PM25']

item['PM10/1h'] = result['PM10']

item['CO/1h'] = result['CO']

item['NO2/1h'] = result['NO2']

item['O3/1h'] = result['O3']

item['O3/8h'] = result['O3_2']

item['SO2/1h'] = result['SO2']

item['city_name'] = result['POINTNAME']

item['level'] = result['CODEAQILEVEL']+'('+result['AQILEVELNAME']+')'

item['live_data_time'] = result['MONITORTIME']

item['live_data_time'] = datetime.datetime.strptime(item['live_data_time'], "%Y年%m月%d日%H")

update_time = item['live_data_time'].strftime('%Y-%m-%d %H:%M:%S')

item['live_data_unit'] = 'μg/m3(CO为mg/m3)'

if(item['city_name'] in city_config):

self.save_mysql(item)

success_count = success_count+1

log_text = '采集的城市:{},采集的结果:{}'.format(item['city_name'],'成功')

self.save_log({'log_type':'0','log_text':log_text})

self.save_log({'log_type':'3','log_text':log_text})

self.update_spider_time(update_time)

# 存储运行日志

def save_log(self,item):

sql = 'INSERT INTO log(log_text,log_type,created_time) VALUES (%s,%s,%s)'

values = [item['log_text'],item['log_type'],datetime.datetime.now()]

self.cursor.execute(sql,values)

self.conn.commit()

def save_mysql(self,item):

# 查询数据库已存在的数据

query_sql = 'select count(1) as count from kongqizhiliang where city_name= %s and live_data_time = %s'

values = [item['city_name'],item['live_data_time']]

self.cursor.execute(query_sql,values)

data = self.cursor.fetchone()

# 如果不存在同一城市同一时刻更新的数据，则新增

if(data['count'] == 0):

sql = ("INSERT kongqizhiliang(city_name,level,live_data_time,live_data_unit,AQI,PM25_1h,PM10_1h,CO_1h"

",NO2_1h,O3_1h,O3_8h,SO2_1h,affect,primary_pollutant,action"

") VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)")

values =[item['city_name'],item['level'],item['live_data_time'],item['live_data_unit'],item['AQI']

,item['PM2.5/1h'],item['PM10/1h'],item['CO/1h'],item['NO2/1h'],item['O3/1h'],item['O3/8h']

,item['SO2/1h'],item['affect'],item['primary_pollutant'],item['action']]

self.cursor.execute(sql,values)

self.conn.commit()

其实当初这个反爬虫措施也困扰了我一段时间的，我这里采用的是最简单的方法解决，虽然效率不高，但能解决我的需求

完整代码如下：其中部分代码是可以不需要的，必须redis和config那个，你们自己改一下，不会的可以问我，这个是当时给别人毕设做的，还有其他功能，所以会有一些其他的

"""

采集空气质量的数据

目标网站：http://sthjt.hubei.gov.cn/hjsj/

"""

import requests

from lxml import etree

import re

from xpinyin import Pinyin

import pymysql

import sys

from settings.config import *

from utils import RedisUtil

import datetime

import json

from selenium import webdriver

class kongqizhiliang:

DEFAULT_REQUEST_HEADERS = {

'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

'Accept-Language': 'en',

'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'

}

url = 'http://sthjt.hubei.gov.cn/wcmapi/service/aqi.xhtml'

redis_key = 'kongqi:config_city'

update_time = 'kongqi:update_time'

# 汉字转拼音

pinyin = Pinyin()

def __init__(self):

self.conn = pymysql.connect(host=host, port=port, user=user, passwd=passwd, db=db, charset=charset)

self.cursor = self.conn.cursor(cursor=pymysql.cursors.DictCursor)

# 将城市名转化为code

def get_code(self,city_name):

return self.pinyin.get_pinyin(city_name, '' )

def get_city_config(self):

redis_util = RedisUtil.get_redis()

city_list = redis_util.list_get_range(self.redis_key)

return city_list

def update_spider_time(self,update_time):

redis_util = RedisUtil.get_redis()

redis_util.str_set(self.update_time,update_time)

def get_data(self):

city_config = self.get_city_config()

log_text = '采集开始,准备采集的城市:{},计划采集的数据量:{}'.format(city_config,len(city_config))

self.save_log({'log_type':'2','log_text':log_text})

success_count = 0

update_time = ''

driverPath = 'browser\\chromedriver.exe'

options = webdriver.ChromeOptions()

options.add_experimental_option("excludeSwitches", ["enable-automation"])

options.add_experimental_option('useAutomationExtension', False)

# options.add_argument(('--proxy-server=http://' + ip))

browser = webdriver.Chrome(options=options, executable_path=driverPath)

browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {

"source": """

Object.defineProperty(navigator, 'webdriver', {

get: () => undefined

})

"""

})

browser.get(self.url)

html = browser.page_source

browser.quit()

# print(html)

reponse = etree.HTML(html)

data = reponse.xpath('//body/text()')[0]

json_data = json.loads(data)

# print(json_data)

result_list = json_data['data']['hour']

print(result_list)

for result in result_list:

item = dict()

item['affect'] = result['AFFECTINFO']

item['action'] = result['SUGGEST']

if('AQIPRIMPOLLUTE' in result):

item['primary_pollutant'] = result['AQIPRIMPOLLUTE']

else:

item['primary_pollutant'] = '无'

item['AQI'] = result['AQI']

item['PM2.5/1h'] = result['PM25']

item['PM10/1h'] = result['PM10']

item['CO/1h'] = result['CO']

item['NO2/1h'] = result['NO2']

item['O3/1h'] = result['O3']

item['O3/8h'] = result['O3_2']

item['SO2/1h'] = result['SO2']

item['city_name'] = result['POINTNAME']

item['level'] = result['CODEAQILEVEL']+'('+result['AQILEVELNAME']+')'

item['live_data_time'] = result['MONITORTIME']

item['live_data_time'] = datetime.datetime.strptime(item['live_data_time'], "%Y年%m月%d日%H")

update_time = item['live_data_time'].strftime('%Y-%m-%d %H:%M:%S')

item['live_data_unit'] = 'μg/m3(CO为mg/m3)'

if(item['city_name'] in city_config):

self.save_mysql(item)

success_count = success_count+1

log_text = '采集的城市:{},采集的结果:{}'.format(item['city_name'],'成功')

self.save_log({'log_type':'0','log_text':log_text})

self.save_log({'log_type':'3','log_text':log_text})

self.update_spider_time(update_time)

# 存储运行日志

def save_log(self,item):

sql = 'INSERT INTO log(log_text,log_type,created_time) VALUES (%s,%s,%s)'

values = [item['log_text'],item['log_type'],datetime.datetime.now()]

self.cursor.execute(sql,values)

self.conn.commit()

def save_mysql(self,item):

# 查询数据库已存在的数据

query_sql = 'select count(1) as count from kongqizhiliang where city_name= %s and live_data_time = %s'

values = [item['city_name'],item['live_data_time']]

self.cursor.execute(query_sql,values)

data = self.cursor.fetchone()

# 如果不存在同一城市同一时刻更新的数据，则新增

if(data['count'] == 0):

sql = ("INSERT kongqizhiliang(city_name,level,live_data_time,live_data_unit,AQI,PM25_1h,PM10_1h,CO_1h"

",NO2_1h,O3_1h,O3_8h,SO2_1h,affect,primary_pollutant,action"

") VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)")

values =[item['city_name'],item['level'],item['live_data_time'],item['live_data_unit'],item['AQI']

,item['PM2.5/1h'],item['PM10/1h'],item['CO/1h'],item['NO2/1h'],item['O3/1h'],item['O3/8h']

,item['SO2/1h'],item['affect'],item['primary_pollutant'],item['action']]

self.cursor.execute(sql,values)

self.conn.commit()

if __name__ == "__main__":

app = kongqizhiliang()

app.get_data()

本文首发于爬虫：利用selenium采集某某环境网站的空气质量数据​www.bizhibihui.com