1 #coding=utf-8
 2 from selenium import webdriver
 3 import cx_Oracle                                     #引用模块cx_Oracle
 4 import pandas as pd
 5 import os
 6 #若数据为空，则重复采集
 7 def paqu(uurl):
 8     browser.get(uurl)
 9     dfs = pd.read_html(browser.page_source, header=0)[0]
10     if len(dfs)>1:
11         return dfs
12     else:
13         return paqu(uurl)
14 os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.UTF8'      #防止oracle数据乱码
15 conn=cx_Oracle.connect('Xho', 'sys', 'localhost:1521/orcl')    #连接数据库
16 cursor=conn.cursor()#获取cursor游标
17
18 #声明浏览器对象
19 browser = webdriver.Chrome('C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe')
20 base_url='https://www.aqistudy.cn/historydata/daydata.php?city='
21 city=['石家庄','保定','唐山','邯郸','邢台','沧州','衡水','廊坊','承德','秦皇岛','张家口']
22
23 year=[201301,201401,201501,201601,201701,201801]
24 list_data=[]
25 list_row=[]
26 for p in range(len(city)):
27     for i in year:
28         for j in range(12):
29             num = i + j
30             if num > 201311 and num < 201809:
31                 uurl=base_url+city[p]+'&month='+str(num)
32                 dfs=paqu(uurl)
33                 #time.sleep(1)
34                 dfs['city']=city[p]
35                 for s in range(0, len(dfs)):
36                     date = dfs.iloc[s, 0]
37                     list_row.append(date)
38                     aqi = dfs.iloc[s, 1]
39                     list_row.append(aqi)
40                     grade = dfs.iloc[s, 2]
41                     list_row.append(grade)
42                     pm25 = dfs.iloc[s, 3]
43                     list_row.append(pm25)
44                     pm10 = dfs.iloc[s, 4]
45                     list_row.append(pm10)
46                     so2 = dfs.iloc[s, 5]
47                     list_row.append(so2)
48                     co = dfs.iloc[s, 6]
49                     list_row.append(co)
50                     no2 = dfs.iloc[s, 7]
51                     list_row.append(no2)
52                     o3 = dfs.iloc[s, 8]
53                     list_row.append(o3)
54                     chsh=dfs.iloc[s,9]
55                     list_row.append(chsh)
56
57                     list_data.append(list_row)
58                     list_row=[]
59     for n in range(len(list_data)):
60         sql = 'insert into AIR_ZL (A_DATE,AQI,GRADE,PM25,PM10,SO2,CO,NO2,O3_8H,CITY) VALUES (:A_DATE,:AQI,:GRADE,:PM25,:PM10,:SO2,:CO,:NO2,:O3_8H,:CITY)'
61         x = cursor.execute(sql, (list_data[n][0], float(list_data[n][1]), list_data[n][2],float(list_data[n][3]),float(list_data[n][4]),float(list_data[n][5]),float(list_data[n][6]),float(list_data[n][7]),float(list_data[n][8]),list_data[n][9]))
62         conn.commit();
63     list_data=[]
64 cursor.close()  # 关闭cursor
65 conn.close()  # 关闭连接
66 browser.close()