1 import requests
 2 from lxml import etree
 3
 4
 5 # 将可以使用的代理IP的信息存储到文件
 6 def write_proxy(proxies):
 7     print(proxies)
 8     for proxy in proxies:
 9         with open("ip_proxy.txt", 'a+') as f:
10             print("正在写入：", proxy)
11             f.write(proxy + '\n')
12     print("录入完成！！！")
13
14
15 # 解析网页，并得到网页中的代理IP
16 def get_proxy(html):
17     # 对获取的页面进行解析
18     selector = etree.HTML(html)
19     # print(selector.xpath("//title/text()"))
20     proxies = []
21     # 信息提取
22     for each in selector.xpath('//table[@id="ip_list"]/tr')[1:]:
23         # ip.append(each[0])
24         ip = each.xpath("./td[2]/text()")[0]
25         port = each.xpath("./td[3]/text()")[0]
26         proxy = ip + ":" + port
27
28         proxies.append(proxy)
29     print(len(proxies))
30     test_proxies(proxies)
31
32
33 # 验证已得到IP的可用性，本段代码通过访问百度网址，返回的response状态码判断（是否可用）。
34 def test_proxies(proxies):
35     proxies = proxies
36     url = "http://www.baidu.com/"
37     header = {
38         "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
39         }
40     normal_proxies = []
41     count = 1
42     for proxy in proxies:
43         print("第%s个。。" % count)
44         count += 1
45         try:
46             response = requests.get(url, headers=header, proxies={"http": proxy}, timeout=1)
47             if response.status_code == 200:
48                 print("该代理IP可用：", proxy)
49                 normal_proxies.append(proxy)
50             else:
51                 print("该代理IP不可用：", proxy)
52         except Exception:
53             print("该代理IP无效：", proxy)
54             pass
55     # print(normal_proxies)
56     write_proxy(normal_proxies)
57
58
59 def get_html(url):
60     header = {
61         "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
62     }
63     response = requests.get(url,headers=header,)
64     # print(response.text)
65     get_proxy(response.text)
66
67
68 if __name__ == "__main__":
69     base_url = "http://www.xicidaili.com/nn/%s/"
70     # 爬取3页数据
71     for i in range(1, 4):
72         url = base_url % i
73         get_html(url)