importrequestsfrom lxml importetreeimporttimeimportrandomimportcsvdeftest_ip(ip_address):'''测试ip是否可用

:param ip_address: 代理ip'''url= 'http://icanhazip.com/'headers={#headers 头部文件

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0',

}

ip_pool=[]for ip_test inip_address:#print(ip_test)

try:

response= requests.get(url=url,headers=headers,proxies=ip_test,timeout=5)if response.status_code == '200':

ip_pool.append(ip_test)

time.sleep(random.randint(2,8))exceptException as e:pass

print(ip_pool)

files_save(ip_pool)deffiles_save(ip_list):'''将可用代理ip保存

:param ip_list:代理ip

:return:'''with open('./代理ip.csv','a+',encoding='utf-8')as f:

write=csv.writer(f)

write.writerow(ip_list)pass

defget_page_data(nums):'''获取西刺代理的页面信息

:return:'''ip_list=[]

headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:74.0) Gecko/20100101 Firefox/74.0',

}for i in range(1,nums+1):

url= "https://www.xicidaili.com/nn/{}".format(i)

response= requests.request('get',url=url,headers=headers)

page_data=etree.HTML(response.text)#获取https信息

#https_infos = page_data.xpath(".//tr[@class='odd']")

#获取http信息

#http_infos = page_data.xpath(".//tr[@class='']")

page_infos= page_data.xpath(".//tr[@class='odd']|.//tr[@class='']")for info inpage_infos:

ip_dict={}

ip_address= info.xpath(".//td[2]/text()")[0]

ip_port= info.xpath(".//td[3]/text()")[0]

ip_type= info.xpath(".//td[6]/text()")[0].lower()

ip_dict[ip_type]= ip_type+'://'+ip_address+':'+ip_port

ip_list.append(ip_dict)#print(ip_list)

test_ip(ip_list)pass

pass

if __name__ == '__main__':'''爬取代理ip时应注意

需要测试此ip是否可用

爬取速度

分析：

url信息

页面 url

1 https://www.xicidaili.com/nn/

2 https://www.xicidaili.com/nn/2

3 https://www.xicidaili.com/nn/3'''

#nums = int(input("请输入爬取页数>>"))

nums = 2get_page_data(nums)