'''
爬取豆瓣TOP250电影信息主页:第一页:https://movie.douban.com/top250?start=0&filter=第二页：https://movie.douban.com/top250?start=25&filter=第三页：https://movie.douban.com/top250?start=50&filter=第十页：https://movie.douban.com/top250?start=225&filter=GETUser-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36re正则:# 电影详情页url、图片链接、电影名称、电影评分、导演、主演、电影上映时间、评价人数、简介<div class="item">.*?href="(.*?)">.*?src="(.*?)" class="">.*?<span class="title">(.*?)</span>.*?<div class="bd">.*?导演:(.*?)<br>(.*?)</p>.*?<span class="rating_num".*?>(.*?)</span>.*?<span>(.*?)人评价<span class="inq">(.*?)</span>
'''
import requests
import reheaders = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36'
}
base_url = 'https://movie.douban.com/top250?start={}&filter='n = 0
for line in range(10):print()url = base_url.format(n)print(type(n))n += 25print(url)
# 1、往豆瓣TOP250发送请求获取响应数据
response = requests.get(url, headers=headers)# print(response.text)# 2、通过正则解析提取数据
# 电影详情页url、图片链接、电影名称、电影评分、评价人数
movie_content_list = re.findall(# 正则规则# 电影评分、评价人数、'<div class="item">.*?href="(.*?)">.*?src="(.*?)" class="">.*?<span class="title">(.*?)</span>.*?<div class="bd">.*?导演:(.*?)<br>(.*?)</p>.*?<span class="rating_num".*?>(.*?)</span>.*?<span>(.*?)人评价.*?<span class="inq">(.*?)</span>',# 解析文本
    response.text,# 匹配模式
    re.S)for movie_content in movie_content_list:# 解压赋值每一部电影detail_url, movie_jpg, name, daoyan, timer, point, num, desc = movie_contentdata = f'电影名称:{name},详情页url:{detail_url}, 图片url:{movie_jpg},导演：{daoyan},上映时间：{timer}, 评分: {point}, 评价人数: {num}, 简介：{desc} \n'print(data)# 3、保存数据，把电影信息写入文件中with open('豆瓣top250.txt', 'a', encoding='utf-8') as f:f.write(data)

'''
post请求访问github
请求url:https://github.com/session
请求方式：POST
请求头：# 上一次请求从哪里来Referer:https://github.com/loginUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36
请求体：只有POST请求才会有请求体。commit: 登入utf8: ✓authenticity_token: 1ZoPGWJBCBMW/IzHozPRLB1UJBfw0/SnHtGeqbKULG1Xb1YPxFhKmKKFPs9meYV0IdjGKnXYqTH9nASzPNYOnw==login: shendongnianpassword: 1111webauthn-support: supported'''
import requests
import re# 一、访问login页面获取token信息
'''
请求url:https://github.com/login
请求方式：GET
响应头：Set-Cookie：
请求头：Cookie:User-Agent:
'''
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'
}response = requests.get(url='https://github.com/login', headers = headers)
# print(response.text)
# 把login页返回的cookies信息转换成字典
login_cookies = response.cookies.get_dict()authenticity_token = re.findall('<input type="hidden" name="authenticity_token" value="(.*?)" />',response.text,re.S)[0]print(authenticity_token)# 二、往sessionurl发送POST请求
'''
post请求登录github
请求url:https://github.com/session请求方式：POST请求头：# 上一次请求从哪里来Referer:https://github.com/loginCookie:...User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36请求体：只有POST请求才会有请求体。commit: 登入utf8: ✓authenticity_token:    1ZoPGWJBCBMW/IzHozPRLB1UJBfw0/SnHtGeqbKULG1Xb1YPxFhKmKKFPs9meYV0IdjGKnXYqTH9nASzPNYOnw==login: shendongnianpassword: 1111webauthn-support: supported'''
# 拼接头信息
headers2 = {'Referer':'https://github.com/login','User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'
}# cookies = {#     'Cookies':''
# }# 拼接请求体信息
from_data = {"commit":"登入","utf8": "✓","authenticity_token":authenticity_token,"login":"tankjam","password":"kermit46709394","webauthn-support":"unsupported"
}# 往session地址发送post请求
# 携带请求头、请求体、login页的cookie信息

response2 = requests.post(url='https://github.com/session',data=from_data,headers = headers2,cookies=login_cookies)
print(response2.status_code)
# print(response2.text)
with open('github.html','w',encoding='utf-8') as f:f.write(response2.text)

'''
post请求访问github
请求url:https://github.com/session
请求方式：POST
请求头：# 上一次请求从哪里来Referer:https://github.com/loginUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36
请求体：只有POST请求才会有请求体。commit: 登入utf8: ✓authenticity_token: 1ZoPGWJBCBMW/IzHozPRLB1UJBfw0/SnHtGeqbKULG1Xb1YPxFhKmKKFPs9meYV0IdjGKnXYqTH9nASzPNYOnw==login: shendongnianpassword: 1111webauthn-support: supported'''
import requests
import re# 一、访问login页面获取token信息
'''
请求url:https://github.com/login
请求方式：GET
响应头：Set-Cookie：
请求头：Cookie:User-Agent:
'''
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'
}response = requests.get(url='https://github.com/login', headers = headers)
# print(response.text)
# 把login页返回的cookies信息转换成字典
login_cookies = response.cookies.get_dict()authenticity_token = re.findall('<input type="hidden" name="authenticity_token" value="(.*?)" />',response.text,re.S)[0]print(authenticity_token)# 二、往sessionurl发送POST请求
'''
post请求登录github
请求url:https://github.com/session请求方式：POST请求头：# 上一次请求从哪里来Referer:https://github.com/loginCookie:...User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36请求体：只有POST请求才会有请求体。commit: 登入utf8: ✓authenticity_token:    1ZoPGWJBCBMW/IzHozPRLB1UJBfw0/SnHtGeqbKULG1Xb1YPxFhKmKKFPs9meYV0IdjGKnXYqTH9nASzPNYOnw==login: shendongnianpassword: 1111webauthn-support: supported'''
# 拼接头信息
headers2 = {'Referer':'https://github.com/login','User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'
}# cookies = {#     'Cookies':''
# }# 拼接请求体信息
from_data = {"commit":"登入","utf8": "✓","authenticity_token":authenticity_token,"login":"tankjam","password":"kermit46709394","webauthn-support":"unsupported"
}# 往session地址发送post请求
# 携带请求头、请求体、login页的cookie信息

response2 = requests.post(url='https://github.com/session',data=from_data,headers = headers2,cookies=login_cookies)
print(response2.status_code)
# print(response2.text)
with open('github.html','w',encoding='utf-8') as f:f.write(response2.text)

import requestshttps = http + ssl(携带证书)#证书验证(大部分网站都是https)
import requests
# 如果是ssl请求,首先检查证书是否合法,不合法则报错,程序终端
response = requests.get('https://www.xiaohuar.com')
print(response.status_code)# 改进1:去掉报错,但是会报警告
# import requests
# response = requests.get('https://www.xiaohuar.com', verify=False)  # True携带证书 False不携带证书
# # 不验证证书,报警告,返回200
# print(response.status_code)# 改进2:去掉报错,并且去掉警报信息
import requests
import urllib3
urllib3.disable_warnings()  # 关闭警告
response = requests.get('https://www.xiaohuar.com', verify=False)
print(response.status_code)# 改进3:加上证书
# 很多网站都是https,但是不用证书也可以访问,大多数情况都是可以携带也可以不携带证书
# 知乎\百度等都是可带可不带
# 有硬性要求的,则必须带，比如对于定向的用户,拿到证书后才有权限访问某个特定网站
import requests
import urllib3
# urllib3.disable_warnings()  # 关闭警告
# 伪代码
response = requests.get('https://www.xiaohuar.com',# verify=False,# /path/server.crt证书的存放目录，/path/keycert=('/path/server.crt', '/path/key'))
print(response.status_code)'''
超时设置
'''超时设置
两种超时:float or tuple
timeout=0.1  # 代表接收数据的超时时间
timeout=(0.1,0.2)  # 0.1代表链接超时  0.2代表接收数据的超时时间import requestsresponse = requests.get('https://www.baidu.com',timeout=0.0001)print()代理设置:先发送请求给代理,然后由代理帮忙发送(封ip是常见的事情)
import requests
proxies={# 带用户名密码的代理,@符号前是用户名与密码'http':'http://tank:123@localhost:9527','http':'http://localhost:9527','https':'https://localhost:9527',
}
response=requests.get('https://www.12306.cn',proxies=proxies)print(response.status_code)'''
爬取西刺免费代理：1.访问西刺免费代理页面2.通过re模块解析并提取所有代理3.通过ip测试网站对爬取的代理进行测试4.若test_ip函数抛出异常代表代理作废，否则代理有效5.利用有效的代理进行代理测试<tr class="odd"><td class="country"><img src="//fs.xicidaili.com/images/flag/cn.png" alt="Cn"></td><td>112.85.131.99</td><td>9999</td><td><a href="/2019-05-09/jiangsu">江苏南通</a></td><td class="country">高匿</td><td>HTTPS</td><td class="country"><div title="0.144秒" class="bar"><div class="bar_inner fast" style="width:88%"></div></div></td><td class="country"><div title="0.028秒" class="bar"><div class="bar_inner fast" style="width:97%"></div></div></td><td>6天</td><td>19-05-16 11:20</td></tr>
re:<tr class="odd">(.*?)</td>.*?<td>(.*?)</td>'''
import requests
import re
import timeHEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
}def get_index(url):time.sleep(1)response = requests.get(url, headers=HEADERS)return responsedef parse_index(text):ip_list = re.findall('<tr class="odd">.*?<td>(.*?)</td>.*?<td>(.*?)</td>', text, re.S)for ip_port in ip_list:ip = ':'.join(ip_port)yield ipdef test_ip(ip):print('测试ip: %s' % ip)try:proxies = {'https': ip}# ip测试网站ip_url = 'https://www.ipip.net/'# 使用有效与无效的代理对ip测试站点进行访问，若返回的结果为200则代表当前测试ip正常response = requests.get(ip_url, headers=HEADERS, proxies=proxies, timeout=1)if response.status_code == 200:print(f'有用的ip:{ip}')return ip# 若ip代理无效则抛出异常except Exception as e:print(e)# 使用代理爬取nba
def spider_nba(good_ip):url = 'https://china.nba.com/'proxies = {'https': good_ip}response = requests.get(url, headers=HEADERS, proxies=proxies)print(response.status_code)print(response.text)if __name__ == '__main__':base_url = 'https://www.xicidaili.com/nn/{}'for line in range(1, 3677):ip_url = base_url.format(line)response = get_index(ip_url)# 解析西刺代理获取每一个ip列表ip_list = parse_index(response.text)# 循环每一个ipfor ip in ip_list:# print(ip)# 对爬取下来的ip 进行测试good_ip = test_ip(ip)if good_ip:# 真是代理，开始测试
                spider_nba(good_ip)# 认证设置
'''
登录网站时，会弹出一个框，要求你输入用户名与密码（类似于alert），此时无法进入html页面，待授权通过后才能进入html页面。Requests模块为我们提供了多种身份认证方式，包括基本身份认证等...其原理指的是通过输入用户名与密码获取用户的凭证来识别用户，然后通过token对用户进行授权。
基本身份认证:HTTP Basic Auth是HTTP1.0提出的认证方式。客户端对于每一个realm，通过提供用户名和密码来进行认证的方式当认证失败时，服务器收到客户端请求，返回401。'''
import requests
# 通过访问github的api来测试
url = 'https://api.github.com/user'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
}# 测试1，失败返回401
response = requests.get(url, headers=HEADERS)
print(response.status_code)  # 401
print(response.text)
'''
打印结果:{"message": "Requires authentication","documentation_url": "https://developer.github.com/v3/users/#get-the-authenticated-user"}
'''# 测试2，通过requests.auth内的HTTPBasicAuth进行认证，认证成功返回用户信息
from requests.auth import HTTPBasicAuth
response = requests.get(url, headers=HEADERS, auth=HTTPBasicAuth('tankjam', 'kermit46709394'))
print(response.text)# 测试3，通过requests.get请求内的auth参数默认就是HTTPBasicAuth，认证成功返回用户信息
# response = requests.get(url, headers=HEADERS, auth=('tankjam', 'kermit46709394'))
# print(response.text)
'''
打印结果:{"login": "TankJam","id": 38001458,"node_id": "MDQ6VXNlcjM4MDAxNDU4","avatar_url": "https://avatars2.githubusercontent.com/u/38001458?v=4","gravatar_id": "","url": "https://api.github.com/users/TankJam","html_url": "https://github.com/TankJam","followers_url": "https://api.github.com/users/TankJam/followers","following_url": "https://api.github.com/users/TankJam/following{/other_user}","gists_url": "https://api.github.com/users/TankJam/gists{/gist_id}","starred_url": "https://api.github.com/users/TankJam/starred{/owner}{/repo}","subscriptions_url": "https://api.github.com/users/TankJam/subscriptions","organizations_url": "https://api.github.com/users/TankJam/orgs","repos_url": "https://api.github.com/users/TankJam/repos","events_url": "https://api.github.com/users/TankJam/events{/privacy}","received_events_url": "https://api.github.com/users/TankJam/received_events","type": "User","site_admin": false,"name": "kermit","company": null,"blog": "","location": null,"email": null,"hireable": null,"bio": null,"public_repos": 6,"public_gists": 0,"followers": 0,"following": 0,"created_at": "2018-04-02T09:39:33Z","updated_at": "2019-05-14T07:47:20Z","private_gists": 0,"total_private_repos": 1,"owned_private_repos": 1,"disk_usage": 8183,"collaborators": 0,"two_factor_authentication": false,"plan": {"name": "free","space": 976562499,"collaborators": 0,"private_repos": 10000}}
'''上传文件
import requests# 上传文本文件
files1 = {'file': open('user.txt', 'rb')}
# files参数是POST请求固定参数
response = requests.post('http://httpbin.org/post', files=files1)
print(response.status_code)  # 200
print(response.text)  # 200# 上传图片文件
files2 = {'jpg': open('一拳超人.jpg', 'rb')}
response = requests.post('http://httpbin.org/post', files=files2)
print(response.status_code)  # 200
print(response.text)  # 200# 上传视频文件
files3 = {'movie': open('love_for_GD.mp4', 'rb')}response = requests.post('http://httpbin.org/post', files=files3)
print(response.status_code)  # 200
print(response.text)  # 200

'''
selenium模块讲解
一、什么是selenium?最初是一个自动化测试工具。可以使用它帮我们驱动浏览器自动去执行某些自定义好的操作。例如在页面中执行JS代码、跳过登录验证。（selenium最初是一个自动化测试工具，而爬虫中使用它主要是为了解决requests无法执行javaScript代码的问题。selenium可以驱动浏览器自动执行自定义好的逻辑代码，也就是可以通过代码完全模拟成人类使用浏览器自动访问目标站点并操作，那我们也可以拿它来做爬虫。selenium本质上是通过驱动浏览器，完全模拟浏览器的操作，比如跳转、输入、点击、下拉等...进而拿到网页渲染之后的结果，可支持多种浏览器。）二、为什么要使用selenium？1、优点：要使用requests模块登录需要分析大量的复杂通信流程，使用selenium可以轻松跳过登录验证。2、缺点：浏览器会加载css、js、图片、视频...数据，爬虫效率相比requests模块要低。三、如何使用？下载selenium模块：pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple selenium下载浏览器驱动：http://npm.taobao.org/mirrors/chromedriver/2.38/
'''# selenium之第一次测试
from selenium import webdriver  # 用来驱动浏览器的# 调用得到一个动作链对象，破解华东验证码的时候用的，可以拖动图片
from selenium.webdriver import ActionChains# 按照什么方式查找属性，By.ID,By.CSS_SELECTOR，By.Class
from selenium.webdriver.common.by import Byfrom selenium.webdriver.common.keys import Keys  # 键盘按键操作# 和下面WebDriverWait一起用的,EC 是expected_conditions的别名
from selenium.webdriver.support import expected_conditions as EC# 等待页面加载某些元素
from selenium.webdriver.support.wait import WebDriverWaitimport time# 通过谷歌浏览器驱动打开谷歌浏览器
# chrome = webdriver.Chrome(r'chromedriver.exe的绝对路径')
chrome = webdriver.Chrome(r'D:\Python\chromedriver_win32\chromedriver')  # 括号内输入chromedriver.exe的绝对路径
'''
# chromedriver.exe存放于python解释器的Scripts文件夹中# chrome是一个驱动对象
chrome = webdriver.Chrome()
'''
'''
实例1
'''# 若try出现异常
try:# 往tank博客主页发送get请求# chrome.get('https://www.cnblogs.com/kermitjam')# 参数1：驱动对象  参数2：等待时间wait = WebDriverWait(chrome, 10)# 1、访问百度chrome.get('https://www.baidu.com/')# 2、查找input输入框# 调用EC的presence_of_element_located（）input_tag = wait.until(EC.presence_of_element_located((By.ID, "kw")))# 此处可以写一个元组# 参数1：查找属性的方式# 参数2：属性的名字# 3、搜索一拳超人input_tag.send_keys('一拳超人')# 4、按键盘回车键
    input_tag.send_keys(Keys.ENTER)time.sleep(3)# 无论发生什么都会关闭浏览器
finally:# 关闭浏览器
    chrome.close()'''
实例2
'''
try:# 往tank博客主页发送get请求# chrome.get('https://www.cnblogs.com/kermitjam')# 参数1：驱动对象  参数2：等待时间wait = WebDriverWait(chrome, 10)# 1、访问百度chrome.get('https://www.jd.com/')# 2、查找input输入框input_tag = wait.until(EC.presence_of_element_located((By.ID, "key")))# 3、搜索唐诗三百首input_tag.send_keys('唐诗三百首')# 4、根据class属性名称查找标签search_button = wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'button')))# 5、点击搜索按钮
    search_button.click()time.sleep(3)# 无论发生什么都会关闭浏览器
finally:# 关闭浏览器chrome.close()

from selenium import webdriver
import time'''
隐式等待
'''
# 获取驱动对象
driver = webdriver.Chrome(r'D:\Python\chromedriver_win32\chromedriver')# 获取
try:# 显式等待：等待某个元素加载# 参数1：驱动对象  参数2：等待时间# wait = WebDriverWait(chrome, 10)
driver.get('https://china.nba.com')# 隐式等待：等待页面所有元素加载driver.implicitly_wait(10)news_tag = driver.find_element_by_class_name('nav-news')# 获取标签对象print(news_tag)# 获取标签名字print(news_tag.ag_name)time.sleep(10)finally:chrome.close()'''
===============所有方法===================element是查找一个标签elements是查找所有标签1、find_element_by_link_text  通过链接文本去找2、find_element_by_id 通过id去找3、find_element_by_class_name4、find_element_by_partial_link_text5、find_element_by_name6、find_element_by_css_selector7、find_element_by_tag_name
'''
try:# 往百度主页发送请求driver.get('https://www.baidu.com/')driver.implicitly_wait(10)# 1、find_element_by_link_text 通过链接文本去找# 根据登录send_tag = driver.find_element_by_link_text('登录')send_tag.click()# 2、find_element_by_partial_link_text 通过局部文本查找a标签login_button = driver.find_element_by_partial_link_text('登')login_button.click()time.sleep(1)# 3、find_element_by_class_name 根据class属性名查找login_tag = driver.find_element_by_class_name('tang-pass-footerBarULogin')login_tag.click()time.sleep(1)# 4、find_element_by_name  根据name属性查找username = driver.find_element_by_name('userName')username.send_keys('15055303102')time.sleep(1)# 5、find_element_by_id  通过id属性名查找password = driver.find_element_by_id('TANGRAM__PSP_10__password')password.send_keys('shenjin981002')time.sleep(1)# 6、find_element_by_css_selector 根据属性选择器查找# 根据id查找登录按钮login_submit = driver.find_element_by_css_selector('#TANGRAM__PSP_10__submit')# login_submit = driver.find_element_by_css_selector('.pass-button-submit')
    login_submit.click()# 7、find_element_by_tag_name  根据标签名称查找标签div = driver.find_element_by_tag_name('div')print(div.tag_name)time.sleep(10)finally:driver.close()

#爬取代理IPfrom selenium import webdriver
from selenium.webdriver.common.proxy import Proxy
from selenium.webdriver.common.proxy import ProxyType
from selenium.webdriver.common import desired_capabilities
import time
from random import randint
from bs4 import BeautifulSoup
import pymysql'''
PhantomJS常用配置
'''
#增加头信息
headers = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",'Accept-Charset': 'utf-8',"User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64;rv:47.0) Gecko/20100101 Firefox/47.0","Host": "www.kuaidaili.com","Connection": "keep-alive"
}#DesiredCapabilities 手动设置爬取时你所期望的功能配置
for key in headers:webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.{}'.format(key)] = headers[key]
proxy=Proxy({'proxyType':ProxyType.MANUAL,    #manual 手工的，手动的'httpProxy':'181.16.136.6:38319'}
)desired_capabilities = desired_capabilities.DesiredCapabilities.PHANTOMJS.copy()
#把代理ip加入到技能中
proxy.add_to_capabilities(desired_capabilities)#禁止加载图片，我们只需要爬取数据，并不需要加载图片，这样可以加快爬取速度
desired_capabilities["phantomjs.page.settings.loadImages"]=False#启动磁盘缓存
desired_capabilities["phantomjs.page.settings.disk-cache"]=True#将配置好的PhantomJS期望的功能加入到PhantomJS驱动中
browser = webdriver.PhantomJS( desired_capabilities=desired_capabilities )#用来存放爬取的数据
class Item(object):ip=Noneport=Noneptype=Noneclass spyderKuaidaili(object):def __init__(self,url):#连接数据库self.db = pymysql.connect(host='localhost', port=3306,user='root', passwd='a', db='python', charset='utf8')self.cursor=self.db.cursor()self.url=urlself.items=[]try:browser.get(url)for i in range(10):   #循环十次，爬取十页              time.sleep( randint(2,5) )  #爬取太勤快会被封ip的result=browser.page_source   #得到网页源码list1=self.parseHtml(result)  #去解析该网页，得到需要的数据self.items.append(list1)     #得到的数据添加到列表里去#这一页代理获取完成，开始selenium自动点击下一页，首先需得到下一页的标签aTags=browser.find_elements_by_xpath("//div[@id='listnav']/ul/li/a")for aElement in aTags:#如果元素是 invisable 的，则用text取不到，只能用innerHTML取#print( aElement.get_attribute('innerHTML') )if aElement.get_attribute('innerHTML')==str(i+2):#满足条件，则是需要点击的那一页print('点击了第：',str(i+2),'页')aElement.click()        #aElement本身是个a标签，点击则执行href跳转到相应页面
                        browser.switch_to.window(browser.window_handles[0])break                        except Exception as e:print('a==========%s' % e )print("爬取完毕...")self.cursor.close()self.db.close()browser.close()browser.quit()print("已关闭浏览器...")def parseHtml(self,content):items=[]soup=BeautifulSoup(content,'lxml')divTag=soup.find('div',attrs={'id':'list'})tags=divTag.table.tbody.find_all('tr')for tag in tags:item=Item()item.ip=tag.find('td',attrs={'data-title':'IP'}).get_text().strip()item.port=tag.find('td',attrs={'data-title':'PORT'}).get_text().strip()item.ptype=tag.find('td',attrs={'data-title':'类型'}).get_text().strip()items.append(item)   #将得到的item对象存入列表print( item.ip,' ',item.port,' ',item.ptype )#将一页的数据一起存入数据库
        self.insert(items)return itemsdef insert(self,items):  #[Item,Item...Item]print("开始插入数据库")sql='insert into kuaidaili(kip,kport,kptype) values(%s,%s,%s)'params=[]for item in items:params.append( ( item.ip,item.port,item.ptype ) )try:#进行批量插入
            self.cursor.executemany(sql,params)self.db.commit()except Exception as e:print("插入数据库异常")self.db.rollback()if __name__=='__main__':url='https://www.kuaidaili.com/free/inha/'spyderKuaidaili(url)

import requests
import pymysql#将上面存入数据库的代理IP拿出来做判断
def getIpFromMysql():db=pymysql.connect(host='localhost',port=3306,user='root',passwd='a',db='python',charset='utf8')cursor=db.cursor()sql='select * from kuaidaili'try:cursor.execute(sql)results=cursor.fetchall()allIp=[]for row in results:lst=[]kip=row[1]kport=row[2]kptype=row[3]ip=[kip,kport,kptype]allIp.append(ip)checkIp( allIp )    #将所有ip拿去检测except:print('查询报错')class checkIp(object):def __init__(self,allIp):self.allIp=allIpself.check()def check(self):for ip in self.allIp:kip=ip[0]kport=ip[1]kptype=ip[2].lower()url=kptype+"://"+kip+":"+kportprint(url)try:#使用代理ip去访问百度，如果ip不能用，则会报错requests.get( 'http://www.baidu.com', proxies={ kptype:url } )except:print('=============================failure')else:print('=============================success')    if __name__=='__main__':getIpFromMysql()

import requests# 一定要添加浏览器，不然可能会遇到网络防火墙
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:58.0) Gecko/20100101 Firefox/58.0'}# 第一次去抽屉拿到需要授权的cookies
r1 = requests.get(url='http://dig.chouti.com/',headers=headers
)# 获取第一次进入主页得到的cookie
r1_cookie_dic = r1.cookies.get_dict()
print(r1_cookie_dic)# 登录的提交参数
post_data = {"phone": '86' + '你抽屉官网注册的手机号','password': '你的抽屉官网密码','oneMonth': 1  # 一个月内免登陆
}# 第二次登录抽屉
response = requests.post(url='https://dig.chouti.com/login',# 为了模拟浏览器得行为，每次都要带着请求头去这样这样不会容易被发现是爬虫。headers=headers,data=post_data,# 第二次登陆发送post请求时，应该将第一次得cookies带过去授权，所以要带着cookies。cookies=r1_cookie_dic
)
print(response.text)
# 获取第二次登录返回的cookie，尽管有返回，但是第二次返回的这个cookie没有用
cookie_dic = response.cookies.get_dict()
print(cookie_dic)# 进入个人设置页面
response = requests.get(url='https://dig.chouti.com/profile',# 模拟浏览器headers=headers,# 提交cookie(一般网站使用下面这句代码就行，抽屉不行)# cookies=cookie_dic,# 抽屉官网有点淘气，阴了我们一下，将授权好的第一次的cookies带进去访问cookies={'gpsd': r1_cookie_dic.get('gpsd')}
)
# 输出返回的页面
print(response.text)