import pymysql
from http import cookiejar
from urllib import request,parse
from urllib.error import HTTPError,URLErrorclass Session_cookie(object):def __init__(self): # 初始化cookie = cookiejar.CookieJar()handler = request.HTTPCookieProcessor(cookie)self.opener = request.build_opener(handler)# request.install_opener(opener) # 这个是将opner变成全局参数，慎用，尤其在大型项目开发中def main(self,url,headers=None,data=None):return main(url,headers,data,self.opener)# 爬虫连接
# ------------------------------------------------------------
def main(url,headers=None,data=None,opener=None): # 调用函数if not data:print('无data数据')return get_response(url,headers=headers,opener=opener)else:print('有data数据')return get_response(url,headers=headers,data=data,opener=opener)def get_response(url,data=None,headers=None,opener=None):if not headers:headers = {'User-Agent':get_random('p_useragent')[0][1]}html = '' # 定义一个空值，避免出错无法返回try:if data:data = parse.urlencode(data)data = bytes(data,encoding='utf-8')req = request.Request(url, data=data, headers=headers)else:req = request.Request(url,headers=headers)if not opener:response = request.urlopen(req)else:print('使用opener')response = opener.open(req)html = response.read().decode()except HTTPError as e: # 总的错误信息，不适合用于调试print(e)except URLError as e:print(e)return html  # 返回数据 如果写在try里面，当httperror时无返回值，外面就接不到值# 数据库处理
# ------------------------------------------------------------
def mysql_connect(table,sql,data=None): # 数据库连接conn = pymysql.connect('127.0.0.1', 'root', '123456', 'PaChong', charset='utf8')cursor = conn.cursor()if not data:row = cursor.execute(sql) # 查询类else:row = cursor.execute(sql,data) # 需要提交的类conn.commit() # 提交return cursordef get_random(table=None): # 由数据库获取随机信息 proxy agentsql = 'SELECT * FROM {} WHERE id >= ((SELECT MAX(Id) FROM {})-(SELECT MIN(Id) FROM {})) * RAND() + (SELECT MIN(Id) FROM {})  LIMIT 1'.format(table, table, table, table)cursor = mysql_connect(table,sql=sql)return cursor.fetchall()def proxies_save_mysql(data): # 保存代理table = 'p_proxies'keys = ','.join(data.keys())values = ','.join(['%s']*len(data))sql = 'insert into {}({}) values({})'.format(table,keys,values)data = tuple(data.values())mysql_connect(table,sql=sql,data=data)
# ------------------------------------------------------------if __name__ == '__main__':# url = 'http://fanyi.baidu.com/sug'# data = {'kw':'中国'}# import json# res = json.loads(main(url,data=data))# print(res)# url = 'http://www.baidu.com'# res = main(url)# print(res)pass