自学的python和爬虫技术。使用到Redis，MySQL数据库；request请求模块；re，Xpath解析模块；dumps, loads序列化和反序列化。还可以配合代理池使用。

爬取的是https://weixin.sogou.com/。网站只能微信扫码登陆，不登录只能访问十页，这里

使用Session保持会话实现登陆状态爬取100页，cookie是自己添加的。

第一次发文，试一试页面效果hhhh，有机会再具体解析思路。

算是我自学的一个阶段性成果，因为我是外行，代码中肯定有些愚蠢的地方，贴出源代码来供大家批判指正。

python3.6环境，运行确保安装相关python库以及数据库

from requests import Request, ConnectionError, ReadTimeout

from requests import Session

from pickle import dumps, loads

from urllib.parse import urlencode

from lxml import etree

import pymysql

#控制代理池的开启

from run import main

import time

import re

from redis import StrictRedis

TIMEOUT = 10

MAX_FAILED_TIME = 3

REDIS_KEY = 'weixinrequests_liu'

REDIS_HOST = "localhost"

REDIS_PORT = 6379

REDIS_PASSWORD = None

MYSQL_HOST = "localhost"

MYSQL_NAME = 'root'

MYSQL_PASSWORD = '*****'

MYSQL_DATABASE = 'samp_db'

MYSQL_PORT = 3306

PROXY_POOL_URL = 'http://api.xdaili.cn/xdaili-api//greatRecharge/getGreatIp?spiderId=2f80314147e34f04bf5705006d837a86&orderno=YZ2018112477906eltR9&returnType=1&count=10'

# PROXY_POOL_URL = 'http://localhost:5555/random'

KEY_WORDS = "杨超越"

COOKIE = 'SUV=0081272D655E8569592959DA71D95636; SUID=8AB1E7652F20910A00000000595B0BDA; ld=4Zllllllll2bLZA5lllllVsKxm1lllllBqf3vZllll9lllllxllll5@@@@@@@@@@; ABTEST=3|1543134403|v1; IPLOC=CN3100; weixinIndexVisited=1; sct=1; ppinf=5|1543150392|1544359992|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZToxODolRTYlOUQlOEUlRTUlOEQlOUF8Y3J0OjEwOjE1NDMxNTAzOTJ8cmVmbmljazoxODolRTYlOUQlOEUlRTUlOEQlOUF8dXNlcmlkOjQ0Om85dDJsdUgyb3R2dks1bFoydFpTU1Q3MDJWY0VAd2VpeGluLnNvaHUuY29tfA; pprdig=QPtri4HHDEm4Gz9hxTvUj8MO9ymKgOe2EgkwA3uuYG0JIjd1IM8NnWkE6f1vrIt4mlMoC1Nmomb6ntUbGAjANhGeEkJOid0_Yk4g0yBHTA0FQ3_WMsbYhS0SQN_Sbmvj66AVXN93ZhbqAzmflqUoyNdw7YGswhD3tZ7J0xu0i0U; sgid=12-38075013-AVv6mzhP6ltMX5qfqx8JPJc; SUIR=AC6DB982E7ED939ECB44C2A1E85C066B; SNUID=AA6ABE84E0E49B9DDE6C0D11E196941C; JSESSIONID=aaaZoBR-hzstpadzRe7Cw; ppmdig=1543197423000000309d9113ade27bf0e5a2598fd5f1caba'

START_ID = 10

DIC_TIME = 0

HEADERS = {

'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

'Accept-Encoding': 'gzip, deflate, br',

'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',

'Cache-Control': 'max-age=0',

'Connection': 'keep-alive',

'Cookie': COOKIE,

# 'Host':'weixin.sogou.com',

'Upgrade-Insure-Request': '1',

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',

}

TABLE_NAME = 'seshi3'

MAX_PAGE = 100

PROXY_ON = False

class MySQL():

def __init__(self, host=MYSQL_HOST, username=MYSQL_NAME, password=MYSQL_PASSWORD, port=MYSQL_PORT, database=MYSQL_DATABASE):

try:

self.db = pymysql.connect(host=host, user=username, password=password, db=database, port=port)

self.cursor = self.db.cursor()

except Exception as e:

print(e)

def insert(self, table, data):

keys = ','.join(data.keys())

values = ','.join(len(data)*['%s'])

print(values)

sql_query = 'INSERT INTO {}({}) values ({})'.format(table, keys, values)

try:

self.cursor.execute(sql_query, tuple(data.values()))

self.db.commit()

print('真的插入成功了')

except Exception as e:

print(e,'插入失败')

self.db.rollback()

class WeixinRequest(Request):

def __init__(self, url, callback, method='GET', headers=None, need_proxy=False, fail_time=0, timeout=TIMEOUT):

Request.__init__(self, method, url, headers)

self.callback = callback

self.need_proxy = need_proxy

self.fail_time = fail_time

self.timeout = timeout

class RedisQueue():

def __init__(self):

self.db = StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD)

def add(self, request):

if isinstance(request, WeixinRequest):

return self.db.rpush(REDIS_KEY, dumps(request))

else:

print("meiyoujiaaa")

return False

def pop(self):

if self.db.llen(REDIS_KEY):

return loads(self.db.lpop(REDIS_KEY))

else:

return False

def empty(self):

return self.db.llen(REDIS_KEY) == 0

def get_proxy():

if PROXY_ON:

try:

response = requests.get(PROXY_POOL_URL)

if response.status_code == 200:

print('获得代理', response.text)

return response.text

return None

except Exception as e:

print(e)

return None

False

class Spider():

base_url = r'https://weixin.sogou.com/weixin'

session = Session()

queue = RedisQueue()

mysql = MySQL()

def __init__(self, dictime, start_id, table_name, headers, key_words, max_page):

self.rtime = start_id

self.dictime = dictime

self.start_id = start_id

self.table_name = table_name

self.headers = headers

self.key_words = key_words

self.max_page = max_page

def start(self):

self.session.headers.update(self.headers)

start_url = 'https://weixin.sogou.com/weixin?query={}&_sug_type_=&sut=4722&lkt=1%2C1542979381375%2C1542979381375&s_from=input&_sug_=y&type=2&sst0=1542979381478&page={}&ie=utf8&w=01019900&dr=1'.format(self.key_words, self.start_id)

wexin_request = WeixinRequest(start_url, callback=self.parse_index, need_proxy=False)

self.queue.add(wexin_request)

print(wexin_request.url, "是开始获取的url")

def parse_index(self, r):

html = etree.HTML(r.text)

html_li = html.xpath("//ul[@class='news-list']/li")

for url_list in html_li:

url = url_list.xpath(".//h3/a/@href")[0]

print('parese_index获取到这个微信url(dictime,rtime)', (self.dictime,self.rtime), url)

self.dictime += 1

yield WeixinRequest(url=url, callback=self.parse_detail, need_proxy=True, headers=self.headers, method='GET')

pattern2 = re.compile(r'''sogou_next.*?="(.*?)"''')

next = re.findall(pattern2, r.text)[0] if len(re.findall(pattern2, r.text)) >= 1 else 0

if next:

next_url = self.base_url + next

self.rtime += 1

print('哈哈哈parese_index自动获取到这个下一页url(dictime,rtime)', (self.dictime, self.rtime), next_url)

yield WeixinRequest(url=next_url, callback=self.parse_index, need_proxy=True, headers=self.headers)

else:

next_url = 'https://weixin.sogou.com/weixin?query={}&_sug_type_=&sut=4722&lkt=1%2C1542979381375%2C1542979381375&s_from=input&_sug_=y&type=2&sst0=1542979381478&page={}&ie=utf8&w=01019900&dr=1'.format(self.key_words,self.rtime+1)

self.rtime += 1

if self.rtime <= self.max_page:

print('啊啊啊自动构造这个下一页url(dictime,rtime)', (self.dictime, self.rtime), next_url)

yield WeixinRequest(url=next_url, callback=self.parse_index, need_proxy=True, headers=self.headers)

def parse_detail(self, response):

r = response.text

pattern = re.compile(r'publish_time = "(.*?)"')

date = re.findall(pattern, r)[0] if len(re.findall(pattern, r))>=1 else "日期出错"

# print(r.text)

html = etree.HTML(r)

title = html.xpath("//h2[@class='rich_media_title']//text()")[0].strip() if len(html.xpath("//h2[@class='rich_media_title']//text()"))>=1 else "标题出错"

lend = len(html.xpath("//div[contains(@id,'meta_content') and @class='rich_media_meta_list']/span"))

wechat = html.xpath("//div[contains(@id,'meta_content') and @class='rich_media_meta_list']/span[last()]/a/text()")[0].strip('\n').strip() if len(html.xpath("//div[contains(@id,'meta_content') and @class='rich_media_meta_list']/span[last()]/a/text()"))>=1 else "wechat出错"

pattern3 = re.compile(r'''js_name.*?>(.*?)

nickname = re.findall(pattern3,r)[0].strip('\n').strip() if len(re.findall(pattern3,r))>0 else 'nickname出错'

if lend == 3:

original = \

html.xpath("//div[contains(@id,'meta_content') and @class='rich_media_meta_list']/span[last()-2]/text()")[0].strip('\n').strip() if len(html.xpath("//div[contains(@id,'meta_content') and @class='rich_media_meta_list']/span[last()-2]/text()"))>=1 else "original出错"

else:

original = ''

content_list = html.xpath("//div[@id='img-content']/div[2]//p//text()") if len(html.xpath("//div[@id='img-content']/div[2]//p//text()")) >=1 else "内容出错"

neirong = ""

for i in content_list:

neirong += i.strip()

data = {

'title':title,

'original':original,

'nickname':nickname,

'date': date,

'wechat':wechat,

'content':neirong

}

yield data

def schedule(self):

while not self.queue.empty():

print("开始调度次", "\n", "="*20)

weixin_request = self.queue.pop()

print("调度选出了这个url", weixin_request.url, '选出了这个url')

# 存在两种可能的请求

callback = weixin_request.callback

# 先请求

response = self.request(weixin_request)

print(response, '获得相应*********')

if hasattr(response, 'status_code'):

print(response.status_code, '上面url的状态')

if response and (response.status_code == 200):

# 再解析

results = list(callback(response))

if results:

# print("解析结果",results)

for result in results:

if isinstance(result, WeixinRequest):

self.queue.add(result)

print('解析结果，搜索页文章列表', result.url)

# self.rtime += 1

if isinstance(result, dict):

self.mysql.insert(table=self.table_name, data=result)

print("插入到MYSQL",result)

else:

self.error(weixin_request)

else:

self.error(weixin_request)

else:

self.error(weixin_request)

def error(self, weixin_request):

# 接受一个请求

print("休息二秒")

weixin_request.fail_time += 1

print("请求失败", weixin_request.fail_time, "次", weixin_request.url)

time.sleep(20)

if weixin_request.fail_time < MAX_FAILED_TIME:

self.queue.add(weixin_request)

def request(self,weixin_request):

try:

proxy = get_proxy()

print("代理状态，", proxy)

if proxy:

proxies ={

'http': 'http://' + proxy,

'https': 'https://' + proxy

}

return self.session.send(weixin_request.prepare(), proxies=proxies)

time.sleep(3)

return self.session.send(weixin_request.prepare())

except (ConnectionError, ReadTimeout) as e:

print(e)

return False

def run(self):

self.start()

self.schedule()

if __name__ == '__main__':

#mysql内容编码utf8m4

spider = Spider(

dictime=DIC_TIME,

start_id=START_ID,

table_name=TABLE_NAME,

key_words=KEY_WORDS,

headers=HEADERS,

max_page=MAX_PAGE

)

spider.run()

print(spider.rtime, "请求页数")

print(spider.dictime, "请求篇数")

# MySQL存储结果

# 运行结果