.*?(http://.*?)"'

data2 = re.compile(listurlpat, re.S)

result = data2.findall(data1)

listurl.append(result)

#print(listurl)

print("共获取到" + str(len(listurl)) + "页")

return listurl

except urllib.error.URLError as e:

if hasattr(e, "code"):

print(e.code)

if hasattr(e, "reason"):

print(e.reason)

time.sleep(10)

except Exception as e:

print("exception:" + str(e))

time.sleep(1)

其中，key对应于关键词，pagestart对应于起始页码，默认为第一页，pageend对应于终止页码。

下载模块

我们需要从对应链接中下载内容，才能做到后面内容的提取，这里，下载模块我提供了两种方案，一种使用了代理服务器，一种没有使用，下面是没有使用代理服务器的代码：

def downloader(url):

headers = ("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "

"(KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 "

"Core/1.63.4793.400 QQBrowser/10.0.702.400")

opener = urllib.request.build_opener()

opener.addheaders = [headers]

urllib.request.install_opener(opener)

try:

data = urllib.request.urlopen(url).read()

data = data.decode('utf-8')

return data

except urllib.error.URLError as e:

if hasattr(e, "code"):

print(e.code)

if hasattr(e, "reason"):

print(e.reason)

time.sleep(10)

except Exception as e:

print("exception:" + str(e))

time.sleep(1)

这里使用了一些简单的伪装，以防止微信对爬虫的阻止，但这样并不安全，所以在完整代码中还提供了代理服务器模块，但是因为免费的代理服务器几乎都不能用，所以在此不多讲解。

完整代码如下

import re

import urllib.request

import time

import urllib.error

'''

def use_proxy(proxy_addr, url):

headers = ("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "

"(KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 "

"Core/1.63.4793.400 QQBrowser/10.0.702.400")

opener = urllib.request.build_opener()

opener.addheaders = [headers]

urllib.request.install_opener(opener)

try:

proxy = urllib.request.ProxyHandler({'http':proxy_addr})

opener = urllib.request.build_opener(proxy, urllib.request.HTTPHandler)

urllib.request.install_opener(opener)

data = urllib.request.urlopen(url).read().decode('utf-8')

return data

except urllib.error.URLError as e:

if hasattr(e, "code"):

print(e.code)

if hasattr(e, "reason"):

print(e.reason)

time.sleep(10)

except Exception as e:

print("exception:" + str(e))

time.sleep(1)

'''

def downloader(url):

headers = ("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "

"(KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 "

"Core/1.63.4793.400 QQBrowser/10.0.702.400")

opener = urllib.request.build_opener()

opener.addheaders = [headers]

urllib.request.install_opener(opener)

try:

data = urllib.request.urlopen(url).read()

data = data.decode('utf-8')

return data

except urllib.error.URLError as e:

if hasattr(e, "code"):

print(e.code)

if hasattr(e, "reason"):

print(e.reason)

time.sleep(10)

except Exception as e:

print("exception:" + str(e))

time.sleep(1)

def getlisturl(key, pagestart, pageend):

try:

page = pagestart

keycode = urllib.request.quote(key)

pagecode = urllib.request.quote("&page")

for page in range(pagestart, pageend+1):

url = "http://weixin.sogou.com/weixin?type=2&query=" + keycode + pagecode + str(page)

data1 = downloader(url)

listurlpat = '