import requests
from requests.exceptions import ReadTimeout,HTTPError,RequestException
import re
from bs4 import BeautifulSoup
import os
import urllib.request
import urllib.parse
import string
import time#爬取图片URL信息数据
def CrawImgInfo(req_url):resultdata=[]headers={"content-type":"text/html; charset=utf-8"}try:res=requests.get(req_url,headers=headers)if res.status_code==200:html=BeautifulSoup(res.text,"html.parser")#选取所有照片HTMLimgItems=html.select(".mdui-container-fluid div.mdui-row-gapless div.mdui-col-xs-4")for item in imgItems:#获取大图信息页面URLimginfo_url=item.select("a")[0].attrs["href"]resultdata.append(imginfo_url)else:print("fail")except ReadTimeout:print("timeout")except HTTPError:print("HttpError")except RequestException:print("httpError") return resultdata#获取图片的URL
def CrawImgUrl(req_url):imgurl=""headers={"content-type":"text/html; charset=utf-8"}try:res=requests.get(req_url,headers=headers)if res.status_code==200:html=BeautifulSoup(res.text,"html.parser")#获取大图的URLimgurl=html.select(".mdui-container-fluid div.mdui-row-gapless a")[0].attrs["href"]else:print("fail")except ReadTimeout:print("timeout")except HTTPError:print("HttpError")except RequestException:print("httpError") return imgurl#写入数据，保存到本地
def SaveImg(filepath,imgurl):#URL含中文，需要编码imgurl_encode=urllib.parse.quote(imgurl,safe=string.printable)try:resp=urllib.request.urlopen(imgurl_encode)imgname=imgurl.split("/")[-1]if resp.code==200:img=resp.read()imgfilename=filepath+imgnamewith open(imgfilename,"wb") as f:f.write(img)except urllib.error.HTTPError as e:print("The server conldn\'t fulfill the request.")except urllib.error.URLError as e:print('We failed to reach a server.')def GetData():rootUrl="网站地址"pageUrl="翻页地址"#实际总页数加1pageCount=267savePath="本地存储地址"imgCount=0for pageIndex in range(1,pageCount):req_url=pageUrl+str(pageIndex)curpage_imginfos=CrawImgInfo(req_url)if len(curpage_imginfos)>0:for item in curpage_imginfos:imgUrl=CrawImgUrl(rootUrl+item)if len(imgUrl)>0:SaveImg(savePath,imgUrl)imgCount=imgCount+1print("图片：%s 下载完成！(下载总数：%s|%s/%s)" %(imgUrl,imgCount,pageIndex,pageCount-1))#暂停指定秒数，对服务器要温柔一些time.sleep(1)if __name__ == '__main__':GetData()