[Python] 纯文本查看 复制代码import requestsimport os

import time

from lxml import etree

'''

xpath获取的element对象返回的是个字典.tag(标签名称) .attrib(标签属性) .text(标签文本)

'''

def get_xpath_text(url,headers):

response = requests.get(url, headers)

response.encoding = 'utf-8'

xpath_html = etree.HTML(response.text)

return xpath_html

def xpath_get_urls(html):

xpath_html = etree.HTML(html)

urls = xpath_html.xpath("//div[@class='item']/div[@class='p']/a")

return urls

def get_group_pageinfo(page_url):

grouppage_urls=[]

xpath_html = get_xpath_text(page_url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36'})

pagenumstr = xpath_html.xpath("//div[@class='page']/li[1]/a")[0].text

group_pic_name = xpath_html.xpath("//h1")[0].text

pagenumstr = pagenumstr.replace('共', '').replace('页:', '')

for i in range(1,int(pagenumstr)+1):

if(i==1):

grouppage_urls.append(page_url)

else:

grouppage_urls.append(page_url.replace('.html','_%d.html'%i))

return grouppage_urls,group_pic_name

def get_picurls(page_url):

picurls = []

xpath_html = get_xpath_text(page_url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36'})

picurl_elements = xpath_html.xpath('//div[@class="contents"]/a/img')

for pic_url_element in picurl_elements:

picurls.append(pic_url_element.attrib['src'])

return picurls

if __name__ == '__main__':

#存储每组图片的url

group_pic_urls=[]

#选取10组图片

for num in range(1,2):

page_url = 'http://www.beautyleg7.com/siwameitui/list_3_%d.html' % num

response = requests.get(page_url,headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36'})

response.encoding = 'utf-8'

for ul in xpath_get_urls(response.text):

group_pic_urls.append(ul.attrib['href'])

#for循环访问每组套图的url 每个页面获取图片的xpath：//div[@class="contents"]/a

#获取总共的页数xpath：//div[@class="page"]/li[1]/a

for group_pic_url in group_pic_urls:

group_pageurls,group_pic_name = get_group_pageinfo(group_pic_url)

num = 1

//文件的存储路径可根据需要自己选择

filepath = "F:\crawler\leg\\"+group_pic_name.replace('/','-')

print(filepath)

if( not os.path.exists(filepath)):

os.makedirs(filepath.strip())

os.chdir(filepath)

for group_pageurl in group_pageurls:

picurls = get_picurls(group_pageurl)

for pic_url in picurls:

print(pic_url)

response = requests.get(pic_url)

img = response.content

with open(filepath+'\%d.jpg'%num,'wb') as f:

f.write(img)

num=num+1