本文我们将尝试下载如下所示的京东商品晒单图。

来到京东首页，在搜索框中输入关键字，比如：手机。打开浏览器的开发者工具(Ctrl+Shift+I)，切换到Network选项。

点击按钮搜索之前输入的关键词，可以看到有如下请求：

请求带到服务器的参数主要有：

keyword: 手机

enc: utf-8

响应内容为一个HTML文件，包含了搜索到的商品的URL。

访问搜索到的某个商品，同样打开浏览器的开发者工具，切换到Network选项，可以看到有如下请求(没有看到的话刷新一下页面)：

返回的数据包含了评论相关的数据，晒单图也在其中。

至此，我们将整个过程分析完毕，用代码实现上述过程即可。

import requests

from bs4 import BeautifulSoup

from urllib.parse import urljoin

import re

import os

import json

def parse_goods_url(keyword):

headers = {

"User-Agent":

"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36"

}

params = {"keyword": keyword, "enc": "utf-8"}

base_url = "https://search.jd.com/Search"

response = requests.get(base_url, params=params, headers=headers)

soup = BeautifulSoup(response.text, "lxml")

goods_url_list = soup.select("#J_goodsList .gl-item .p-name a")

for good in goods_url_list:

yield urljoin("https://item.jd.com", good.get("href"))

def get_good_comment_image_url(good_url, page_num=1):

headers = {

"User-Agent":

"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",

"Referer":

good_url

}

productId = re.findall("(\d+)", good_url)[0]

base_url = "https://sclub.jd.com/comment/productPageComments.action"

for page in range(page_num):

params = {

"callback": "fetchJSON_comment98vv3487",

"productId": productId,

"score": 0,

"sortType": 5,

"page": page,

"pageSize": 10,

"isShadowSku": 0,

"fold": 1

}

response = requests.get(base_url, params=params, headers=headers)

origin_data = re.findall("(\{.*\})", response.text)[0]

comments = json.loads(origin_data)["comments"]

for comment in comments:

for image in comment.get("images"):

yield urljoin(base_url, image.get("imgUrl"))

def download_image(image_url, good_url):

file_name = image_url.split('/')[-1]

fold_name = re.findall("(\d+)", good_url)[0]

if not os.path.exists(fold_name):

os.makedirs(fold_name)

headers = {

"User-Agent":

"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",

"Referer":

good_url

}

response = requests.get(image_url, headers=headers)

with open("{}/{}".format(fold_name, file_name), "wb") as f_obj:

f_obj.write(response.content)

if __name__ == '__main__':

keyword = "手机"

for url in parse_goods_url(keyword):

print("商品：{}".format(url))

for image_url in get_good_comment_image_url(url):

print("下载：{}".format(image_url))

download_image(image_url, url)

运行结果如下：