爬取京东辣条商品详情

01_爬取一页京东辣条商品详情在控制台显示代码

from bs4 import BeautifulSoup
import requests
import re#静态加载的前30条数据
def beforePage():url = 'https://search.jd.com/Search?keyword=%E8%BE%A3%E6%9D%A1&enc=utf-8&wq=la&pvid=95d02412228d4974928eda8db3952a38'headers={          #headers模拟浏览器，防止反爬'cookie': '__jdu=1732201767; areaId=5; shshshfpa=946c48b7-3dab-80c8-2211-6df817ea8dd0-1583387122; shshshfpb=r2KaWBum7i2jWCp2aNTPUbg%3D%3D; PCSYCityID=CN_130000_131000_131022; user-key=1e0460ef-ae67-489c-a748-c7f8cd75c7a7; cn=0; ipLoc-djd=5-142-143-5008; xtest=7314.cf6b6759; qrsc=3; unpl=V2_ZzNtbRVTFhV8AUYHexBZV2IEFV0RVkcQfAwTXHkeWwA3UBpYclRCFnQUR1ZnGV8UZgsZXkJcRxNFCEdkeBBVAWMDE1VGZxBFLV0CFSNGF1wjU00zQwBBQHcJFF0uSgwDYgcaDhFTQEJ2XBVQL0oMDDdRFAhyZ0AVRQhHZHsfXgZkAhJbRVFzJXI4dmR%2bEF8DYQsiXHJWc1chVEJWch1UDCoDFF9BVEIVcw9AZHopXw%3d%3d; rkv=V0500; __jdv=76161171|baidu|-|organic|not set|1583994333826; __jda=122270672.1732201767.1582290570.1583979524.1583994334.8; __jdc=122270672; shshshfp=97731bbbb1ddc34489f0306102c6ded9; 3AB9D23F7A4B3C9B=STXCJX2N26USPJKCXXSNGZG3SY72RMSJ7GCY3UVE3GZSR7LGVN2SWBOS5IOVQTAIDXSOJYM4YHTXW3FRQDS3LD5HXM','referer': 'https://www.jd.com/','user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'}response = requests.get(url,headers=headers)       #获取网址response.encoding='utf-8'         #指定编码方式，不然会出现乱码soup = BeautifulSoup(response.text, 'lxml')jdtitles=soup.select('#J_goodsList ul li div .p-name a em')   #获取商品名称jdprices=soup.select('#J_goodsList ul li div .p-price strong i')   #获取商品价格jdhrefs=soup.select('#J_goodsList ul li div .p-name a')  #获取商品链接"抓取静态加载的前30页商品信息"count=0for jdtitle,jdprice,jdhref in zip(jdtitles,jdprices,jdhrefs):count=count+1res=re.match("(http|https|ftp)://[^\s]+", jdhref["href"])if(res==None):                 #如果不是以https开头的，拼接成超链接jdhref["href"]=('https:'+jdhref["href"])print("{}:".format(count))print("商品名称为：{}".format(jdtitle.get_text().strip()))         #用get_text()方法筛选出标签中的文本信息，由于筛选结果有换行符\n所以用strip()将换行符去掉print("商品价格为：￥{}".format(jdprice.get_text()))         #用get_text()方法筛选出标签中的文本信息，由于筛选结果有换行符\n所以用strip()将换行符去掉print("商品链接为：{}".format(jdhref["href"]))         #用get_text()方法筛选出标签中的文本信息，由于筛选结果有换行符\n所以用strip()将换行符去掉print('----------------------------------------------------------------------------------------------')#抓取动态加载的后30条数据(通过接口)
def lastPage():url='https://search.jd.com/s_new.php?keyword=%E8%BE%A3%E6%9D%A1&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=la&stock=1&page=2&s=27&scrolling=y&log_id=1584022492.13126&tpl=1_M&show_items=5628182,33148629864,100001248522,27006485882,22217648287,7307139,100005128356,4961761,5410849,10358431652,2994821,16806977845,1693886696,100007908040,100001252493,4775568,2297172,3924103,6401637,100007908062,100010946294,100004549721,8586192,8667369,65809387826,2543190,2688516,100010882270,100010749366,100008348244'headers={"cookie":"__jdu=1732201767; areaId=5; shshshfpa=946c48b7-3dab-80c8-2211-6df817ea8dd0-1583387122; shshshfpb=r2KaWBum7i2jWCp2aNTPUbg%3D%3D; PCSYCityID=CN_130000_131000_131022; user-key=1e0460ef-ae67-489c-a748-c7f8cd75c7a7; cn=0; ipLoc-djd=5-142-143-5008; xtest=7314.cf6b6759; qrsc=3; unpl=V2_ZzNtbRVTFhV8AUYHexBZV2IEFV0RVkcQfAwTXHkeWwA3UBpYclRCFnQUR1ZnGV8UZgsZXkJcRxNFCEdkeBBVAWMDE1VGZxBFLV0CFSNGF1wjU00zQwBBQHcJFF0uSgwDYgcaDhFTQEJ2XBVQL0oMDDdRFAhyZ0AVRQhHZHsfXgZkAhJbRVFzJXI4dmR%2bEF8DYQsiXHJWc1chVEJWch1UDCoDFF9BVEIVcw9AZHopXw%3d%3d; rkv=V0500; __jdv=76161171|baidu|-|organic|not set|1583994333826; 3AB9D23F7A4B3C9B=STXCJX2N26USPJKCXXSNGZG3SY72RMSJ7GCY3UVE3GZSR7LGVN2SWBOS5IOVQTAIDXSOJYM4YHTXW3FRQDS3LD5HXM; wlfstk_smdl=iepmgxn5lk1over8ydd4ik7rfvd8cetv; __jdc=122270672; shshshfp=97731bbbb1ddc34489f0306102c6ded9; __jda=122270672.1732201767.1582290570.1584018274.1584022566.11; __jdb=122270672.2.1732201767|11.1584022566; shshshsID=6582b0d1fbf0500d3f2639f064df2e33_2_1584022602218","referer":"https://search.jd.com/Search?keyword=%E8%BE%A3%E6%9D%A1&enc=utf-8&wq=la&pvid=95d02412228d4974928eda8db3952a38","user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36"}response = requests.get(url,headers=headers)       #获取网址response.encoding='utf-8'         #指定编码方式，不然会出现乱码soup = BeautifulSoup(response.text, 'lxml')jdtitles=soup.select('li div .p-name em')   #获取商品名称jdprices=soup.select('li div .p-price strong i')   #获取商品价格jdhrefs=soup.select('li div .p-name a')  #获取商品链接count=30for jdtitle,jdprice,jdhref in zip(jdtitles,jdprices,jdhrefs):count=count+1res=re.match("(http|https|ftp)://[^\s]+", jdhref["href"])if(res==None):                 #如果不是以https开头的，拼接成超链接jdhref["href"]=('https:'+jdhref["href"])print("第{}条数据:".format(count))print("商品名称为：{}".format(jdtitle.get_text().strip()))         #用get_text()方法筛选出标签中的文本信息，由于筛选结果有换行符\n所以用strip()将换行符去掉print("商品价格为：￥{}".format(jdprice.get_text()))         #用get_text()方法筛选出标签中的文本信息，由于筛选结果有换行符\n所以用strip()将换行符去掉print("商品链接为：{}".format(jdhref["href"]))         #用get_text()方法筛选出标签中的文本信息，由于筛选结果有换行符\n所以用strip()将换行符去掉print('----------------------------------------------------------------------------------------------')if __name__ == '__main__':beforePage()lastPage()

02_爬取多页数京东辣条商品详情写入文件

from bs4 import BeautifulSoup
import requests
import re
import time#将数据写入文件
def dataWrite(jdtitles,jdprices,jdhrefs,count):for jdtitle,jdprice,jdhref in zip(jdtitles,jdprices,jdhrefs):count=count+1res=re.match("(http|https|ftp)://[^\s]+", jdhref["href"])if(res==None):                 #如果不是以https开头的，拼接成超链接jdhref["href"]=('https:'+jdhref["href"])with open('JD_LATIAO.txt','a',newline='',encoding='utf-8')as f:        #打开文件追加写入x=jdtitle.get_text().strip()y=jdprice.get_text()z=jdhref["href"]f.writelines("第"+str(count)+"条数据：\n商品名称："+x+"\n商品价格："+y+"\n商品链接："+z+"\n\n")f.close()              #关闭文件#静态加载的前30条数据
def beforePage(n):#构造每一页的url变化url = 'https://search.jd.com/Search?keyword=%E8%BE%A3%E6%9D%A1&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=la&stock=1&page='+str(2*n-1)headers={          #headers模拟浏览器，防止反爬'cookie': '__jdu=1732201767; areaId=5; shshshfpa=946c48b7-3dab-80c8-2211-6df817ea8dd0-1583387122; shshshfpb=r2KaWBum7i2jWCp2aNTPUbg%3D%3D; PCSYCityID=CN_130000_131000_131022; user-key=1e0460ef-ae67-489c-a748-c7f8cd75c7a7; cn=0; ipLoc-djd=5-142-143-5008; xtest=7314.cf6b6759; qrsc=3; unpl=V2_ZzNtbRVTFhV8AUYHexBZV2IEFV0RVkcQfAwTXHkeWwA3UBpYclRCFnQUR1ZnGV8UZgsZXkJcRxNFCEdkeBBVAWMDE1VGZxBFLV0CFSNGF1wjU00zQwBBQHcJFF0uSgwDYgcaDhFTQEJ2XBVQL0oMDDdRFAhyZ0AVRQhHZHsfXgZkAhJbRVFzJXI4dmR%2bEF8DYQsiXHJWc1chVEJWch1UDCoDFF9BVEIVcw9AZHopXw%3d%3d; rkv=V0500; __jdv=76161171|baidu|-|organic|not set|1583994333826; __jda=122270672.1732201767.1582290570.1583979524.1583994334.8; __jdc=122270672; shshshfp=97731bbbb1ddc34489f0306102c6ded9; 3AB9D23F7A4B3C9B=STXCJX2N26USPJKCXXSNGZG3SY72RMSJ7GCY3UVE3GZSR7LGVN2SWBOS5IOVQTAIDXSOJYM4YHTXW3FRQDS3LD5HXM','referer': 'https://www.jd.com/','user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'}response = requests.get(url,headers=headers)       #获取网址response.encoding='utf-8'         #指定编码方式，不然会出现乱码soup = BeautifulSoup(response.text, 'lxml')jdtitles=soup.select('#J_goodsList ul li div .p-name a em')   #获取商品名称jdprices=soup.select('#J_goodsList ul li div .p-price strong i')   #获取商品价格jdhrefs=soup.select('#J_goodsList ul li div .p-name a')  #获取商品链接"静态加载的前30条商品信息"count=0dataWrite(jdtitles,jdprices,jdhrefs,count)#抓取动态加载的后30条数据(通过接口)
def afterPage(n):#url='https://search.jd.com/s_new.php?keyword=%E8%BE%A3%E6%9D%A1&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=la&stock=1&page=2&s=27&scrolling=y&log_id=1584022492.13126&tpl=1_M&show_items=5628182,33148629864,100001248522,27006485882,22217648287,7307139,100005128356,4961761,5410849,10358431652,2994821,16806977845,1693886696,100007908040,100001252493,4775568,2297172,3924103,6401637,100007908062,100010946294,100004549721,8586192,8667369,65809387826,2543190,2688516,100010882270,100010749366,100008348244'#这里的showitems是裤子的id,page是翻页的，可以看出来我们只需要改动两处就可以打开不同的网页了#获取当前的Unix时间戳，并且保留小数点后5位a=time.time()b='%.5f'%aurl='https://search.jd.com/s_new.php?keyword=%E8%BE%A3%E6%9D%A1&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=la&stock=1&page='+str(2*n)+'&scrolling=y&log_id='+str(b)headers={"cookie":"__jdu=1732201767; areaId=5; shshshfpa=946c48b7-3dab-80c8-2211-6df817ea8dd0-1583387122; shshshfpb=r2KaWBum7i2jWCp2aNTPUbg%3D%3D; PCSYCityID=CN_130000_131000_131022; user-key=1e0460ef-ae67-489c-a748-c7f8cd75c7a7; cn=0; ipLoc-djd=5-142-143-5008; xtest=7314.cf6b6759; qrsc=3; unpl=V2_ZzNtbRVTFhV8AUYHexBZV2IEFV0RVkcQfAwTXHkeWwA3UBpYclRCFnQUR1ZnGV8UZgsZXkJcRxNFCEdkeBBVAWMDE1VGZxBFLV0CFSNGF1wjU00zQwBBQHcJFF0uSgwDYgcaDhFTQEJ2XBVQL0oMDDdRFAhyZ0AVRQhHZHsfXgZkAhJbRVFzJXI4dmR%2bEF8DYQsiXHJWc1chVEJWch1UDCoDFF9BVEIVcw9AZHopXw%3d%3d; rkv=V0500; __jdv=76161171|baidu|-|organic|not set|1583994333826; 3AB9D23F7A4B3C9B=STXCJX2N26USPJKCXXSNGZG3SY72RMSJ7GCY3UVE3GZSR7LGVN2SWBOS5IOVQTAIDXSOJYM4YHTXW3FRQDS3LD5HXM; wlfstk_smdl=iepmgxn5lk1over8ydd4ik7rfvd8cetv; __jdc=122270672; shshshfp=97731bbbb1ddc34489f0306102c6ded9; __jda=122270672.1732201767.1582290570.1584018274.1584022566.11; __jdb=122270672.2.1732201767|11.1584022566; shshshsID=6582b0d1fbf0500d3f2639f064df2e33_2_1584022602218","referer":"https://search.jd.com/Search?keyword=%E8%BE%A3%E6%9D%A1&enc=utf-8&wq=la&pvid=95d02412228d4974928eda8db3952a38","user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36"}response = requests.get(url,headers=headers)       #获取网址response.encoding='utf-8'         #指定编码方式，不然会出现乱码soup = BeautifulSoup(response.text, 'lxml')jdtitles=soup.select('li div .p-name em')   #获取商品名称jdprices=soup.select('li div .p-price strong i')   #获取商品价格jdhrefs=soup.select('li div .p-name a')  #获取商品链接"动态加载的后30条商品信息"count=30dataWrite(jdtitles,jdprices,jdhrefs,count)#主函数...
if __name__ == '__main__':#清空文件原内容with open('JD_LATIAO.txt','w+',encoding='utf-8')as f:''#爬取前3页数据for i in range(1,4):#下面的print函数主要是为了方便查看当前抓到第几页了print('***************************************************')try:print('   First_Page:   ' + str(i))beforePage(i)print('   Finish')except Exception as e:print(e)print('------------------')try:print('   Last_Page:   ' + str(i))afterPage(i)print('   Finish')except Exception as e:print(e)

爬取京东辣条商品详情相关推荐

python爬取京东商品图片_python利用urllib实现爬取京东网站商品图片的爬虫实例
本例程使用urlib实现的,基于python2.7版本,采用beautifulsoup进行网页分析,没有第三方库的应该安装上之后才能运行,我用的IDE是pycharm,闲话少说,直接上代码! # -* ...
selenium/requess爬取京东手机商品的详细信息1~selenium练习版
selenium/requess爬取京东手机商品的详细信息1~selenium!! 前言因为我也是个学生,所以代码可能会有点繁琐,我们都是超能100,一点点积累进步,其实有很多的地方可以简化,因为我 ...
python爬取京东网页商品实例（一）
# Copyright (c)2018, 东北大学软件学院学生 # All rightsreserved # 文件名称:justForTest.py # 作者:孔云 #问题描述:打开京东页面,选取一 ...
Python爬取京东任意商品数据实战总结
利用Python爬取京东任意商品数据今天给大家展示爬取京东商品数据首先呢还是要分思路的,我分为以下几个步骤: 第一步:得到搜索指定商的url 第二步:获得搜索商品列表信息第三步:对得到的商品数据 ...
用selenium爬取京东平台商品列表,爬取商品名称、价格、店铺信息
#用selenium爬取京东平台商品列表,爬取商品名称.价格.店铺信息from selenium import webdriver from selenium.webdriver.common.by ...
Python爬虫教程：Python爬取京东商城商品大图详解
Python爬取京东商城商品大图详解做为一个爬虫初学者,在做爬取网址图片的练习中以京东网为例爬取商品大图并保存在相应的文件夹 1.导入模块 import urllib.request import ...
Scrapy练习——爬取京东商城商品信息
刚刚接触爬虫,花了一段时间研究了一下如何使用scrapy,写了一个比较简单的小程序,主要用于爬取京东商城有关进口牛奶页面的商品信息,包括商品的名称,价格,店铺名称,链接,以及评价的一些信息等.简单记录 ...
JS+Selenium+excel追加写入，使用python成功爬取京东任何商品
之前一直是requests库做爬虫,这次尝试下使用selenium做爬虫,效率不高,但是却没有限制,文章是分别结合大牛的selenium爬虫以及excel追加写入操作而成,还有待优化,打算爬取更多信息 ...
利用python爬虫爬取京东商城商品图片
笔者曾经用python第三方库requests来爬取京东商城的商品页内容,经过解析之后发现只爬到了商品页一半的图片.(这篇文章我们以爬取智能手机图片为例) 当鼠标没有向下滑时,此时查看源代码的话,就会 ...
简单爬取京东百万商品的缺货记录
申明:转载请注明作者( ChenReason.com)及出处,谢谢. 最近在学python,首先就想找个小功能来试着实现以下当作练手,最后决定就以爬取京东商品的缺货记录作为目标. 要爬京东数据,当然要 ...

爬取京东辣条商品详情

爬取京东辣条商品详情相关推荐

最新文章

热门文章