一、顶点小说全站抓取（代码运行下去理论上能抓就行）

先获取所有分类链接import requests
from lxml import etree
import pymysql
headers = {'Host': 'www.23us.co',
'Referer': 'https://www.baidu.com/link?url=Kf8hqBe68nV-DipI3bExcurGlRpGxvq2j0kBbVOrk37&wd=&eqid=91dc6e33001b84210000000263787966',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'}db = pymysql.connect(user='root',password='123456',db='顶点小说')#建立链接
cursor = db.cursor()#游标def get_max_page(urls):source = requests.get(urls).content.decode('gbk')max_page = etree.HTML(source).xpath('//a[@class="last"]/text()')return int(max_page[0])def save_chapter_list(**kwargs):book_name = kwargs.get('book_name','未取到')book_id = kwargs.get('book_id','未取到')status = kwargs.get('status',0)chapter_list_url = kwargs.get('chapter_list_url','未取到')#写入sql语句sql = 'insert into book(book_name,book_id,status,chapter_list_url) values ("{}","{}","{}","{}")'.format(book_name,book_id,status,chapter_list_url)cursor.execute(sql)db.commit()def save_book(**kwargs):chapter_name = kwargs.get('chapter_name','未取到')bid = kwargs.get('bid','未取到')status = kwargs.get('status',0)chapter_contents = kwargs.get('chapter_contents','未取到')chapter_url = kwargs.get('chapter_url','未取到')sql = 'insert into chapters(chapter_name,bid,status,chapter_contents,chapter_url) values ("{}","{}","{}","{}","{}")'.format(chapter_name,bid,status,chapter_contents,chapter_url)# print(sql)cursor.execute(sql)db.commit()pass# 第3步：获取章节列表页信息
def get_book_chapter_list(urls):source = requests.get(urls).content.decode('gbk')hrefs = etree.HTML(source).xpath('//tr/td[1]/a')  #获取书的链接# 把get_book_id('http://www.23us.co/class/3_1.html')变成http://www.23us.co/html/3/3553for i in hrefs:hrefs = i.xpath('@href')[0]  #公共部分book_name = i.xpath('text()')[0]  #书名book_id = hrefs.split('/')[-1]  #根据/拆分，取最后一个chapter_list_url = 'http://23us.co/html/{}/{}/'.format(book_id[:2],book_id) #章节列表页链接#存值和数据库里面的字段对上save_chapter_list(book_name=book_name,book_id=book_id,status=0,chapter_list_url=chapter_list_url)yield chapter_list_urldef get_contents(urls):#正则匹出书的id，'https://23.us.co/html/75/75084/29219318.html'import resource = requests.get(urls).content.decode('gbk')demo = re.compile('http://23us\.co/html/\d+/(\d+)/\d+\.html')book_id = demo.findall(source)[0]  #书的idtitle = etree.HTML(source).xpath('//h1/text()')[0]  #题目contents = ''.join(etree.HTML(source).xpath('//dd[@id="contents"]/text()'))  #内容return save_book(bid=book_id,chapter_name = title,chapter_contents=contents,chapter_url = urls)def get_contents_title(urls):source = requests.get(urls).content.decode('gbk')hrefs = etree.HTML(source).xpath('//tr/td/a/@href')if len(hrefs)>0:  #先做个判断，完整的hrefs29219318.htmlfor href in hrefs:chapter_url = urls+href  #章节的链接get_contents(chapter_url)for type in range(1,11):  #遍历first_page_url = 'http://23us.co/class/{}_1.html'.format(type)  #和链接拼接，获取所有分类下的第1页的链接max_page = get_max_page(first_page_url)  #最大页数for page in range(1,max_page+1):  #遍历起始页~最大页#'http://23us.co/class/{}_1.html'利用：split('_')[0]，根据_分割取前半部分，_页数再加.html，可得到每一页的链接every_page_url = first_page_url.split('_')[0]+'_'+str(page)+'.html'  #每一页的链接chapter_list_url = get_book_chapter_list(every_page_url)for urls in chapter_list_url:get_contents_title(urls)

二、京东商城三页数据抓取

# 首先寻找s_new.php?keyword=%E8%8B%B9%E6%9E%9C&qrst=1&sugges
# 首页链接
# https://search.jd.com/Search?keyword=%E8%8B%B9%E6%9E%9C&enc=utf-8&suggest=1.his.0.0&wq=&pvid=652f33ee3c6940a8a8a1fbb376df10f3
#
# https://search.jd.com/s_new.php?keyword=%E8%8B%B9%E6%9E%9C&qrst=1&suggest=1.his.0.0&wq=%E8%8B%B9%E6%9E%9C&stock=1&pvid=652f33ee3c6940a8a8a1fbb376df10f3&page=1click=0
# 第1页下半部本
# https://search.jd.com/s_new.php?keyword=%E8%8B%B9%E6%9E%9C&qrst=1&suggest=1.his.0.0&wq=%E8%8B%B9%E6%9E%9C&stock=1&pvid=652f33ee3c6940a8a8a1fbb376df10f3&page=2&s=26&scrolling=y&log_id=1669279334193.4075&tpl=1_M&isList=0&show_items=100026667858,100026667872,10060060919711,100026667910,100027743464,100026667880,10053064404367,100038005189,100038004339,100038004353,100014352501,100015852622,10059999976386,100008348508,100027743410,100038004397,100034710036,100008348542,10060552162473,100008348530,100038089871,100038004367,100038089809,100026809200,10051966429575,100038089887,100038325457,10058881289310,10037139655204,100038089855
# 第2页上半部分和下半部本
# https://search.jd.com/s_new.php?keyword=%E8%8B%B9%E6%9E%9C&qrst=1&suggest=1.his.0.0&wq=%E8%8B%B9%E6%9E%9C&stock=1&pvid=652f33ee3c6940a8a8a1fbb376df10f3&page=3&s=56&click=0
# https://search.jd.com/s_new.php?keyword=%E8%8B%B9%E6%9E%9C&qrst=1&suggest=1.his.0.0&wq=%E8%8B%B9%E6%9E%9C&stock=1&pvid=652f33ee3c6940a8a8a1fbb376df10f3&page=4&s=86&scrolling=y&log_id=1669279373020.5179&tpl=1_M&isList=0&show_items=10049638316236,100030101538,100044025921,10058908310957,100013068007,10047839401820,10050485280145,55503948289,100014352527,10065644760300,10049727109533,10033629792612,10044392193508,10020180486779,10057405983433,72247401303,3877149,10062570130862,10045630151903,100030101516,10055840647627,10056531996648,10059507231925,10060422239266,66753551547,10057799807780,100001660806,10060339498310,10057405983435,10037452592874
# import requests
# from lxml import etree
# headers = {# 'referer': 'https://search.jd.com/',
# 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.56'
# }
# # source1 = requests.get('https://search.jd.com/s_new.php?keyword=%E8%8B%B9%E6%9E%9C&qrst=1&suggest=1.his.0.0&wq=%E8%8B%B9%E6%9E%9C&stock=1&pvid=652f33ee3c6940a8a8a1fbb376df10f3&page=1click=0',headers=headers).text
# source2 = requests.get('https://search.jd.com/s_new.php?keyword=%E8%8B%B9%E6%9E%9C&qrst=1&suggest=1.his.0.0&wq=%E8%8B%B9%E6%9E%9C&stock=1&pvid=652f33ee3c6940a8a8a1fbb376df10f3&page=2&s=26&scrolling=y&log_id=1669279334193.4075&tpl=1_M&isList=0&show_items=100026667858,100026667872,10060060919711,100026667910,100027743464,100026667880,10053064404367,100038005189,100038004339,100038004353,100014352501,100015852622,10059999976386,100008348508,100027743410,100038004397,100034710036,100008348542,10060552162473,100008348530,100038089871,100038004367,100038089809,100026809200,10051966429575,100038089887,100038325457,10058881289310,10037139655204,100038089855',headers=headers).text
# # print(source1,source2)
# divs = etree.HTML(source2).xpath('//div[@class="gl-i-wrap"]')
# # content = etree.HTML(source).xpath('//div[@class="gl-i-wrap"]/div[3]/a/em//text()')
# print(len(divs))
# for div in divs:
#     price = div.xpath("div[2]/strong/i/text()")[0]
#     content = ''.join(div.xpath("div[3]/a/em//text()"))
#     print(price,content)import requests
from lxml import etree
import re
headers = {'referer': 'https://search.jd.com/',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.56'
}# 写循环翻页，取出3页数据
p = 1 #计数
for page in range(1,7,2):source1 = requests.get('https://search.jd.com/s_new.php?keyword=%E8%8B%B9%E6%9E%9C&qrst=1&suggest=1.his.0.0&wq=%E8%8B%B9%E6%9E%9C&stock=1&pvid=652f33ee3c6940a8a8a1fbb376df10f3&page={}click=0'.format(page),headers=headers).textdemo = re.compile('wids:\'(.*?)\'')num = demo.findall(source1)[0]print(num)source2 = requests.get('https://search.jd.com/s_new.php?keyword=%E8%8B%B9%E6%9E%9C&qrst=1&suggest=1.his.0.0&wq=%E8%8B%B9%E6%9E%9C&stock=1&pvid=652f33ee3c6940a8a8a1fbb376df10f3&page={}&s=26&scrolling=y&log_id=1669279334193.4075&tpl=1_M&isList=0&show_items={}'.format(page+1,num),headers=headers).text# print(source1,source2)divs1 = etree.HTML(source1).xpath('//div[@class="gl-i-wrap"]')divs2 = etree.HTML(source2).xpath('//div[@class="gl-i-wrap"]')# content = etree.HTML(source).xpath('//div[@class="gl-i-wrap"]/div[3]/a/em//text()')divs = divs1+divs2print(len(divs))for div in divs:price = div.xpath("div[2]/strong/i/text()")[0]content = ''.join(div.xpath("div[3]/a/em//text()"))print(price,content)print('当前第{}页'.format(p))p+=1

三、震坤行三页数据抓取

# 找到了带有口罩信息的pc，这个十post请求，提交比较重要的信息
# https://www.zkh.com/servezkhApi/search/product/pc?traceId=339496351669356422414
# {"catalogueId":"","cityCode":510100,"clp":true,"suggestModelDTO":null,"extraFilter":{"showIndustryFeatured":false,"inStock":false},"from":0,"fz":false,"keyword":"口罩","productFilter":{"brandIds":[],"properties":{}},"searchType":{"notNeedCorrect":false},"size":20,"suggestPriceOnly":false,"searchScene":1,"showType":"pic","brandId":"","rangeFilter":[{"min":"","max":"","rangeType":"price"}],"sort":0,"section":2,"spuSize":6,"needClassifiedBrand":true,"searchSceneNo":""}
# https://www.zkh.com/servezkhApi/search/product/pc?traceId=672217471669356556959
# {"catalogueId":"","cityCode":510100,"clp":true,"suggestModelDTO":null,"extraFilter":{"showIndustryFeatured":false,"inStock":false},"from":0,"fz":false,"keyword":"口罩","productFilter":{"brandIds":[],"properties":{}},"searchType":{"notNeedCorrect":false},"size":20,"suggestPriceOnly":false,"searchScene":1,"showType":"pic","brandId":"","rangeFilter":[{"min":"","max":"","rangeType":"price"}],"sort":0,"section":3,"spuSize":6,"needClassifiedBrand":true,"searchSceneNo":""}headers={'origin': 'https://www.zkh.com',
'referer': 'https://www.zkh.com/search.html?keywords=%E5%8F%A3%E7%BD%A9&hasLinkWord=1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.56',
'cookie': 'AGL_USER_ID=f5353657-0d49-42b9-a71b-4befb2d69bca; _bl_uid=h8lqhag8p5R0aswX0nX4tqgxjjqv; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2218493e308593d1-0fdf9f5bef6ccb-7d5d5475-921600-18493e3085a716%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_utm_medium%22%3A%22sem1%22%2C%22%24latest_utm_campaign%22%3A%22C-%E5%93%81%E7%89%8C%22%2C%22%24latest_utm_content%22%3A%22%E6%A0%B8%E5%BF%83%22%7D%2C%22identities%22%3A%22eyIkaWRlbnRpdHlfY29va2llX2lkIjoiMTg0OTNlMzA4NTkzZDEtMGZkZjlmNWJlZjZjY2ItN2Q1ZDU0NzUtOTIxNjAwLTE4NDkzZTMwODVhNzE2In0%3D%22%2C%22history_login_id%22%3A%7B%22name%22%3A%22%22%2C%22value%22%3A%22%22%7D%2C%22%24device_id%22%3A%2218493e308593d1-0fdf9f5bef6ccb-7d5d5475-921600-18493e3085a716%22%7D; anonymous_id=18493e308593d1-0fdf9f5bef6ccb-7d5d5475-921600-18493e3085a716; webSource=https%3A%2F%2Fwww.zkh.com%2F%3Futm_source%3Dbaidu%26utm_medium%3Dsem1%26utm_ter%3D%25E9%259C%2587%25E5%259D%25A4%25E8%25A1%258Cmro%26utm_content%3D%25E6%25A0%25B8%25E5%25BF%2583%26utm_campaign%3DC-%25E5%2593%2581%25E7%2589%258C%26sdclkid%3DAL2D152ibrDiAOqpALe_%26bd_vid%3D11204147592075616689; sensorsdata2015session=%7B%7D; citycode=%7B%22provinceName%22%3A%22%E5%9B%9B%E5%B7%9D%E7%9C%81%22%2C%22cityName%22%3A%22%E6%88%90%E9%83%BD%E5%B8%82%22%2C%22provinceCode%22%3A510000%2C%22cityCode%22%3A510100%7D; Hm_lvt_c9156633fc15595028b4d81a3571a23f=1668928445,1669356182; utmStore=%7B%22flow_type%22%3A%22%E5%85%8D%E8%B4%B9%22%2C%22%24utm_source%22%3A%22baidu%22%2C%22%24utm_medium%22%3A%22sem1%22%2C%22%24utm_content%22%3A%22%E6%A0%B8%E5%BF%83%22%2C%22%24utm_campaign%22%3A%22C-%E5%93%81%E7%89%8C%22%7D; p_pub_key=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC36XGQaO8cG2ifwZNixxe7HVyqlzELwo2DC+LwgvE0Q8rjwLXxSucPAJrYnA3C3c8/moKiVHEs9U4rciZv4jW2FyG6ivXRnHouHpSVjl83LfYbL2QwXyDurSfGSelPDgC5QCs11TgF26N3FEa4f/kvypcEfNIgkK0MHBBK7Gp4cwIDAQAB; p_pub_gr=1669356018056; zaf_ukey=d368446d79fd4ebdb66ed5c753f50530; Hm_lpvt_c9156633fc15595028b4d81a3571a23f=1669357141; JSESSIONID=E307A9B5A975766DC4EA2527E66B7FD3'
}

20221119day12作业：顶点小说全栈抓取、京东3页数据抓取、震坤行3页数据抓取相关推荐

全栈出征，京东技术基石如何为“618”大促护航？
6月18日24时,2020年的618大促落下帷幕,京东618大促期间累计下单金额达到2692亿元,创下新纪录. 与往年不同,今年618是京东核心业务全面上云的第一年,也是京东融合人工智能.云计算.物联 ...
Web全栈开发学习笔记—Part2 与服务端通信—d.在服务端将数据Alert出来
目录 REST Sending Data to the Server Changing the importance of notes Extracting communication with th ...
python全栈(中国历史天气可视化系统)
在前面博客中写过python爬虫爬取各个城市历史天气及数据可视化,感觉还是有点粗糙,本文章是借这个例子抛砖引玉,写出个更NB的python全栈项目. 此项目预览页面全国各城市天气可视化系统 scrap ...
从零到部署：用 Vue 和 Express 实现迷你全栈电商应用（三）
这篇文章中,我们将讲解 Vue 实例的 Props 和 Methods,接着我们又讲解了最常见的 Vue 模板语法,并通过实例的方式将这些模板语法都实践了一番,最后我们讲解了 Vue 组件的组合,并完 ...
从芯开始，解读华为云的全栈全场景与全球化布局
华为轮值董事长徐直军这两天,ICT业界最关注的事件莫过于HUAWEI CONNECT 2018,也就是第三届华为全联接大会了.而其中的焦点,则是华为轮值董事长徐直军发布的两款覆盖全场景人工智能的华为 ...
《Web全栈工程师的自我修养》读后感
这本书是在腾讯工作多年的高级UI工程师余果老师写的,作者熟悉前端开发,iOS开发,PHP开发和Ruby开发等,这本书是作者出的第一本书,当时看到书评说本书通熟易懂,特别适合刚入职场工作的程序员,帮助找 ...
亚信科技：新业务成高速增长源动力，吹响全栈式数智化转型号角
数据智能产业创新服务媒体 --聚焦数智 · 改变商业 2021年是亚信科技(01675.HK)在"数智化转型"方面更加满意的一年,"尤其值得高兴的是我们'三新'业务占比进 ...
Java全栈（三）数据库技术：3.数据库之JDBC上
第一章 JDBC概述之前我们学习了JavaSE,编写了Java程序,数据保存在变量.数组.集合等中,无法持久化,后来学习了IO流可以将数据写入文件,但不方便管理数据以及维护数据的关系: 后来我们学习 ...
爬虫实战：使用Selenium爬取京东宝贝信息
有些页面数据是采用Ajax获取的,但是这些Ajax接口参数比较复杂,可能会加入加密秘钥.对于这种页面,最方便的方法是通过selenium.可以用Selenium来模拟浏览器操作,抓取京东商品信息. 网 ...

20221119day12作业：顶点小说全栈抓取、京东3页数据抓取、震坤行3页数据抓取

文章目录

一、顶点小说全站抓取（代码运行下去理论上能抓就行）

二、京东商城三页数据抓取

三、震坤行三页数据抓取

20221119day12作业：顶点小说全栈抓取、京东3页数据抓取、震坤行3页数据抓取相关推荐

最新文章

热门文章