公司官网产品爬取写入ES

爬取的信息包括：公司名称，产品url，产品名称，产品类别，产品详细信息

1. 翻页模版

import os
from lxml import etree
from baseUtils import *
from hashlib import md5
import datetime_companyName = os.path.basename(__file__).replace(".py", "")
timeNow = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
_result = []def getGoodsUrls():goodsUrls = []pages = [f"http://www.ahhtdq.com/info/30816.html?page={_}" for _ in range(1, 5)]for page in pages:response = request(page)root = etree.HTML(response.text)goodsUrls += ["http://www.ahhtdq.com"+_ for _ inlist(set(root.xpath("//li[@class='iFCB-PRO-5 clearfix clone']//a/@href")))]# print(goodsUrls)return goodsUrlsdef saveContent(_url):response = request(_url)record ={}root = etree.HTML(response.text)_name = getContent(root.xpath("//div[@class='display_title']/h1/text()")).replace("\r", "").replace("\n", "").replace("\t", "")_class = getContent(root.xpath("//span[@class='ico']/a[3]/text()")).replace("\r", "").replace("\n", "").replace("\t", "")detailInfo = repr("".join(root.xpath("//div[@id='info_content']//text()"))).replace("\\r", "").replace("\\t","").replace(" ", "").replace("\\n", "").replace("\\xa0", "").replace("\'", "").replace("\\u3000", "").strip()record["goodsUrl"] = _urlrecord["campany_name"] = _companyNamerecord["category"] = _classrecord["goodsName"] = _namerecord["detailInfo"] = detailInforecord["System_update_time"] = timeNow_id = md5(_url.encode()).hexdigest()print(record)_result.append((_id,record))def start():goodsUrls = getGoodsUrls()pool = threadpool.ThreadPool(5)tasks = threadpool.makeRequests(saveContent, goodsUrls)[pool.putRequest(task) for task in tasks]pool.wait()write2ES(_result)
if __name__ == '__main__':start()

翻页模版是最容易爬取的一种情况

2. 菜单模版

import json
import re
from lxml import etree
from baseUtils import *
from hashlib import md5
import datetime
import os_companyName = os.path.basename(__file__).replace(".py","")
timeNow = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
_result = []def getGoodsUrls():goodsUrls = []index = "http://www.msensor.com.cn/Products111.html"response = request(index)root = etree.HTML(response.text)_class = ["".join(_).strip() for _ in root.xpath("//ul[@class='navUl']/li[3]//li//a/text()")]_url = ["http://www.msensor.com.cn" + _ for _ in root.xpath("//ul[@class='navUl']/li[3]//li//a/@href")]menu = dict(zip(_class, _url))# print(_class)# print(_url)# print(menu)for _class in menu:response = request(menu[_class])root = etree.HTML(response.text)pages = [int(_) for _ in root.xpath("//div[@class='pageNum']/text()")]pages.append(1)pages = max(pages)requestParam = json.loads(getContent(re.findall('requestParam:(\{.*?\})', response.text)))requestApi = "http://www.msensor.com.cn" + getContent(re.findall('url: \"(/comp/portalResProduct/list\.do\?compId=.*?)\"', response.text))for page in range(1,pages+1):requestParam['currentPage'] = pageresponse = request(requestApi, params=requestParam)root = etree.HTML(response.text)goodsUrls += [[_class, "http://www.msensor.com.cn" + _] for _ inlist(set(root.xpath("//div[@class='e_box p_images p_imagesMo']/a/@href")))]return goodsUrlsdef saveContent(tp):_class = tp[0]_url = tp[1]response = request(_url)record ={}root = etree.HTML(response.text)_name = "".join(root.xpath("//h1[@class='e_title d_Title p_Title h2']//text()")).strip()detailInfo = repr("".join(root.xpath("//div[@class='e_box d_DescribeContent p_DescribeContent']//p//text()"))).replace("\\r", "").replace("\\t","").replace(" ", "").replace("\\n", "").replace("\\xa0", "").replace("\\u3000", "").replace("\'", "").replace("返回","").strip()record["goodsUrl"] = _urlrecord["campany_name"] = _companyNamerecord["category"] = _classrecord["goodsName"] = _namerecord["detailInfo"] = detailInforecord["System_update_time"] = timeNow_id = md5(_url.encode()).hexdigest()# print(record)_result.append((_id,record))def start():goodsUrls = getGoodsUrls()# print(len(goodsUrls))pool = threadpool.ThreadPool(5)tasks = threadpool.makeRequests(saveContent, goodsUrls)[pool.putRequest(task) for task in tasks]pool.wait()write2ES(_result)if __name__ == '__main__':start()

3. list.do模版

import re
from lxml import etree
from baseUtils import *
from hashlib import md5
import datetime
import os_companyName = os.path.basename(__file__).replace(".py","")
timeNow = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
_result = []def getGoodsUrls():goodsUrls = []index = "http://www.jshhby.com/product/5/"response = request(index)root = etree.HTML(response.text)_class = ["".join(_).strip() for _ in root.xpath("//h3[@class='yjTitle link']/a//text()")]_url = ["http://www.jshhby.com" + _ for _ in root.xpath("//h3[@class='yjTitle link']/a/@href")]menu = dict(zip(_class, _url))print(_class)print(_url)print(menu)# for _class in menu:#     response = request(menu[_class])#     root = etree.HTML(response.text)#     pages = [int(_) for _ in root.xpath("//div[@class='pageNum']/text()")]#     pages.append(1)#     pages = max(pages)#     requestParam = json.loads(getContent(re.findall('requestParam:(\{.*?\})', response.text)))#     requestApi = "http://www.jshhby.com" + getContent(#         re.findall('url: \"(/comp/portalResProduct/list\.do\?compId=.*?)\"', response.text))#     for page in range(1,pages+1):#         requestParam['currentPage'] = page#         response = request(requestApi, params=requestParam)#         root = etree.HTML(response.text)#         goodsUrls += [[_class, "http://www.jshhby.com" + _] for _ in#                       list(set(root.xpath("//div[@class='e_box p_Product']//a/@href")))]# return goodsUrlsdef saveContent(tp):_class = tp[0]_url = tp[1]response = request(_url)record ={}root = etree.HTML(response.text)_name = "".join(root.xpath("//h1[@class='e_title d_Title p_Title h2']//text()")).strip()detailInfo = repr("".join(root.xpath("//div[@class='pro_content']//p//text()"))).replace("\\r", "").replace("\\t","").replace(" ", "").replace("\\n", "").replace("\\xa0", "").replace("\\u3000", "").replace("\'", "").replace("返回","").strip()record["goodsUrl"] = _urlrecord["campany_name"] = _companyNamerecord["category"] = _classrecord["goodsName"] = _namerecord["detailInfo"] = detailInforecord["System_update_time"] = timeNow_id = md5(_url.encode()).hexdigest()print(record)_result.append((_id,record))def start():goodsUrls = getGoodsUrls()# print(len(goodsUrls))# pool = threadpool.ThreadPool(5)# tasks = threadpool.makeRequests(saveContent, goodsUrls)# [pool.putRequest(task) for task in tasks]# pool.wait()# write2ES(_result)if __name__ == '__main__':start()

封装模块baseUtils.py：

import requests
import threadpool
from elasticsearch import Elasticsearch, helpers
from retrying import retry
import urllib3
urllib3.disable_warnings()_headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36',
}_proxies = {'http': 'http://192.168.100.210:3128','https': 'http://192.168.100.210:3128',
}@retry(stop_max_attempt_number=20)
def request(url,method="get", header=None, proxies=None, verify=False, _data=None, params=None):if proxies is None:proxies = _proxiesif header is None:header = _headersif method=="get":response = requests.get(url,headers=header, proxies=proxies,params=params,verify=verify)response.encoding = response.apparent_encodingreturn responseelif method=="post":response = requests.post(url,headers=header, proxies=proxies,data=_data,params=params,verify=verify)response.encoding = response.apparent_encodingreturn responsedef getContent(item,default=""):return item[0] if item else defaultdef write2ES(tps):es_config = Elasticsearch(hosts=['http://192.168.100.230:9200'], http_auth=None, timeout=60)Esaction = []for _id,record in tps:Esaction.append({"_op_type": 'update',"_index": 'crawler_company_product',"_type": 'index',"_id": _id,"_source": {"doc": record,"doc_as_upsert": True}})if len(Esaction) >= 1000:helpers.bulk(es_config, Esaction)Esaction = []helpers.bulk(es_config, Esaction)

公司官网产品爬取写入ES相关推荐

scrapy爬虫实战——米哈游官网数据爬取
项目总览前言项目建立爬虫代码编写爬虫运行导出到Excel 后记前言 scrapy是一个强大的爬虫框架,熟练地使用它几乎可以随心所欲地抓取任何网页上想要获得的数据.笔者初学爬虫,需要项目练手 ...
爬虫实战1——校园官网内容爬取
Python使用requests爬取网页简介一. 自动翻页的实现 1.观察: 2.代码实现: 二.时间的限定 1.遇到的困难: 2.解决过程: 三.阅读数的爬取 1.遇到的困难: 2.解决过程: ...
企业网站+Axure企业官网通用模板+公司官网通用模板+web端高保真原型+门户官网+物流企业+门户网站+服务中心+产品中心+新闻中心+帮助中心+企业官网+公司官网+公司网站+登录注册+高保真交互
企业网站+Axure企业官网通用模板+公司官网通用模板+web端高保真原型+门户官网+物流企业+门户网站+服务中心+产品中心+新闻中心+帮助中心+企业官网+公司官网+公司网站+Axure原型+rp原型 ...
web设计网页规划与设计：企业电子网站设计——简单的电子产品公司官网模板(5页)HTML+CSS+JavaScript
HTML5期末大作业:动漫网站设计--简单的电子产品公司官网模板html(5页) HTML+CSS+JavaScript 学生DW网页设计作业成品 web课程设计网页规划与设计计算机毕设网页设计源码 ...
【selenium爬取】最新版可用某网职位爬取数据
文章目录一.某网职位爬取 1.1 selenium的使用 1,2 开始爬取一.某网职位爬取共计8000组数据,需要数据和代码的,请到以下链接获取: https://download.csdn.n ...
集思录REITs基金数据python爬取写入EXCEL表
本文主要讲述REITs基金相关内容, 封闭基金数据获取参见: 集思录封闭基金数据python爬取写入excel表国债数据获取参见: 和讯网债券数据Python爬取保存成CSV文件之一 1.什么是RE ...
分享一个简单好看的科技公司官网模板-纯HTML+CSS
分享一个简单好看的科技公司官网模板-纯HTML+CSS 首页: 首页的HTML源码部分: <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Tr ...
知网爬虫——爬取某个主题下的文章标题以及发表时间
前几天帮朋友做了个知网的爬虫,爬取了"新闻传播"主题下的文章标题及发表时间:自己拖拖拉拉写了2天才写完,自己还是太弱了.个人认为这个作为一个练手的爬虫小项目还是不错的,于是写了主要 ...
公司官网前端开发经验分享
公司官网的重要性和特殊性不言而喻. 很多前端都会经历这个项目,笔者有幸在所就职的每家公司都负责过官网的前端开发工作.开发中遇到过很多问题,反思起来值得和大家一起分享,以减少或者减低大家可能遇到的问题. ...

公司官网产品爬取写入ES

公司官网产品爬取写入ES相关推荐

最新文章

热门文章