python爬虫爬取拉勾网招聘信息

话不多说，直接上代码！

import math
import random
import requests
from pyquery import PyQuery as pq
from urllib3.exceptions import MaxRetryError
import pandas as pdPROXY_POOL_URL = 'http://localhost:5555/random'  # 使用了https://github.com/Python3WebSpider/ProxyPool代理池
PROXY_POOL_SIZE = 5
proxies = None
job_info_file = "C:/Users/86135/Desktop/job_info.csv"
company_info_file = "C:/Users/86135/Desktop/company_info.csv"
comment_info_file = "C:/Users/86135/Desktop/comment_info.csv"'''
根据筛选条件组合并返回请求搜索页面的page_url和会返回职位信息json的ajax_url
'''def combine_page_and_ajax_url(job_type: str, **condition):page_url = 'https://www.lagou.com/jobs/list_' + job_typeajax_url = r'https://www.lagou.com/jobs/positionAjax.json'# 工作城市city = '全国'if 'city' in condition:city = condition['city']page_url = page_url + '?city=' + cityajax_url = ajax_url + '?city=' + city# 排序方式px = 'default'if 'px' in condition:px = condition['px']page_url = page_url + '&px=' + pxajax_url = ajax_url + '&px=' + px# 公司规模 1 少于15人， 2 15-50人以此类推若多选,如1，2都选，就是_1_2if 'gm' in condition:page_url = page_url + '&gm=' + condition['gm']ajax_url = ajax_url + '&gm=' + condition['gm']# 融资阶段if 'jd' in condition:page_url = page_url + '&jd=' + condition['jd']ajax_url = ajax_url + '&jd=' + condition['jd']# 工作性质：全职，兼职，实习if 'gx' in condition:page_url = page_url + '&gx=' + condition['gx']ajax_url = ajax_url + '&gx=' + condition['gx']# 工作经验 isSchoolJob 1 应届if 'gj' in condition:page_url = page_url + 'gj=' + condition['gj']ajax_url = ajax_url + 'gj=' + condition['gj']# 学历if 'xl' in condition:page_url = page_url + '&xl=' + condition['xl']ajax_url = ajax_url + '&xl=' + condition['xl']# 行业if 'hy' in condition:page_url = page_url + '&hy=' + condition['hy']ajax_url = ajax_url + '&hy=' + condition['hy']page_url = page_url + '#filterBox'ajax_url = ajax_url + '&needAddtionalResult=false&isSchoolJob=1'# 解决中文乱码page_url = page_url.encode("utf-8")ajax_url = ajax_url.encode("utf-8")print(page_url)print(ajax_url)return page_url, ajax_url'''
获取符合筛选条件的工作页面url和公司页面url
'''def get_job_and_company_urls(page_url, ajax_url, job_type):# 需要获取的工作和公司数据jobs_id = []companies_id = []jobs_name = []jobs_advantage = []jobs_salary = []jobs_publish_time = []companies_name = []companies_labels = []companies_size = []show_id = NoneJOBS_COUNT_ONE_PAGE = 15remain_page_count = -1page_number = 1first = 'true'user_agent = get_user_agent()session = requests.session()r,session=request_page_result(page_url,session,user_agent)while remain_page_count != 0:# 请求的数据data = {'first': first,'pn': page_number,'kd': job_type}ajax_result=request_ajax_result(ajax_url,page_url,session,user_agent,data)result_json = ajax_result.json()position_result = result_json['content']['positionResult']# 第一次进入循环，获取到json中的总工作数totalCountif remain_page_count == -1:show_id = result_json['content']['showId']print('showId ', show_id)print("type of result", type(position_result))total_count = position_result['totalCount']# 没有符合条件的工作，直接返回if total_count == 0:returnremain_page_count = math.ceil(total_count / JOBS_COUNT_ONE_PAGE)result = position_result['result']for item in result:position_id = item['positionId']job_name = item['positionName']job_advantage = item['positionAdvantage']job_salary = item['salary']publish_time = item['createTime']company_id = item['companyId']company_name = item['companyFullName']company_labels = item['companyLabelList']company_size = item['companySize']jobs_id.append(position_id)jobs_name.append(job_name)jobs_advantage.append(job_advantage)jobs_salary.append(job_salary)jobs_publish_time.append(publish_time)companies_name.append(company_name)companies_id.append(company_id)companies_labels.append(company_labels)companies_size.append(company_size)remain_page_count = remain_page_count - 1page_number = page_number + 1first = 'false'# 存储基本工作信息，公司信息到csv文件中job_df = pd.DataFrame({'job_id': jobs_id, 'job_name': jobs_name, 'job_advantage': jobs_advantage, 'salary': jobs_salary,'publish_time': jobs_publish_time, 'company_id': companies_id})company_df = pd.DataFrame({'company_id': companies_id, 'company_name': companies_name, 'labels': companies_labels,'size': companies_size})job_df.to_csv(job_info_file, mode='w', header=True, index=False)company_df.to_csv(company_info_file, mode='w', header=True, index=False)return show_id'''
根据得到的job_id访问工作页面并存储工作信息到csv文件中
'''def get_and_store_job_info(show_id: str):jobs_detail = []PAGE_SIZE = 500comments_content = []comments_time = []users_id = []company_scores = []interviewer_scores = []describe_scores = []comprehensive_scores = []useful_counts = []tags = []# 从csv文件读取job_id并与show_id组合成工作页面urldf = pd.read_csv(job_info_file)jobs_id = df['job_id']for job_id in jobs_id:user_agent = get_user_agent()session = requests.session()job_page_url = 'https://www.lagou.com/jobs/' + str(job_id) + '.html?show=' + show_id# 访问工作页面获取职位描述和面试评价r = request_page_result(url=job_page_url, session=session, user_agent=user_agent)doc = pq(r.text)job_detail = doc('#job_detail > dd.job_bt > div').text()print("job_detail", job_detail)jobs_detail.append(job_detail)# 获取面试评价review_ajax_url = 'https://www.lagou.com/interview/experience/byPosition.json'data = {'positionId': job_id,'pageSize': PAGE_SIZE,}response = request_ajax_result(review_ajax_url, job_page_url, session, user_agent, data)response_json = response.json()print("response json", response_json)if response_json['content']['data']['data']['totalCount'] != 0:result = response_json['content']['data']['data']['result']for item in result:comment_content = item['content']comment_time = item['createTime']user_id = item['userId']company_score = item['companyScore']interviewer_score = item['interviewerScore']describe_score = item['describeScore']comprehensive_score = item['comprehensiveScore']useful_count = item['usefulCount']tag = item['tags']print("content",comment_content)comments_content.append(comment_content)comments_time.append(comment_time)users_id.append(user_id)company_scores.append(company_score)interviewer_scores.append(interviewer_score)describe_scores.append(describe_score)comprehensive_scores.append(comprehensive_score)useful_counts.append(useful_count)tags.append(tag)j_ids=[]j_ids.extend(job_id for i in range(len(comments_content)))comment_df = pd.DataFrame({'job_id': j_ids,  'content': comments_content})comment_df.to_csv(comment_info_file, mode='a', header=False, index=False)# 将获取到的职位描述面试评价存储到csv文件中df['job_detail'] = jobs_detaildf.to_csv(job_info_file)def request_page_result(url, session, user_agent):try:r = session.get(url, headers={'User-Agent': user_agent})except MaxRetryError as e:print(e)user_agent = get_user_agent()r = session.get(url, headers={'User-Agent': user_agent}, proxies=get_proxy())return r,sessiondef request_ajax_result(ajax_url, page_url, session, user_agent, data):try:result = session.post(ajax_url, headers=get_ajax_header(page_url, user_agent), data=data,allow_redirects=False)except MaxRetryError as e:print(e)user_agent = get_user_agent()result = session.post(ajax_url, headers=get_ajax_header(page_url, user_agent),proxies=get_proxy(),data=data, allow_redirects=False)return result'''
访问拉勾页面所使用的header
'''def get_page_header():page_header = {'User-Agent': get_user_agent()}return page_header'''
访问拉勾ajax json所使用的header
'''def get_ajax_header(url, user_agent):ajax_header = {'Accept': 'application/json, text/javascript, */*; q=0.01','Accept-Encoding': 'gzip, deflate, br','Accept-Language': 'zh-CN,zh;q=0.9','Connection': 'keep-alive','Content-Length': '25','Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8','Host': 'www.lagou.com','Origin': 'https://www.lagou.com','Referer': url,'Sec-Fetch-Mode': 'cors','Sec-Fetch-Site': 'same-origin','User-Agent': user_agent,'X-Anit-Forge-Code': '0','X-Anit-Forge-Token': 'None','X-Requested-With': 'XMLHttpRequest'}return ajax_headerdef get_user_agent():user_agent = ["Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1""Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 ""Safari/536.11","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"]return random.choice(user_agent)def get_proxy():response = requests.get(PROXY_POOL_URL)if response.status_code == 200:print(response.text)return response.textif __name__ == '__main__':page_url, ajax_url = combine_page_and_ajax_url(job_type='java', city='杭州', gx='全职', xl='本科', hy='移动互联网')showId = get_job_and_company_urls(page_url, ajax_url, 'java')get_and_store_job_info(showId)

运行结果：

python爬虫爬取拉勾网招聘信息相关推荐

2.简单爬虫————爬取拉勾网招聘信息(一)
该文章仅供学习,如有错误,欢迎指出 1.开始创建一个项目 mkdir lagou 2.进入到文件夹下创建python3的虚拟环境 pipenv install scrapy 3.进入pipenv 下使 ...
python爬虫爬取拉勾网职业信息
一.前言最近想做一份关于拉勾网数据分析类职业的报告,便顺手写了个简单的爬虫,记录分享如下. 二.思路整理 1.首先我们打开拉勾网,并搜索""数据分析"",显示 ...
Python爬虫实战之一 - 基于Requests爬取拉勾网招聘信息，并保存至本地csv文件
Python爬虫实战之二 - 基于Requests抓取拉勾网招聘信息 ---------------readme--------------- 简介:本人产品汪一枚,Python自学数月,对于小白,本 ...
【python爬虫02】使用Scrapy框架爬取拉勾网招聘信息
使用Scrapy框架爬取拉勾网招聘信息最近接触了Scrapy爬虫框架,简单写了个爬虫爬取拉钩网的招聘信息,加深对Scrapy框架的理解,不得不说Scrapy框架其实还是蛮方便的,就像爬虫流水线一样, ...
Python搭建代理池爬取拉勾网招聘信息
先来看一张图了解下爬虫实现功能多线程爬取拉勾网招聘信息维护代理 ip 池搭建 node 服务器 Taro 使用 echarts 做数据分析 1.多线程爬取拉勾网招聘信息 Tip:涉及知识 1. ...
java爬虫拉勾网_[Java教程]node.js爬虫爬取拉勾网职位信息
[Java教程]node.js爬虫爬取拉勾网职位信息 0 2017-03-14 00:00:21 简介用node.js写了一个简单的小爬虫,用来爬取拉勾网上的招聘信息,共爬取了北京.上海.广州.深圳 ...
什么你还不知道招聘信息，小唐来教你——最新2021爬取拉勾网招聘信息（一）
文章目录前言一.准备我们的库二.分析分析三. 代码四.数据展示小唐的心路历程上一篇:没有啦! 下一篇:什么你还不知道招聘信息,小唐来教你--最新2021爬取拉勾网招聘信息(二) 前言有 ...
什么你还不知道招聘信息，小唐来教你——最新2021爬取拉勾网招聘信息（二）
文章目录前言一.准备我们的库二.数据清洗三.核密度图及词云制作四.完整代码五.扩展上一篇:什么你还不知道招聘信息,小唐来教你--最新2021爬取拉勾网招聘信息(一) 下一篇:没有拉! 前 ...
深圳python爬虫培训南山科技园钽电容回收_记一次python 爬虫爬取深圳租房信息的过程及遇到的问题...
为了分析深圳市所有长租.短租公寓的信息,爬取了某租房公寓网站上深圳区域所有在租公寓信息,以下记录了爬取过程以及爬取过程中遇到的问题: 爬取代码: import requests from reques ...
python解决租房问题_记一次python 爬虫爬取深圳租房信息的过程及遇到的问题
为了分析深圳市所有长租.短租公寓的信息,爬取了某租房公寓网站上深圳区域所有在租公寓信息,以下记录了爬取过程以及爬取过程中遇到的问题: 爬取代码: import requests from reques ...

python爬虫爬取拉勾网招聘信息

python爬虫爬取拉勾网招聘信息

python爬虫爬取拉勾网招聘信息相关推荐

最新文章

热门文章