爬虫：boss直聘自动投递简历+数据获取

boss直聘web端的反爬做得还是不错的，需要通过下载js文件结合浏览器生成token，一时也没太搞明白，所以通过selenium获取数据，顺手把投投简历。
即便是使用selenium也只能获取部分数据，因为获取数据的数量被限制，无论是搜索或是在公司（300条）都限制和获取数据的数目，因此这里有个方向是可以根据推荐进行获取数据。
还有就是访问长了还会提示要验证，而且那个验证也是比较复杂，人工看有时候都得看半天。

from selenium import webdriver
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from bs4 import BeautifulSoup
import redis
import re
import json
import random
import tracebackdef get_info(text, cur_url):soup = BeautifulSoup(text, 'lxml')all_a = soup.findAll('a', class_='company-info')for a in all_a:try:financing, industry = re.findall('<p>(.*?)<span class="vline"></span>(.*?)</p>', str(a.find('p')))[0]data = dict(href=a['href'],ka=a['ka'],img=a.find('img')['src'],company=a.find('h4').text,financing=financing,industry=industry,referer=cur_url)r.hset("companies_hash", data['company'], json.dumps(data))r.sadd('companies_set', data['company'])r.sadd('companies_set_bak', data['company'])print(data)except Exception as e:print(e)def login():driver.get("https://login.zhipin.com/?ka=header-login")time.sleep(1)driver.find_element_by_xpath("//body/div[@id='wrap']/div[2]/div[1]/div[2]/div[2]").click()wait = WebDriverWait(driver, 30)  # 显式等待wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'jobs-list')))def sleep():time.sleep(random.choice(range(6, 12)))def search_city(city):driver.find_element_by_link_text(city).click()time.sleep(1)for i in range(8):print(f"-------------------------------------city:{city}-----page: {i}-----------------------------------------")if i > 0:target = driver.find_element_by_css_selector("a[ka='page-next']")driver.execute_script("arguments[0].scrollIntoView();", target)target.click()text = driver.page_sourcecur_url = driver.current_urlget_info(text, cur_url)sleep()def get_jobs(text, company):soup = BeautifulSoup(text, 'lxml')all_li = soup.find('div', class_='job-list').findAll('li')for li in all_li:try:a = li.find('a')data = {key: a[key] for key in ['data-jid', 'href', 'ka']}for key in ['job-title', 'job-area', 'job-pub-time']:data[key] = li.find('span', class_=key).textdata['salary'] = li.find('span', class_='red').textdata['work_years'], data['education'] = re.findall(re.compile('</span>(.*?)<em class="vline"></em>(.*?)</p>', re.S), str(li.find('p')))[0]data['avator'], data['hr'], data['hr_positon'] = re.findall(re.compile('<img src="(.*?)\?x-oss-process=image/resize,w_40,limit_0"/>(.*?)<em class="vline"></em>(.*?)</h3>',re.S),str(li.find('div', class_='info-publis')))[0]for key in ['work_years', 'education', 'hr', 'hr_positon', 'job-area']:data[key] = re.sub('[\[\]]', '', data[key].strip())data['company'] = companyprint(f"---------{data['job-title']} --- {data['job-area']} --- {company}-------")r.hset("jobs_hash", data['data-jid'], json.dumps(data))r.sadd('jobs_set', data['data-jid'])r.sadd('jobs_set_bak', data['data-jid'])except Exception as e:print(e)def company_info():info = json.loads(r.hget('companies_hash', r.spop('companies_set')))company = info['company']href = info["href"].replace("gongsi", "gongsir").rsplit(".", 1)[0]url = f'{base_url}{href}_100000.html?ka={info["ka"]}'driver.get(url)time.sleep(random.choice([1, 2, 3]))index = 1while True:print(f'------------- 公司：{company} --- page：{index}  ------------------------------')page = driver.page_sourceif '点击进行验证' in page:print(f'-----------------------  点击进行验证  ------------------------------')wait = WebDriverWait(driver, 12 * 60 * 60)  # 显式等待wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'company-banner')))continueget_jobs(page, company)try:target = driver.find_element_by_css_selector('a[class="next"]')driver.execute_script("arguments[0].scrollIntoView();", target)target.click()index += 1sleep()except Exception as e:breakdef deal(info, key, func, soup):try:info[key] = func(soup)except Exception as e:exstr = traceback.format_exc()print(exstr)def save_info(base_name, key, data):r.hset(f"{base_name}_hash", data[key], json.dumps(data))r.sadd(f"{base_name}_set", data[key])r.sadd(f"{base_name}_set_bak", data[key])def get_promotion_jobs(lst):for job in lst:try:job_href = job.find('div', class_='name').find('a')['href']company = job.find('div', class_='info-primary').find('p', class_='gray')company_name = company.find('a').text.strip()job_info = dict(job_name=job.find('div', class_='name').find('a').text.rsplit(" ", 1)[0],job_ka=job.find('div', class_='name').find('a')['ka'],salary=job.find('div', class_='name').find('span', class_='red').text,job_id=job_href.rsplit('/', 1)[1].split('.html')[0],job_href=job_href,company_name=company_name,job_city=company.text.rsplit('·', 1)[1])save_info("boss_jobs", "job_id", job_info)companies_info = dict(company_name=company_name,company_ka=company.find('a')['ka'],company_href=company.find('a')['href'],)save_info("boss_companies", "company_name", companies_info)except Exception as e:exstr = traceback.format_exc()print(exstr)def get_job_info(text, data):soup = BeautifulSoup(text, 'lxml')info = {}for k, v in (('job_id', 'data-jid'), ('job_href', 'href'), ('job_ka', 'ka'), ('job_name', 'job-title'),('job_city', 'job-area'), ('update_date', 'job-pub-time'), ('salary', 'salary'),('experience', 'work_years'), ('education', 'education'), ('hr_avatar', 'avator'),('hr_name', 'hr'), ('hr_position', 'hr_positon'), ('company_name', 'company')):info[k] = data[v]deal(info, "job_describe", lambda x: x.find('div', class_="job-sec").find('div', class_='text').text.strip(), soup)deal(info, "job_tags", lambda x: [i.text for i in x.find('div', class_='job-tags').findAll('span')], soup)deal(info, "update_date", lambda x: x.find('div', class_='sider-company').find('p', class_='gray').text, soup)deal(info, "job_location",lambda x: x.find('div', class_='job-location').find('div', class_='location-address').text, soup)try:info['experience'], info['education'] = re.findall('</a><em class="dolt"></em>(.*?)<em class="dolt"></em>(.*?)</p>',str(soup.find('div', class_='info-primary').find('p')))[0]detail_op = soup.find('div', class_='job-detail').find('div', class_='detail-op')deal(info, "hr_name", lambda x: x.find('h2', class_='name').text, detail_op)deal(info, "hr_position", lambda x: x.find('p', class_='gray').text, detail_op)deal(info, "hr_avatar", lambda x: x.find('img')['src'].split('?')[0], detail_op)save_info("boss_jobs", "job_id", info)promotion_jobs = soup.find('div', class_='promotion-job').findAll('li')get_promotion_jobs(promotion_jobs)except Exception as e:exstr = traceback.format_exc()print(exstr)def communicate(url, info):driver.get(url)time.sleep(1)page = driver.page_sourceif '点击进行验证' in page:print(f'-----------------------  点击进行验证  ------------------------------')wait = WebDriverWait(driver, 12 * 60 * 60)  # 显式等待wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'job-status')))time.sleep(1)try:get_job_info(page, info)driver.find_element_by_css_selector('a[class="btn btn-startchat"]').click()time.sleep(1)driver.find_element_by_css_selector('span[class="btn btn-sure"]').click()except Exception as e:exstr = traceback.format_exc()print(exstr)def run(mode, is_login=True):if is_login:login()if mode == 'city':driver.find_element_by_link_text("公司").click()time.sleep(1)for city in cities:search_city(city)elif mode == 'company':while r.scard('companies_set') != 0:if len(driver.window_handles) > 1:driver.switch_to_window(driver.window_handles[0])company_info()sleep()elif mode == 'send':while r.scard('job_data') != 0:job_id = r.srandmember("job_data")info = json.loads(r.hget('jobs_hash', job_id))print(f'------------- 公司：{info["company"]} --- 工作：{info["job-title"]}  ------------------------------')communicate(f"{base_url}{info['href']}", info)r.srem('job_data', job_id)time.sleep(random.choice([10, 15]))time.sleep(2)if __name__ == '__main__':r = redis.Redis(password='123zxc')cities = ['北京', '杭州', '天津',  '苏州', '上海']  base_url = 'https://zhipin.com'driver = webdriver.Chrome("/mnt/2D97AD940A9AD661/python_project/boss/chromedriver")driver.maximize_window()run('send', True)sleep()driver.quit()

爬虫：boss直聘自动投递简历+数据获取相关推荐

BOSS直聘自动投简历聊天机器人的实现过程
这两年疫情,公司业务越来越差,必须得准备后路了,每天睡前都会在直聘上打一遍招呼,一直到打哈欠有睡意为止...,这样持续了一周,发现很难坚持,身为一名资深蜘蛛侠,怎么能这样下去呢?于是便萌生了对BOSS ...
最新puppeteer爬虫boss直聘招聘公司及职位信息
失踪人口回归了!由于一直在爬虫,遇到的坑比较多,先大致汇报下 1.获取全国所有企业 2.通过企查查补充企业信息 3.爬虫boss直聘获取公司及职位信息本次先说爬虫boss直聘目的: 获取指定城市 ...
goquery爬虫Boss直聘信息
title: goquery爬虫Boss直聘信息 tags: go,goquery author: Clown95 背景 Hello小伙伴们,在之前的文章中,我们对go的爬虫库goquery进行了简单 ...
python后端开发面试简历_【Atman爱特曼Python后端工程师实习生面试】我是在boss直聘上投的简历，从简历筛选到拿到offer一共有如下步骤-看准网...
我是在boss直聘上投的简历,从简历筛选到拿到offer一共有如下步骤: 1.简历筛选:职位上有985/211优先,虽然我是双非,但我还是给HR发了消息.第二天HR就要了简历,然后等了4天简历过了(能 ...
boss直聘自动招聘助手
此脚本用appium来写的一个boss直聘自动化招聘工具,容易上手,有着遍历人才库,匹配自己设定的岗位关键字,岗位关键字可以在excel里面输入,代码读取excel,然后自动匹配在线简历,匹配成功则私 ...
boss直聘账号异常登不上_【技术变现小技巧】-- Boss直聘批量求简历
今天给大家分享内容的基本思路和之前BOSS直聘批量打招呼的那篇差不多,也是使用浏览器的调试器,通过js代码来实现功能. STEP 1进入BOSS直聘企业版页面 STEP 2勾选未读信息其实,未读消 ...
【技术变现小技巧】-- Boss直聘批量求简历
今天给大家分享内容的基本思路和之前BOSS直聘批量打招呼的那篇差不多,也是使用浏览器的调试器,通过js代码来实现功能. STEP 1 进入BOSS直聘企业版页面 STEP 2 勾选未读信息 ...
【软件测试】从企业版BOSS直聘,看求职简历，你没被面上是有原因的
有时候,不是我们不可以,而是我们连面试的机会都没有.希望这篇文章能帮助大家找工作,有一个展示自己的机会. 最近负责测试的面试工作,在等HR推简历的时候害怕错过优秀的伙伴,找HR拿到了公司在BOSS的账 ...
boss直聘改回系统头像_BOSS 直聘找工作，消息却已读不回？| 在线求职5条避坑指南...
前几天在半撇私塾的求职群里,一个同学反馈:在 BOSS 直聘求职的时候,为什么总是被「已读不回」呢?就连逛豆瓣的上班小组,都能遇到这样的反馈: 为什么会出现「已读不回」的情况呢?关于这个问题的答案,在 ...

爬虫：boss直聘自动投递简历+数据获取

爬虫：boss直聘自动投递简历+数据获取相关推荐

最新文章

热门文章