模拟浏览器整理电影榜单

（学习使用）

步骤

主要问题是滑块验证码和加密字体：

1、滑块验证主要用的是ddddocr包:

(1) 找到iframe并进入ifame

(2) 下载滑块图片和背景图片

(3) 将滑块图片和背景图片传入ddddocr 得到滑动距离

(4) 设置滑动路径

(5) 根据滑动路径滑动按钮

2、加密字体主要用的是TTFont和muggle_ocr包

(1) 根据源代码找到woff文件并下载

(2) 将woff文件分割保存为图片

(3) 使用muggle_ocr 识别保存的图片

(4) 根据识别的结果构建解密字典

(5) 爬取榜单并根据字典进行解密

导入模块

import urllib
import re
import os
import shutil
from lxml import etree
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
import time
import muggle_ocr

验证码

需要进入iframe

def into_iframe(wait):iframe = wait.until(EC.presence_of_all_elements_located((By.XPATH, '//iframe')))# 验证码需要转到framewait.until(EC.frame_to_be_available_and_switch_to_it(iframe[0]))

计算滑动距离

# 计算需要滑动的距离
import ddddocrdef use_ddddocr(left=26):det = ddddocr.DdddOcr(det=False, ocr=False)with open('./验证码/block.jpg', 'rb') as f:target_bytes = f.read()with open('./验证码/background.jpg', 'rb') as f:background_bytes = f.read()res = det.slide_match(target_bytes, background_bytes, simple_target=True)print(res)res['target_y'] += left  # 加上左边界distance = res['target'][0] / 2 - res['target_y']  # 渲染的图片和原始图片相比 差不多是0.5倍return distance

生成轨迹

# 轨迹
def get_track(distance):# 移动轨迹track = []# 当前位移current = 0# 减速阈值(到达某个距离后减速)mid = distance * 0.6# 计算间隔t = 0.2# 初速度v = 0# 循环 判断是否超出滑动距离while current < distance:if current < mid:# 加速度为正70a = 70  # 40else:# 加速度为负110a = -110# 初速度为v0v0 = v# 单前速度 v = v0 + atv = v0 + a * t# 移动距离 x = v0t + 0.5 * a * t^2 距离公式move = v0 * t + 0.5 * a * (t ** 2)# 滑块往回拉时也退出if move < 0:break# 当前位移current += move# 加入轨迹track.append(round(move))return track

移动滑块

from selenium.webdriver import ActionChains# 移动滑块
def move_to_gap(slider, track):ActionChains(driver).click_and_hold(slider).perform()  # click_and_hold() 按住底部滑块for x in track:ActionChains(driver).move_by_offset(xoffset=x, yoffset=0).perform()time.sleep(0.5)# release() 松开鼠标ActionChains(driver).release().perform()

下载验证码图片并调用滑块接口

# 看是否有验证码def verification(wait):# 背景图urlslideBg = wait.until(EC.presence_of_element_located((By.XPATH, '//img[@id="slideBg"]')))slideBg_url = slideBg.get_attribute('src')# 滑块图urlslideBlock = wait.until(EC.presence_of_element_located((By.XPATH, '//img[@id="slideBlock"]')))slideBlock_url = slideBlock.get_attribute('src')# 左边界style = wait.until(EC.presence_of_element_located((By.XPATH, '//img[@id="slideBlock"]'))).get_attribute('style').replace(' ', '').replace('px', '')[:-1]  # 最后一个;要去掉style_dict = {}for i in style.split(';'):splip_list = i.split(':')style_dict[splip_list[0]] = int(splip_list[1])left = style_dict['left']# 移动按钮slider = wait.until(EC.presence_of_element_located((By.XPATH, '//div[@class="tc-drag-thumb"]')))# 下载图片os.mkdir('./验证码') if not os.path.exists('./验证码') else 1  # 创建文件夹urllib.request.urlretrieve(slideBg_url, './验证码/background.jpg')  # 下载背景图urllib.request.urlretrieve(slideBlock_url, './验证码/block.jpg')  # 下载滑块图distance = use_ddddocr(left)  # 获得需要滑动的距离print(distance)track = get_track(distance)  # 获得轨迹move_to_gap(slider, track)  # 移动滑块del_img = False  # 是否删除下载的图片if del_img:shutil.rmtree('./验证码')

加密字体

urllib response

def get_response(url, headers):request = urllib.request.Request(url, headers=headers)response = urllib.request.urlopen(request)return response# 获得二进制的html
def get_html(url, headers):request = urllib.request.Request(url, headers=headers)response = urllib.request.urlopen(request)html = response.read()return html

下载woff文件

# 获得woff文件
def get_woff(html, headers):file_name = re.findall(r'//.+(/\w+\.woff)', html)[0][1:]  # woff路径print(file_name)# 下载字体的woff文件stone_url = "http://s3plus.meituan.net/v1/mss_73a511b8f91f43d0bdae92584ea6330b/font/" + file_namehtml = get_html(stone_url, headers)  # 使用urllib库访问woff网站os.mkdir('fonts') if not os.path.exists('fonts') else 1# 把字体文件写入本地with open('fonts/' + file_name, 'wb') as f:f.write(html)return file_name

woff文件分割保存为图片

# 字体拆分保存为图片工具class ReportLabPen(BasePen):def __init__(self, glyphSet, path=None):BasePen.__init__(self, glyphSet)if path is None:path = Path()self.path = pathdef _moveTo(self, p):(x, y) = pself.path.moveTo(x, y)def _lineTo(self, p):(x, y) = pself.path.lineTo(x, y)def _curveToOne(self, p1, p2, p3):(x1, y1) = p1(x2, y2) = p2(x3, y3) = p3self.path.curveTo(x1, y1, x2, y2, x3, y3)def _closePath(self):self.path.closePath()def woff2_image_service(font_name, image_path, fmt="png"):os.mkdir(image_path) if not os.path.exists(image_path) else 1# 获取字体font = TTFont(font_name)gs = font.getGlyphSet()glyph_names = font.getGlyphNames()print('glyph_names_len', glyph_names)for i in glyph_names:if i[0] == '.':  # 跳过'.notdef', '.null'continueg = gs[i]pen = ReportLabPen(gs, Path(fillColor=colors.red, strokeWidth=5))g.draw(pen)w, h = g.width, g.width * 2g = Group(pen.path)g.translate(0, 200)d = Drawing(w, h)d.add(g)image_file = image_path + "/" + i + ".png"renderPM.drawToFile(d, image_file, fmt)img = cv2.imread(image_file, -1)height, width = img.shape[:2]# 缩小图像size = (int(width * 0.05), int(height * 0.05))img = cv2.resize(img, size, cv2.INTER_AREA)cv2.imwrite(image_file, img)image_buffer = Image.open(image_file)mark_img = Image.new('RGBA', size, (255, 0, 0, 0))mark_img.paste(image_buffer, (0, 0))mark_img.save(image_file)return "OK"

muggle_ocr 识别字体

# 借助muggle_ocr 识别字体def ocr_num(sdk, filename=r'./fonts/img'):imgs = os.listdir(filename)unicode_dict = {}for img_code in imgs:# 小写new_img_code = img_code.split('.')[0].lower()with open(f"./fonts/img/{img_code}", "rb") as f:img = f.read()text = sdk.predict(image_bytes=img)# 6比较容易识别成Cif text == 'C' or text == 'c':text = '6'  # 之后用replace 要用str类型unicode_dict[new_img_code] = textreturn unicode_dict

获得加密前字符

# 获取加密前的字符
def get_data(html):# html_etree = etree.HTML(html.decode('utf-8')) # 用scrapy的没掌握html_etree = etree.HTML(html)  # urllib 要借用etree做xpathfilm_list = html_etree.xpath(r'//dl/dd')films = {}for film in film_list:film_detail = {}# film_detail['name'] = film.xpath(r'.//p[@class="name"]/text()').extract_first() scrapy用extract_firstfilm_detail['name'] = film.xpath(r'.//p[@class="name"]/a/@title')[0]  # etree 返回的是数组film_detail['film_url'] = 'https://www.maoyan.com/' + film.xpath('./a/@href')[0]film_detail['star'] = film.xpath('.//p[@class="star"]/text()')[0]film_detail['releasetime'] = film.xpath('.//p[@class="releasetime"]/text()')[0]# 多标签的text 用 //text# 存在 \u 要处理film_detail['realtime_box_office'] = film.xpath('.//p[@class="realtime"]/span//text()')[0].encode('raw_unicode_escape').decode('utf-8').replace('\\u', 'uni') + film.xpath('.//p[@class="realtime"]//text()')[2][0]film_detail['total_box_office'] = film.xpath('.//p[@class="total-boxoffice"]/span//text()')[0].encode('raw_unicode_escape').decode('utf-8').replace('\\u', 'uni') + film.xpath('.//p[@class="total-boxoffice"]//text()')[2][0]# 不能直接replace \u# film_detail['realtime_box_office'] = film.xpath('.//p[@class="realtime"]/span//text()')[0] + film.xpath('.//p[@class="realtime"]//text()')[2][0]## film_detail['total_box_office'] = film.xpath('.//p[@class="total-boxoffice"]/span//text()')[0] + film.xpath('.//p[@class="total-boxoffice"]//text()')[2][0]films[film_detail['name']] = film_detail  # 加入到字典return films

用字典配对解密

# 用ocr识别出数字代替unicode
def change_data(unicode_dict, films):for film_detail in films.values():  # 每一部电影for code in unicode_dict:  # 每一个数字编码film_detail['realtime_box_office'] = film_detail['realtime_box_office'].replace(code, unicode_dict[code]) # 实时票房film_detail['total_box_office'] = film_detail['total_box_office'].replace(code, unicode_dict[code]) # 总票房return films

保存至数据库

from pymongo import MongoClient
def save_mongdb(films):client = MongoClient(host='localhost', port=27017)# 库db = client['miaoyan']# 表collections = db['film']# 插入数据for film in films.values():collections.insert_one(film)

main

headers = {'User-Agent': '   ',"Cookie": "  "}# 设置无头浏览器
option = webdriver.ChromeOptions()
# option.add_argument('--headless')# 隐藏特征(UA)
option.add_argument('user-agent=   ')
driver = webdriver.Chrome(options=option)
# 隐藏特征(js)
with open('stealth.min.js') as f:js = f.read()
# 执行js
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {"source": js
})# 测试验证码url = '猫眼电影的网站'wait = WebDriverWait(driver, 10)driver.get(url)time.sleep(2) # 等待页面加载try: # 看是否出现验证码into_iframe(wait)verification(wait)while True:time.sleep(3) # 等待页面跳转或刷新iframe = driver.find_element(by=By.XPATH, value='//iframe')if iframe is None: # 也许一次验证不成功breakelse:verification(wait) # 以后可以加个判断是二次验证还是验证失败except:  # 未出现验证码print('未出现验证码')# 退回父级主页面 不过这里是跳转
# driver.switch_to.parent_frame()# time.sleep(3) 上面已经休眠过了html = driver.page_source # 获得页面源码# 原始的票房数字编码
films = get_data(html)print(films)# 下载woff 文件
file_name = get_woff(html=html, headers=headers)image_dir = r"./fonts/img"  # 路径中不能出现中文file_path = r'./fonts/' + file_name# woff保存成图片
woff2_image_service(file_path, image_dir)# 初始化sdk；model_type 包含了 ModelType.OCR/ModelType.Captcha 两种模式,分别对应常规图片与验证码
sdk = muggle_ocr.SDK(model_type=muggle_ocr.ModelType.OCR)
unicode_dict = ocr_num(sdk, filename=r'./fonts/img')print('----字典表----')
print(unicode_dict)print('----解码后----')
films = change_data(unicode_dict, films)
print(films)# 保存至数据库
save_mongdb(films)# 是否删除解码的图片和woff
del_img = True
del_woff = Trueif del_img:shutil.rmtree(image_dir)if del_woff:os.remove(file_path)

模拟浏览器整理电影榜单相关推荐

【Python爬虫】猫眼电影榜单Top100
这是一个入门级的Python爬虫,结构易于理解.本文对编写此爬虫的全过程进行了讲述.希望对大家的Python爬虫学习有所帮助. 一.目标爬取猫眼电影榜单Top100,将数据存入Excel文件中,并利 ...
爬取动态网页（2017年年度豆瓣电影榜单）
早几天刚好2017年的豆瓣电影榜单web也出来了,于是打算实践一下,用的chrome浏览器. 图1 0. 观察网页元素由于是一个动态的网页,动态效果主要由翻页或者右边的按钮实现,另外,也可以从首页的 ...
《崔庆才Python3网络爬虫开发实战教程》学习笔记（3）：抓取猫眼电影榜单TOP100电影，并存入Excel表格
本篇博文是自己在学习崔庆才的<Python3网络爬虫开发实战教程>的学习笔记系列,如果你也要这套视频教程的话,关注我公众号[小众技术],关注后回复[PYTHON],无套路免费送你一个学习大 ...
Python爬虫实战 | (1) 爬取猫眼电影官网的TOP100电影榜单
在本篇博客中,我们将使用requests+正则表达式来爬取猫眼电影官网的TOP100电影榜单,获取每部电影的片名,主演,上映日期,评分和封面等内容. 打开猫眼Top100,分析URL的变化:发现Top ...
Python全栈开发-Python爬虫-05 爬取猫眼电影榜单信息
爬取猫眼电影榜单信息(翻页) 一. 获取url及headers 首先进入猫眼电影首页: 猫眼电影之后点击菜单栏的榜单并在下面选择 TOP100榜接着右击检查并刷新界面,在Network中找到4 ...
利用requests库和Xpath爬取猫眼电影榜单【Python】
博主的前几篇有关定向网络爬虫的博客,在解析HTML界面时,都是运用了BeautifulSoup和re库进行解析,这篇博客写一下怎么用Xpath语法进行HTML界面解析,从而得到我们想要的结果. 说明 ...
【python爬虫自学笔记】（实战）----爬取猫眼电影榜单Top100
目的:爬取猫眼电影榜单TOP100的信息并保存在文档中. 查看网站结构,确定思路: 首先请求网页的地址为maoyan.com/board/4,电影信息的内容包含在一个个dd标签之中,分析dd标签中的内 ...
使用PHP+QueryList 爬取猫眼电影榜单信息
爬虫是我一直以来跃跃欲试的技术,现在的爬虫框架很多,比较流行的是基于python,nodejs,java,C#的的框架,其中又以基于python的爬虫流行最为广泛,还有的已经是一套傻瓜式的软件操作,如 ...
html+css实现猫眼电影“榜单”静态页面
首页电影榜单热点热映口碑榜最受期待榜国内票房榜北美票房榜 TOP100榜 2016-10-21已更新榜单规则:将昨日国内热映的影片,按照评分从高到低排列取前10名,每天上午10点更新. ...