import time
import json
import logging
import sqlite3
import requests
from lxml import etree
from urllib.parse import quote
from BaiduFaceIdentify import BaiduFaceIdentifyclass BeautyRank():def __init__(self):self.db_conn = sqlite3.connect('beauty_rank.db')self.db_cursor = self.db_conn.cursor()self.model_count = 0self.bfi = BaiduFaceIdentify()passdef request_name_list_url(self,name_list_url):headers = {'Host': '123fans.cn','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2','Accept-Encoding': 'gzip, deflate, br','Referer': 'https://www.ugirls.com/Models/',}# 访问明星名单urlresponse = requests.get(name_list_url,headers=headers,timeout=10)if response.status_code != 200:logging.warning(f"请求失败：{response.status_code}")return Falsesel = etree.HTML(response.text)# 获取明星名单列表name_lists = sel.xpath('//div[@id="main"]//div[@class="ranking"]//td[@class="name"]//a/text()')star_counts = len(name_lists)count = 0# 遍历明星名单for star_name in name_lists:count += 1try:logging.warning(f'\r\n开始采集{count}/{star_counts}：{star_name}')# 通过搜狗获取明星图片self.request_and_parse_star_img(star_name,'sougou')# 通过360获取明星图片self.request_and_parse_star_img(star_name,'360')except:logging.warning(f'\r\n{count}/{star_counts}：{star_name}出现错误')continuedef request_and_parse_star_img(self, star_name, search_engine):headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0','Accept': 'application/json, text/javascript, */*; q=0.01','Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2','X-Requested-With': 'XMLHttpRequest','Connection': 'close',}# 根据要求使用的搜索引擎构建请求的url和完善头部信息if search_engine == 'sougou':star_img_ajax_request_url = f'http://pic.sogou.com/pics?query={quote(star_name)}&mode=1&start=96&reqType=ajax&reqFrom=result&tn=0'headers['Host'] = 'pic.sogou.com'headers['Referer'] = 'http://pic.sogou.com/pics?ie=utf8&p=40230504&interV=kKIOkrELjboMmLkEk7kTkKILlLELjboLmLkEkrgTkKIMkrELjboImLkEk74TkKILmrELjb8TkKIKmrELjbkI_1093829215&query=%E8%B5%B5%E4%B8%BD%E9%A2%96&'# 下边从json_results中抽取明星图片列表的key nameimg_frames_key = 'items'# 下边从明星图片中抽取图片具体url的key nameimg_url_key = 'pic_url'elif search_engine == '360':star_img_ajax_request_url = f'http://image.so.com/j?q={quote(star_name)}&src=tab_www&correct={quote(star_name)}&pn=60&ch=&sn=120&sid=af5aac1212d41c3b5cd6a96a0c311ba2&ran=0&ras=0&cn=0&gn=10&kn=50'headers['Host'] = 'image.so.com'headers['Referer'] = 'http://image.so.com/i?q=%E8%B5%B5%E4%B8%BD%E9%A2%96&src=tab_www'img_frames_key = 'list'img_url_key = 'img'# 访问搜索引擎，获取明星图片urltry:response = requests.get(star_img_ajax_request_url, headers=headers, timeout=10)if response.status_code != 200:logging.warning('请求出错：%s' % response.status_code)return Falsejson_results = json.loads(response.text)except:return False# 遍历图片列表，调用百度AI接口并接收图片颜值for img_frame in json_results[img_frames_key]:img_url = img_frame[img_url_key]logging.warning(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}开始识别:{img_url}')try:beauty_value = self.bfi.parse_face_pic(img_url)except:logging.warning('百度识别出现错误')continue# 如果颜值没有问题则插入数据库if beauty_value > 1.0:logging.warning(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}即将插入数据库（{search_engine}）：{star_name}-{beauty_value}-{img_url}')self.insert_into_database(star_name, beauty_value, img_url,search_engine)return True# 此函数用于将结果插入数据库def insert_into_database(self,star_name,beauty_value,img_url,search_engine):# 如果表star_beauty_value不存在则先创建sql = '''create table if not exists star_beauty_value(star_name text,beauty_value int,img_url text,search_engine text)'''self.db_cursor.execute(sql)self.db_conn.commit()# 将结果插入数据库sql = f'''insert into star_beauty_valuex values('{star_name}','{beauty_value}','{img_url}','{search_engine}')'''self.db_cursor.execute(sql)self.db_conn.commit()def __del__(self):self.db_cursor.close()self.db_conn.close()passif __name__ == '__main__':# 明星名单urlname_list_url = 'https://123fans.cn/rank.php?c=2'# 实例化beauty_rank = BeautyRank()beauty_rank.request_name_list_url(name_list_url)

import base64
import requests
import json
import logging
import timeclass BaiduFaceIdentify():#此函数用于获取access_token，返回access_token的值#此函数被parse_face_pic调用def get_access_token(self):client_id = 'KuLRFhTzX3zBFBSrbQBsl6Q4'                #此变量赋值成自己API Key的值client_secret = '8ahbIb2hEOePzXhehw9ZDL9kGvbzIHTU'    #此变量赋值成自己Secret Key的值auth_url = 'https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=' + client_id + '&client_secret=' + client_secretresponse_at = requests.get(auth_url,timeout=10)json_result = json.loads(response_at.text)access_token = json_result['access_token']return access_token#此函数进行人脸识别，返回识别到的人脸列表#此函数被parse_face_pic调用def identify_faces(self,url_pic,url_fi):headers = {'Content-Type': 'application/json; charset=UTF-8'}# 因为提交URL让baidu去读取图片，总是返回图片下载错了# 所以我们这里将读取URL指向的图片，将图片转成BASE64编码，改用提交BASE64编码而不是提交URL# pic_obj = urllib.request.urlopen(url_pic)# pic_base64 = base64.b64encode(pic_obj.read())response = requests.get(url_pic,timeout=10)pic_base64 = base64.b64encode(response.content)post_data = {# 'image': url_pic,# 'image_type' : 'URL','image': pic_base64,'image_type': 'BASE64','face_field': 'facetype,gender,age,beauty', #expression,faceshape,landmark,race,quality,glasses'max_face_num': 1}logging.warning(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}即将识别b2：{url_pic}')response_fi = requests.post(url_fi,headers=headers,data=post_data,timeout=10)json_fi_result = json.loads(response_fi.text)# 有些图片是没有人脸的，或者识别有问题，这个我们不细管直接捕获异常就返回空列表try:# if json_fi_result['result'] is None:#     return []# else:return json_fi_result['result']['face_list']except Exception:return []#下边的print也许是最直观，你最想要的#print(json_fi_result['result']['face_list'][0]['age'])#print(json_fi_result['result']['face_list'][0]['beauty'])#此函数用于解析进行人脸图片，返回图片中人物颜值#此函数调用get_access_token、identify_facesdef parse_face_pic(self,url_pic):#调用get_access_token获取access_tokenaccess_token = self.get_access_token()# access_token = '24.69a8b9206b4989703e38f4fb92878127.2592000.1534075146.282335-11407672'url_fi = 'https://aip.baidubce.com/rest/2.0/face/v3/detect?access_token=' + access_token#调用identify_faces，获取人脸列表logging.warning(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}开始识别b1:{url_pic}')json_faces = self.identify_faces(url_pic,url_fi)# 如果没有人脸，那么就以0.0为颜值评分返回if len(json_faces) == 0:# logging.warning('未识别到人脸')return 0.0else:for json_face in json_faces:logging.debug('种类：'+json_face['face_type']['type'])logging.debug('性别：'+json_face['gender']['type'])logging.debug('年龄：'+str(json_face['age']))logging.debug('颜值：'+str(json_face['beauty']))# 如果识别到的不是妹子，也以1.0为颜值评分返回# 如果识别到的是妹子，直接以值颜值返回if json_face['gender']['type'] != 'female':# logging.info('图片不是妹子')return 1.0else:return json_face['beauty']if __name__ == '__main__':#uil_pic赋值成自己要测试的图片的url地址url_pic = 'http://pic.ikafan.com/imgp/L3Byb3h5L2h0dHAvdXNlcmltYWdlMy4zNjBkb2MuY29tLzEzLzA1MDkvMDUvNjYyMjAxMF8yMDEzMDUwOTA1Mzk0NjA4MDMuanBn.jpg'bfi = BaiduFaceIdentify()bfi.parse_face_pic(url_pic)

View Code