爬取百度和Flickr图像

import requests
from threading import Thread
import re
import time
import hashlibclass BaiDu:"""爬取百度图片"""def __init__(self, name, page):self.start_time = time.time()self.name = nameself.page = page#self.url = 'https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&rn=60&'self.url = 'https://image.baidu.com/search/acjson'self.header = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'}# 添加为自己的浏览器版本，具体操作网上一大推self.num = 0self.all_num = 0self.thread_all = [] # thread numdef queryset(self):"""将字符串转换为查询字符串形式"""pn = 0for i in range(int(self.page)):pn += 60 * iname = {'word': self.name, 'pn': pn, 'tn':'resultjson_com', 'ipn':'rj', 'rn':60}url = self.urlself.all_num += 60self.getrequest(i, url, name)def getrequest(self, index, url, data):"""发送请求"""print('[INFO]: 开始发送请求：' + url)ret = requests.get(url, headers=self.header, params=data)if str(ret.status_code) == '200':print('[INFO]: request 200 ok :' + ret.url)else:print('[INFO]: request {}, {}'.format(ret.status_code, ret.url))response = ret.content.decode()img_links = re.findall(r'thumbURL.*?\.jpg', response)links = []# 提取urlfor link in img_links:links.append(link[11:])self.build_thread(index, links)def saveimage(self, links):"""保存图片"""for i, link in enumerate(links):if not link:continue#print('[INFO]:正在保存图片：' + link)m = hashlib.md5()m.update(link.encode())name = m.hexdigest()try:ret = requests.get(link, headers = self.header)image_content = ret.contentfilename = './images/' + name + '.jpg'with open(filename, 'wb') as f:f.write(image_content)#print('[INFO]:保存成功，图片名为：{}.jpg'.format(name))except Exception:passself.num += 1def run(self):for thred_p in self.thread_all:thred_p.start()for thred_p in self.thread_all:thred_p.join() def build_thread(self, i, links):"""多线程"""self.thread_all.append(Thread(target=self.saveimage, args=(links,)))def __del__(self):end_time = time.time()print('request total images: {}, actual download images: {}, time cost {} second'.format(self.all_num, self.num, (end_time - self.start_time)))def main():hand_name = ['人脸', 'head','arm']for name in hand_name:#name = '手势图像'#input('请输入你要爬取的图片类型: ')page = 10 #input('请输入你要爬取图片的页数(60张一页):')baidu = BaiDu(name, page)baidu.queryset()baidu.run()if __name__ == '__main__':main()

#coding:utf-8import flickrapi
import urllib
import os
from threading import Thread
from tqdm import tqdmclass CrawlFlickr:def __init__(self, API_KEY="", API_SECRET="", SavePath="", PerPage=10, Text="", Tags="", ThreadNum=4,MaxCounter=10):self.urls = []self.ThreadNum = ThreadNumself.SavePath = SavePathself.Thread_All = []self.MaxCounter = MaxCounterflickr = flickrapi.FlickrAPI(API_KEY, API_SECRET, cache=True)self.photos=flickr.walk(text=Text,tag_mode='all',tags=Tags,extras='url_c',per_page=PerPage,           # may be you can try different numbers..sort='relevance')self.get_url()self.build_thread()def get_url(self):for i, photo in enumerate(self.photos):url = photo.get('url_c')if str(url) == "None":continueself.urls.append(url)if i >= self.MaxCounter:breakif i%200==0:print('get {} url, max {}\n'.format(len(self.urls), self.MaxCounter))print('\nget {} url finish.....\n'.format(len(self.urls)))def build_thread(self):if self.ThreadNum >= len(self.urls):raise ValueError(f"Input Thread number is large: {self.ThreadNum},""while data is small: {len(self.urls)}")part = len(self.urls) // self.ThreadNumfor i in range(self.ThreadNum)[::-1]:self.Thread_All.append(Thread(target=self.get_img, args=(self.urls[i * part:],)))self.urls = self.urls[:i * part]print('build thread finish...\n')def run(self):for thred in self.Thread_All:thred.start()for thred in self.Thread_All:thred.join()print('download image finish...\n')def get_img(self, urls):for url in urls:img_name = url.split('/')[-1]if '.jpg' in img_name or '.png' in img_name:urllib.request.urlretrieve(url, os.path.join(self.SavePath, img_name))print('download {}\n'.format(os.path.join(self.SavePath, img_name)))if __name__ == "__main__":param = dict(API_KEY="", API_SECRET="", SavePath="./images", PerPage=10,Text="human pose", Tags="", ThreadNum=8,MaxCounter=500)crawl_flickr = CrawlFlickr(**param)crawl_flickr.run()