用 threading 模块多线程读图片加速，flickr25k 和 nuswide 两个数据集的图片准备见 [1-4]，图像预处理程序来自 [5]。

为了测试，写了一个叫 LazyImage 的类，和其多线程版本 LazyImage_MT，代码：

import time
import multiprocessing
import threading
import os
import os.path as osp
import numpy as np
from PIL import Image#
# 普通顺序加载：LazyImage、ImageF25k、ImageNUS
#class LazyImage:"""mimics np.ndarray, but uses lazy loading"""def __init__(self, image_path, image_size=224):"""image_size: int"""self.image_path = image_pathself.image_size = image_size# used in resizingself.lower_half = image_size // 2self.upper_half = (image_size + 1) // 2def __getitem__(self, index):if isinstance(index, int):return self._load_image(index)elif isinstance(index, (np.ndarray, list, tuple)):if isinstance(index, np.ndarray):assert 1 == index.ndim, "* index should be vector"return np.vstack([np.expand_dims(self._load_image(i), 0) for i in index])raise NotImplementeddef _load_image(self, full_index):"""loads single image & resizesInput:- full_index: int, the sample ID"""img = Image.open(self._get_image_path(full_index))xsize, ysize = img.sizeseldim = min(xsize, ysize)rate = float(self.image_size) / seldimimg = img.resize((int(xsize * rate), int(ysize * rate)))nxsize, nysize = img.sizecx, cy = nxsize / 2.0, nysize / 2.0box = (cx - self.lower_half, cy - self.lower_half, cx + self.upper_half, cy + self.upper_half)img = img.crop(box)img = img.convert("RGB")  # can deal with grey-scale imagesimg = img.resize((224, 224))img = np.array(img, dtype=np.float32)return img  # [H, W, C]def _get_image_path(self, full_index):"""get image path according to sample ID"""raise NotImplementedclass ImageF25k(LazyImage):def _get_image_path(self, full_index):# shift to 1-basereturn osp.join(self.image_path, "im{}.jpg".format(full_index + 1))class ImageNUS(LazyImage):"""depends on (github) iTomxy/data/nuswide/make.image.link.py"""def _get_image_path(self, full_index):# remain 0-base as isreturn osp.join(self.image_path, "{}.jpg".format(full_index))#
# 多线程加载：LazyImage_MT、ImageF25k_MT、ImageNUS_MT
#class LazyImage_MT:"""multi-threading version"""def __init__(self, image_path, image_size=224, n_thread=None):"""image_size: int"""self.image_path = image_pathself.image_size = image_size# used in resizingself.lower_half = image_size // 2self.upper_half = (image_size + 1) // 2self._mutex_put = threading.Lock()self._buffer = []self.n_thread = n_thread if (n_thread is not None) else \max(1, multiprocessing.cpu_count() - 2)def __getitem__(self, index):if isinstance(index, int):return self._load_image(index)elif isinstance(index, (np.ndarray, list, tuple)):if isinstance(index, np.ndarray):assert 1 == index.ndim, "* index should be vector"if self.n_thread < 2:return np.vstack([np.expand_dims(self._load_image(i), 0) for i in index])self._buffer = []batch_size = (len(index) + self.n_thread - 1) // self.n_threadt_list = []for tid in range(self.n_thread):t = threading.Thread(target=self._load_image_mt, args=(index, range(tid * batch_size, min((tid + 1) * batch_size, len(index)))))t_list.append(t)t.start()for t in t_list:t.join()del t_listassert len(self._buffer) == len(index)self._buffer = [t[1] for t in sorted(self._buffer, key=lambda _t: _t[0])]return np.vstack(self._buffer)raise NotImplementeddef _load_image_mt(self, indices, seg_meta_indices):batch_images = [(mid, np.expand_dims(self._load_image(indices[mid]), 0))for mid in seg_meta_indices]self._mutex_put.acquire()self._buffer.extend(batch_images)self._mutex_put.release()def _load_image(self, full_index):"""loads single image & resizesInput:- full_index: int, the sample ID"""img = Image.open(self._get_image_path(full_index))xsize, ysize = img.sizeseldim = min(xsize, ysize)rate = float(self.image_size) / seldimimg = img.resize((int(xsize * rate), int(ysize * rate)))nxsize, nysize = img.sizecx, cy = nxsize / 2.0, nysize / 2.0box = (cx - self.lower_half, cy - self.lower_half, cx + self.upper_half, cy + self.upper_half)img = img.crop(box)img = img.convert("RGB")  # can deal with grey-scale imagesimg = img.resize((224, 224))img = np.array(img, dtype=np.float32)return img  # [H, W, C]def _get_image_path(self, full_index):"""get image path according to sample ID"""raise NotImplementedclass ImageF25k_MT(LazyImage_MT):def _get_image_path(self, full_index):# shift to 1-basereturn osp.join(self.image_path, "im{}.jpg".format(full_index + 1))class ImageNUS_MT(LazyImage_MT):"""depends on (github) iTomxy/data/nuswide/make.image.link.py"""def _get_image_path(self, full_index):# remain 0-base as isreturn osp.join(self.image_path, "{}.jpg".format(full_index))#
# 测试：速度、一致性
#if "__main__" == __name__:batch_size = 128N_F25K = 25000N_NUS = 269648indices_f25k = np.arange(N_F25K)indices_nus = np.arange(N_NUS)print("-> flickr25k")tic = time.time()im_f25k = ImageF25k("data/flickr25k/mirflickr")for i in range(0, N_F25K, batch_size):_ = im_f25k[indices_f25k[i: i + batch_size]]print(time.time() - tic)  # 214.9833734035492# del im_f25kprint("-> flickr25k multi-threading")tic = time.time()im_f25k_mt = ImageF25k_MT("data/flickr25k/mirflickr")for i in range(0, N_F25K, batch_size):_ = im_f25k_mt[indices_f25k[i: i + batch_size]]print(time.time() - tic)  # 56.653871297836304# del im_f25k_mtprint("-> nuswide")tic = time.time()im_nus = ImageNUS("data/nuswide-tc21/images")for i in range(0, N_NUS, batch_size):_ = im_nus[indices_nus[i: i + batch_size]]print(time.time() - tic)  # 631.8568336963654# del im_nusprint("-> nuswide multi-threading")tic = time.time()im_nus_mt = ImageNUS_MT("data/nuswide-tc21/images")for i in range(0, N_NUS, batch_size):_ = im_nus_mt[indices_nus[i: i + batch_size]]print(time.time() - tic)  # 207.77122569084167# del im_nus_mtprint("-> consistency")for i in range(0, N_F25K, batch_size):i_s = im_f25k[indices_f25k[i: i + batch_size]]i_mt = im_f25k_mt[indices_f25k[i: i + batch_size]]print("f25k diff:", (i_s != i_mt).sum())  # 0i_s = im_nus[indices_nus[i: i + batch_size]]i_mt = im_nus_mt[indices_nus[i: i + batch_size]]print("nus diff:", (i_s != i_mt).sum())  # 0break

输出

-> flickr25k
214.9833734035492
-> flickr25k multi-threading
56.653871297836304
-> nuswide
631.8568336963654
-> nuswide multi-threading
207.77122569084167
-> consistency
f25k diff: 0
nus diff: 0

References

MIR-Flickr25K数据集预处理
iTomxy/data/flickr25k
NUS-WIDE数据集预处理
iTomxy/data/nuswide
DeXie0808/GCH/load_data.py

python多线程读图片相关推荐

Python 多线程下载图片
多线程下载图片参考链接: Python标准库-urllib和urllib3 urllib实战2–urllib基础urlretrieve().urlcleanup().info().getcode() ...
Python多线程下载图片
文章目录导包模拟浏览器登录参数一:单线程爬取 1.生成网页列表 2.爬取图片的网址 3.下载图片到本地二:多线程下载图片 0.加锁 1.获取图片网址 2.下载图片 3.函数调用 4.问题完整 ...
【python学习】多线程下载图片实战
python多线程下载图片(setu) 前言 python的学习其实早在两三个星期前就开始了,奈何我是个急性子,所以学的很浮躁,甚至连md笔记都没写.但是想了想,我学python的目的反正也仅仅是为了 ...
python多线程，多进程，线程池，进程池
https://blog.csdn.net/somezz/article/details/80963760 python 多线程线程(Thread)也叫轻量级进程,是操作系统能够进行运算调度的最小单 ...
2021-03-10 Python多线程爬虫快速批量下载图片
Python多线程爬虫快速批量下载图片 1.完成这个需要导入的模块 urllib,random,queue(队列),threading,time,os,json 第三方模块的安装键盘win+R,输入 ...
Python多线程下载网络URL图片的方法
Python多线程下载网络URL图片的方法采用多线程的方法,通过URL地址,下载资源图片 GitHub地址:https://github.com/PanJinquan/python-learning ...
python多线程下载小姐姐图片
python多线程下载小姐姐图片闲谈思路实现过程单线程实现代码功能问题描述多线程处理办法完成效果闲谈今日闲来无事,翻看博客,看到一篇关于python自动下载图片的文章,就萌生了也写一 ...
图片爬虫，手把手教你Python多线程下载获取图片
图片站lemanoosh数据为异步加载的形式,往下拉会展示更多数据,也就是下一页数据,通过谷歌浏览器可以很清晰的看到数据接口地址,以及数据展现形式,与其他网站返回json数据的不同之处是,该网站返回的 ...
python 读图片性能测试
pil读图片需要0ms,但是转换opencv需要13ms opencv读图片需要13ms import numpy as np import cv2 import timefrom PIL impor ...

python多线程读图片

References

python多线程读图片相关推荐

最新文章

热门文章