1. x = x.cuda(non_blocking=True)
2. 进行一些和x无关的操作
3. 执行和x有关的操作

class DataPrefetcher():def __init__(self, dataset, batch_size=64, shuffle=True, num_workers=2):self.data_loader = DataLoader(dataset=dataset,batch_size=batch_size,shuffle=shuffle,pin_memory=True,num_workers=num_workers,drop_last=True)self.loader = iter(self.data_loader)self.stream = torch.cuda.Stream()self.preload()def preload(self):try:self.next_x, self.next_y = next(self.loader)except StopIteration:self.loader = iter(self.data_loader)self.next_x, self.next_y = next(self.loader)with torch.cuda.stream(self.stream):self.next_x = self.next_x.cuda(non_blocking=True)self.next_y = self.next_y.cuda(non_blocking=True)self.next_x = self.next_x.permute(0, 3, 1, 2).float().div(255)self.next_y = self.next_y.float()def next(self):torch.cuda.current_stream().wait_stream(self.stream)data = (self.next_x, self.next_y)self.preload()return data

1. x = x.cuda().permute(0, 3, 1, 2).float().div(255)
2. x = x.permute(0, 3, 1, 2).float().div(255).cuda()

import numpy as np
import cv2
import os
import lmdblist_path = "img_list/test.txt"
img_root = './data/test/edu/test/JPEGImages'
img_type = ".jpg"
label_root = './data/test/edu/test/labels'
label_type = ".txt"env_db = lmdb.open("./test", map_size=1024*1024*1024*8)
txn = env_db.begin(write=True)for line in open(list_path):img_name = line.strip() + img_typeimg_path = os.path.join(img_root, img_name)img = cv2.imread(img_path)label_name = line.strip() + label_typelabel_path = os.path.join(label_root, label_name)label = np.loadtxt(label_path).reshape(-1, 5)print(img_name)# txn.put(img_name.encode(), img) ### 不编码txn.put(img_name.encode(), cv2.imencode('.jpg', img)[1])txn.put(label_name.encode(), label)txn.commit()
env_db.close()

img = np.frombuffer(self.txn.get(img_path.encode()), dtype=np.uint8)
img = cv2.imdecode(img, cv2.IMREAD_COLOR)label = np.frombuffer(self.txn.get(label_path.encode())).reshape(-1, 5) #lmdb存储后numpy会丢失形状

f = h5py.File('test.h5', 'w')
img_dataset = f.create_dataset('img', shape=(840, 1080, 1920, 3), maxshape=(None, 1080, 1920, 3), chunks=(1, 1080, 1920, 3), dtype=np.uint8)
for i, line in enumerate(open(list_path)):img_name = line.strip() + img_typeimg_path = os.path.join(img_root, img_name)img = cv2.imread(img_path)img_dataset[i, :] = imgprint(img_name, img.shape)
f.close()### read，h5的多线程读写存在问题，使用压缩无法正常使用
f = h5py.File('test.h5', "r", swmr=True)
# f = h5py.File('test.h5', "r")
for i in range(840):img = f["img"][i, ...]

img = cv2.imread(img_path, cv2.IMREAD_REDUCED_COLOR_2)
img = cv2.imread(img_path, cv2.IMREAD_REDUCED_COLOR_4)
img = cv2.imread(img_path, cv2.IMREAD_REDUCED_COLOR_8)

pipeline = DataPipeline(**dataset_dict)
pipeline.build()
loader = DALIGenericIterator(pipeline, ["imgs", "labels"], img_len, fill_last_batch=False)for j in range(epoch):for i, data in enumerate(loader):x = data[0]["imgs"]y = data[0]["labels"].cuda()# trainloader.reset()