1.首先对给的数据进行划分，类型为每个类单独放在一个文件夹中

import json
import shutil
import os
from glob import glob
from tqdm import tqdm
# 此文件的作用是创建每个类的文件夹，以及根据给出来的Json中已经做好的分类，对数据进行对号入座划分。
# 加载json文件得出一个字典，然后根据Key值来提取每个文件到相应的文件夹中，（注意去除了不合理数据）try:for i in range(0,59):os.mkdir("./data/train/" + str(i))
except:passfile_train = json.load(open("./data/temp/labels/AgriculturalDisease_train_annotations.json","r",encoding="utf-8"))
file_val = json.load(open("./data/temp/labels/AgriculturalDisease_validation_annotations.json","r",encoding="utf-8"))file_list = file_train + file_valfor file in tqdm(file_list):filename = file["image_id"]origin_path = "./data/temp/images/" + filenameids = file["disease_class"]if ids ==  44:continueif ids == 45:continueif ids > 45:ids = ids -2save_path = "./data/train/" + str(ids) + "/"shutil.copy(origin_path,save_path)

2.获取增强数据集类的定义

# 数据增强的多种方式，使用自定义的方法。调用只需在dataloader.py文件中的get_item函数中调用类自身参数
# transforms，transforms中集合了compose，compose中列出详细所使用的增强方式。
from __future__ import division
import cv2
import numpy as np
from numpy import random
import math
from sklearn.utils import shuffle
# 常用的增强方式几乎都在这里，只需在compose中列出类名即可
__all__ = ['Compose','RandomHflip', 'RandomUpperCrop', 'Resize', 'UpperCrop', 'RandomBottomCrop',"RandomErasing",'BottomCrop', 'Normalize', 'RandomSwapChannels', 'RandomRotate', 'RandomHShift',"CenterCrop","RandomVflip",'ExpandBorder', 'RandomResizedCrop','RandomDownCrop', 'DownCrop', 'ResizedCrop',"FixRandomRotate"]# 组合# “随机翻转”，“随机顶部切割”，“调整大小”，“上切割”，“随机底部切割”、# “随机擦除”，“底部切割”，“正则化”，“随机交换频道”，“随机旋转”，# “随机HShift”，“中央切割”，“随机Vflip”，“扩展边界”，“随机调整切割”，# “随机下降”，“下降切割”， “调整切割”，“固定随机化”。# 每个增强方式类需要调用普通方法描述如下：
def rotate_nobound(image, angle, center=None, scale=1.):(h, w) = image.shape[:2]# if the center is None, initialize it as the center of# the imageif center is None:center = (w // 2, h // 2)# perform the rotationM = cv2.getRotationMatrix2D(center, angle, scale)rotated = cv2.warpAffine(image, M, (w, h))return rotateddef scale_down(src_size, size):w, h = sizesw, sh = src_sizeif sh < h:w, h = float(w * sh) / h, shif sw < w:w, h = sw, float(h * sw) / wreturn int(w), int(h)def fixed_crop(src, x0, y0, w, h, size=None):out = src[y0:y0 + h, x0:x0 + w]if size is not None and (w, h) != size:out = cv2.resize(out, (size[0], size[1]), interpolation=cv2.INTER_CUBIC)return out# 固定随机旋转
class FixRandomRotate(object):def __init__(self, angles=[0,90,180,270], bound=False):self.angles = anglesself.bound = bounddef __call__(self,img):do_rotate = random.randint(0, 4)angle=self.angles[do_rotate]if self.bound:img = rotate_bound(img, angle)else:img = rotate_nobound(img, angle)return imgdef center_crop(src, size):h, w = src.shape[0:2]new_w, new_h = scale_down((w, h), size)x0 = int((w - new_w) / 2)y0 = int((h - new_h) / 2)out = fixed_crop(src, x0, y0, new_w, new_h, size)return outdef bottom_crop(src, size):h, w = src.shape[0:2]new_w, new_h = scale_down((w, h), size)x0 = int((w - new_w) / 2)y0 = int((h - new_h) * 0.75)out = fixed_crop(src, x0, y0, new_w, new_h, size)return outdef rotate_bound(image, angle):# grab the dimensions of the image and then determine the# centerh, w = image.shape[:2](cX, cY) = (w // 2, h // 2)M = cv2.getRotationMatrix2D((cX, cY), angle, 1.0)cos = np.abs(M[0, 0])sin = np.abs(M[0, 1])# compute the new bounding dimensions of the imagenW = int((h * sin) + (w * cos))nH = int((h * cos) + (w * sin))# adjust the rotation matrix to take into account translationM[0, 2] += (nW / 2) - cXM[1, 2] += (nH / 2) - cYrotated = cv2.warpAffine(image, M, (nW, nH))return rotated# 常用增强方式，以类的方式体现：
# 将多个transform组合起来使用
crop切割  filp旋转
class Compose(object):def __init__(self, transforms):self.transforms = transformsdef __call__(self, img):for t in self.transforms:img = t(img)return img
class RandomRotate(object):def __init__(self, angles, bound=False):self.angles = anglesself.bound = bounddef __call__(self,img):do_rotate = random.randint(0, 2)if do_rotate:angle = np.random.uniform(self.angles[0], self.angles[1])if self.bound:img = rotate_bound(img, angle)else:img = rotate_nobound(img, angle)return img
class RandomBrightness(object):def __init__(self, delta=10):assert delta >= 0assert delta <= 255self.delta = deltadef __call__(self, image):if random.randint(2):delta = random.uniform(-self.delta, self.delta)image = (image + delta).clip(0.0, 255.0)# print('RandomBrightness,delta ',delta)return imageclass RandomContrast(object):def __init__(self, lower=0.9, upper=1.05):self.lower = lowerself.upper = upperassert self.upper >= self.lower, "contrast upper must be >= lower."assert self.lower >= 0, "contrast lower must be non-negative."# expects float imagedef __call__(self, image):if random.randint(2):alpha = random.uniform(self.lower, self.upper)# print('contrast:', alpha)image = (image * alpha).clip(0.0,255.0)return imageclass RandomSaturation(object):def __init__(self, lower=0.8, upper=1.2):self.lower = lowerself.upper = upperassert self.upper >= self.lower, "contrast upper must be >= lower."assert self.lower >= 0, "contrast lower must be non-negative."def __call__(self, image):if random.randint(2):alpha = random.uniform(self.lower, self.upper)image[:, :, 1] *= alpha# print('RandomSaturation,alpha',alpha)return imageclass RandomHue(object):def __init__(self, delta=18.0):assert delta >= 0.0 and delta <= 360.0self.delta = deltadef __call__(self, image):if random.randint(2):alpha = random.uniform(-self.delta, self.delta)image[:, :, 0] += alphaimage[:, :, 0][image[:, :, 0] > 360.0] -= 360.0image[:, :, 0][image[:, :, 0] < 0.0] += 360.0# print('RandomHue,alpha:', alpha)return imageclass ConvertColor(object):def __init__(self, current='BGR', transform='HSV'):self.transform = transformself.current = currentdef __call__(self, image):if self.current == 'BGR' and self.transform == 'HSV':image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)elif self.current == 'HSV' and self.transform == 'BGR':image = cv2.cvtColor(image, cv2.COLOR_HSV2BGR)else:raise NotImplementedErrorreturn imageclass RandomSwapChannels(object):def __call__(self, img):if np.random.randint(2):order = np.random.permutation(3)return img[:,:,order]return imgclass RandomCrop(object):def __init__(self, size):self.size = sizedef __call__(self, image):h, w, _ = image.shapenew_w, new_h = scale_down((w, h), self.size)if w == new_w:x0 = 0else:x0 = random.randint(0, w - new_w)if h == new_h:y0 = 0else:y0 = random.randint(0, h - new_h)out = fixed_crop(image, x0, y0, new_w, new_h, self.size)return outclass RandomResizedCrop(object):def __init__(self, size,scale=(0.49, 1.0), ratio=(1., 1.)):self.size = sizeself.scale = scaleself.ratio = ratiodef __call__(self,img):if random.random() < 0.2:return cv2.resize(img,self.size)h, w, _ = img.shapearea = h * wd=1for attempt in range(10):target_area = random.uniform(self.scale[0], self.scale[1]) * areaaspect_ratio = random.uniform(self.ratio[0], self.ratio[1])new_w = int(round(math.sqrt(target_area * aspect_ratio)))new_h = int(round(math.sqrt(target_area / aspect_ratio)))if random.random() < 0.5:new_h, new_w = new_w, new_hif new_w < w and new_h < h:x0 = random.randint(0, w - new_w)y0 = (random.randint(0, h - new_h))//dout = fixed_crop(img, x0, y0, new_w, new_h, self.size)return out# Fallbackreturn center_crop(img, self.size)class DownCrop():def __init__(self, size,  select, scale=(0.36,0.81)):self.size = sizeself.scale = scaleself.select = selectdef __call__(self,img, attr_idx):if attr_idx not in self.select:return img, attr_idxif attr_idx == 0:self.scale=(0.64,1.0)h, w, _ = img.shapearea = h * ws = (self.scale[0]+self.scale[1])/2.0target_area = s * areanew_w = int(round(math.sqrt(target_area)))new_h = int(round(math.sqrt(target_area)))if new_w < w and new_h < h:dw = w-new_wx0 = int(0.5*dw)y0 = h-new_hout = fixed_crop(img, x0, y0, new_w, new_h, self.size)return out, attr_idx# Fallbackreturn center_crop(img, self.size), attr_idxclass ResizedCrop(object):def __init__(self, size, select,scale=(0.64, 1.0), ratio=(3. / 4., 4. / 3.)):self.size = sizeself.scale = scaleself.ratio = ratioself.select = selectdef __call__(self,img, attr_idx):if attr_idx not in self.select:return img, attr_idxh, w, _ = img.shapearea = h * wd=1if attr_idx == 2:self.scale=(0.36,0.81)d=2if attr_idx == 0:self.scale=(0.81,1.0)target_area = (self.scale[0]+self.scale[1])/2.0 * area# aspect_ratio = random.uniform(self.ratio[0], self.ratio[1])new_w = int(round(math.sqrt(target_area)))new_h = int(round(math.sqrt(target_area)))# if random.random() < 0.5:#     new_h, new_w = new_w, new_hif new_w < w and new_h < h:x0 =  (w - new_w)//2y0 = (h - new_h)//d//2out = fixed_crop(img, x0, y0, new_w, new_h, self.size)# cv2.imshow('{}_img'.format(idx2attr_map[attr_idx]), img)# cv2.imshow('{}_crop'.format(idx2attr_map[attr_idx]), out)## cv2.waitKey(0)return out, attr_idx# Fallbackreturn center_crop(img, self.size), attr_idxclass RandomHflip(object):def __call__(self, image):if random.randint(2):return cv2.flip(image, 1)else:return image
class RandomVflip(object):def __call__(self, image):if random.randint(2):return cv2.flip(image, 0)else:return imageclass Hflip(object):def __init__(self,doHflip):self.doHflip = doHflipdef __call__(self, image):if self.doHflip:return cv2.flip(image, 1)else:return imageclass CenterCrop(object):def __init__(self, size):self.size = sizedef __call__(self, image):return center_crop(image, self.size)class UpperCrop():def __init__(self, size, scale=(0.09, 0.64)):self.size = sizeself.scale = scaledef __call__(self,img):h, w, _ = img.shapearea = h * ws = (self.scale[0]+self.scale[1])/2.0target_area = s * areanew_w = int(round(math.sqrt(target_area)))new_h = int(round(math.sqrt(target_area)))if new_w < w and new_h < h:dw = w-new_wx0 = int(0.5*dw)y0 = 0out = fixed_crop(img, x0, y0, new_w, new_h, self.size)return out# Fallbackreturn center_crop(img, self.size)class RandomUpperCrop(object):def __init__(self, size, select, scale=(0.09, 0.64), ratio=(3. / 4., 4. / 3.)):self.size = sizeself.scale = scaleself.ratio = ratioself.select = selectdef __call__(self,img, attr_idx):if random.random() < 0.2:return img, attr_idxif attr_idx not in self.select:return img, attr_idxh, w, _ = img.shapearea = h * wfor attempt in range(10):s = random.uniform(self.scale[0], self.scale[1])d = 0.1 + (0.3 - 0.1) / (self.scale[1] - self.scale[0]) * (s - self.scale[0])target_area = s * areaaspect_ratio = random.uniform(self.ratio[0], self.ratio[1])new_w = int(round(math.sqrt(target_area * aspect_ratio)))new_h = int(round(math.sqrt(target_area / aspect_ratio)))# new_w = int(round(math.sqrt(target_area)))# new_h = int(round(math.sqrt(target_area)))if new_w < w and new_h < h:dw = w-new_wx0 = random.randint(int((0.5-d)*dw), int((0.5+d)*dw)+1)y0 = (random.randint(0, h - new_h))//10out = fixed_crop(img, x0, y0, new_w, new_h, self.size)return out, attr_idx# Fallbackreturn center_crop(img, self.size), attr_idx
class RandomDownCrop(object):def __init__(self, size, select, scale=(0.36, 0.81), ratio=(3. / 4., 4. / 3.)):self.size = sizeself.scale = scaleself.ratio = ratioself.select = selectdef __call__(self,img, attr_idx):if random.random() < 0.2:return img, attr_idxif attr_idx not in self.select:return img, attr_idxif attr_idx == 0:self.scale=(0.64,1.0)h, w, _ = img.shapearea = h * wfor attempt in range(10):s = random.uniform(self.scale[0], self.scale[1])d = 0.1 + (0.3 - 0.1) / (self.scale[1] - self.scale[0]) * (s - self.scale[0])target_area = s * areaaspect_ratio = random.uniform(self.ratio[0], self.ratio[1])new_w = int(round(math.sqrt(target_area * aspect_ratio)))new_h = int(round(math.sqrt(target_area / aspect_ratio)))## new_w = int(round(math.sqrt(target_area)))# new_h = int(round(math.sqrt(target_area)))if new_w < w and new_h < h:dw = w-new_wx0 = random.randint(int((0.5-d)*dw), int((0.5+d)*dw)+1)y0 = (random.randint((h - new_h)*9//10, h - new_h))out = fixed_crop(img, x0, y0, new_w, new_h, self.size)# cv2.imshow('{}_img'.format(idx2attr_map[attr_idx]), img)# cv2.imshow('{}_crop'.format(idx2attr_map[attr_idx]), out)## cv2.waitKey(0)return out, attr_idx# Fallbackreturn center_crop(img, self.size), attr_idxclass RandomHShift(object):def __init__(self, select, scale=(0.0, 0.2)):self.scale = scaleself.select = selectdef __call__(self,img, attr_idx):if attr_idx not in self.select:return img, attr_idxdo_shift_crop = random.randint(0, 2)if do_shift_crop:h, w, _ = img.shapemin_shift = int(w*self.scale[0])max_shift = int(w*self.scale[1])shift_idx = random.randint(min_shift, max_shift)direction = random.randint(0,2)if direction:right_part = img[:, -shift_idx:, :]left_part = img[:, :-shift_idx, :]else:left_part = img[:, :shift_idx, :]right_part = img[:, shift_idx:, :]img = np.concatenate((right_part, left_part), axis=1)# Fallbackreturn img, attr_idxclass RandomBottomCrop(object):def __init__(self, size, select, scale=(0.4, 0.8)):self.size = sizeself.scale = scaleself.select = selectdef __call__(self,img, attr_idx):if attr_idx not in self.select:return img, attr_idxh, w, _ = img.shapearea = h * wfor attempt in range(10):s = random.uniform(self.scale[0], self.scale[1])d = 0.25 + (0.45 - 0.25) / (self.scale[1] - self.scale[0]) * (s - self.scale[0])target_area = s * areanew_w = int(round(math.sqrt(target_area)))new_h = int(round(math.sqrt(target_area)))if new_w < w and new_h < h:dw = w-new_wdh = h - new_hx0 = random.randint(int((0.5-d)*dw), min(int((0.5+d)*dw)+1,dw))y0 = (random.randint(max(0,int(0.8*dh)-1), dh))out = fixed_crop(img, x0, y0, new_w, new_h, self.size)return out, attr_idx# Fallbackreturn bottom_crop(img, self.size), attr_idxclass BottomCrop():def __init__(self, size,  select, scale=(0.4, 0.8)):self.size = sizeself.scale = scaleself.select = selectdef __call__(self,img, attr_idx):if attr_idx not in self.select:return img, attr_idxh, w, _ = img.shapearea = h * ws = (self.scale[0]+self.scale[1])/3.*2.target_area = s * areanew_w = int(round(math.sqrt(target_area)))new_h = int(round(math.sqrt(target_area)))if new_w < w and new_h < h:dw = w-new_wdh = h-new_hx0 = int(0.5*dw)y0 = int(0.9*dh)out = fixed_crop(img, x0, y0, new_w, new_h, self.size)return out, attr_idx# Fallbackreturn bottom_crop(img, self.size), attr_idxclass Resize(object):def __init__(self, size, inter=cv2.INTER_CUBIC):self.size = sizeself.inter = interdef __call__(self, image):return cv2.resize(image, (self.size[0], self.size[0]), interpolation=self.inter)class ExpandBorder(object):def __init__(self,  mode='constant', value=255, size=(336,336), resize=False):self.mode = modeself.value = valueself.resize = resizeself.size = sizedef __call__(self, image):h, w, _ = image.shapeif h > w:pad1 = (h-w)//2pad2 = h - w - pad1if self.mode == 'constant':image = np.pad(image, ((0, 0), (pad1, pad2), (0, 0)),self.mode, constant_values=self.value)else:image = np.pad(image,((0,0), (pad1, pad2),(0,0)), self.mode)elif h < w:pad1 = (w-h)//2pad2 = w-h - pad1if self.mode == 'constant':image = np.pad(image, ((pad1, pad2),(0, 0), (0, 0)),self.mode,constant_values=self.value)else:image = np.pad(image, ((pad1, pad2), (0, 0), (0, 0)),self.mode)if self.resize:image = cv2.resize(image, (self.size[0], self.size[0]),interpolation=cv2.INTER_LINEAR)return image
class AstypeToInt():def __call__(self, image, attr_idx):return image.clip(0,255.0).astype(np.uint8), attr_idxclass AstypeToFloat():def __call__(self, image, attr_idx):return image.astype(np.float32), attr_idximport matplotlib.pyplot as plt
class Normalize(object):def __init__(self,mean, std):''':param mean: RGB order:param std:  RGB order'''self.mean = np.array(mean).reshape(3,1,1)self.std = np.array(std).reshape(3,1,1)def __call__(self, image):''':param image:  (H,W,3)  RGB:return:'''# plt.figure(1)# plt.imshow(image)# plt.show()return (image.transpose((2, 0, 1)) / 255. - self.mean) / self.stdclass RandomErasing(object):def __init__(self, select,EPSILON=0.5,sl=0.02, sh=0.09, r1=0.3, mean=[0.485, 0.456, 0.406]):self.EPSILON = EPSILONself.mean = meanself.sl = slself.sh = shself.r1 = r1self.select = selectdef __call__(self, img,attr_idx):if attr_idx not in self.select:return img,attr_idxif random.uniform(0, 1) > self.EPSILON:return img,attr_idxfor attempt in range(100):area = img.shape[1] * img.shape[2]target_area = random.uniform(self.sl, self.sh) * areaaspect_ratio = random.uniform(self.r1, 1 / self.r1)h = int(round(math.sqrt(target_area * aspect_ratio)))w = int(round(math.sqrt(target_area / aspect_ratio)))if w <= img.shape[2] and h <= img.shape[1]:x1 = random.randint(0, img.shape[1] - h)y1 = random.randint(0, img.shape[2] - w)if img.shape[0] == 3:# img[0, x1:x1+h, y1:y1+w] = random.uniform(0, 1)# img[1, x1:x1+h, y1:y1+w] = random.uniform(0, 1)# img[2, x1:x1+h, y1:y1+w] = random.uniform(0, 1)img[0, x1:x1 + h, y1:y1 + w] = self.mean[0]img[1, x1:x1 + h, y1:y1 + w] = self.mean[1]img[2, x1:x1 + h, y1:y1 + w] = self.mean[2]# img[:, x1:x1+h, y1:y1+w] = torch.from_numpy(np.random.rand(3, h, w))else:img[0, x1:x1 + h, y1:y1 + w] = self.mean[1]# img[0, x1:x1+h, y1:y1+w] = torch.from_numpy(np.random.rand(1, h, w))return img,attr_idxreturn img,attr_idxif __name__ == '__main__':import matplotlib.pyplot as pltclass FSAug(object):def __init__(self):self.augment = Compose([AstypeToFloat(),# RandomHShift(scale=(0.,0.2),select=range(8)),# RandomRotate(angles=(-20., 20.), bound=True),ExpandBorder(select=range(8), mode='symmetric'),# symmetric# Resize(size=(336, 336), select=[ 2, 7]),AstypeToInt()])def __call__(self, spct,attr_idx):return self.augment(spct,attr_idx)trans = FSAug()img_path = '/media/gserver/data/FashionAI/round2/train/Images/coat_length_labels/0b6b4a2146fc8616a19fcf2026d61d50.jpg'img = cv2.cvtColor(cv2.imread(img_path),cv2.COLOR_BGR2RGB)img_trans,_ = trans(img,5)# img_trans2,_ = trans(img,6)plt.figure()plt.subplot(221)plt.imshow(img)plt.subplot(222)plt.imshow(img_trans)# plt.subplot(223)# plt.imshow(img_trans2)# plt.imshow(img_trans2)plt.show()

方式二：  用于线下增强数据，采用的方法是

• 高斯噪声
• 亮度变化
• 左右翻转
• 上下翻转
• 色彩抖动
• 对化
• 锐度变化

  from PIL import Image,ImageEnhance,ImageFilter,ImageOps
  import os
  import shutil
  import numpy as np
  import cv2
  import random
  from skimage.util import random_noise
  from skimage import exposureimage_number = 0raw_path = "./data/train/"new_path = "./aug/train/"# 加高斯噪声
  def addNoise(img):'''注意：输出的像素是[0,1]之间,所以乘以5得到[0,255]之间'''return random_noise(img, mode='gaussian', seed=13, clip=True)*255def changeLight(img):rate = random.uniform(0.5, 1.5)# print(rate)img = exposure.adjust_gamma(img, rate) #大于1为调暗，小于1为调亮;1.05return imgtry:for i in range(59):os.makedirs(new_path + os.sep + str(i))except:passfor raw_dir_name in range(59):raw_dir_name = str(raw_dir_name)saved_image_path = new_path + raw_dir_name+"/"raw_image_path = raw_path + raw_dir_name+"/"if not os.path.exists(saved_image_path):os.mkdir(saved_image_path)raw_image_file_name = os.listdir(raw_image_path)raw_image_file_path = []for i in raw_image_file_name:raw_image_file_path.append(raw_image_path+i)for x in raw_image_file_path:img = Image.open(x)cv_image = cv2.imread(x)# 高斯噪声gau_image = addNoise(cv_image)# 随机改变light = changeLight(cv_image)light_and_gau = addNoise(light)cv2.imwrite(saved_image_path + "gau_" + os.path.basename(x),gau_image)cv2.imwrite(saved_image_path + "light_" + os.path.basename(x),light)cv2.imwrite(saved_image_path + "gau_light" + os.path.basename(x),light_and_gau)#img = img.resize((800,600))#1.翻转 img_flip_left_right = img.transpose(Image.FLIP_LEFT_RIGHT)img_flip_top_bottom = img.transpose(Image.FLIP_TOP_BOTTOM)#2.旋转 #img_rotate_90 = img.transpose(Image.ROTATE_90)#img_rotate_180 = img.transpose(Image.ROTATE_180)#img_rotate_270 = img.transpose(Image.ROTATE_270)#img_rotate_90_left = img_flip_left_right.transpose(Image.ROTATE_90)#img_rotate_270_left = img_flip_left_right.transpose(Image.ROTATE_270)#3.亮度#enh_bri = ImageEnhance.Brightness(img)#brightness = 1.5#image_brightened = enh_bri.enhance(brightness)#4.色彩#enh_col = ImageEnhance.Color(img)#color = 1.5#image_colored = enh_col.enhance(color)#5.对比度enh_con = ImageEnhance.Contrast(img)contrast = 1.5image_contrasted = enh_con.enhance(contrast)#6.锐度#enh_sha = ImageEnhance.Sharpness(img)#sharpness = 3.0#image_sharped = enh_sha.enhance(sharpness)#保存 img.save(saved_image_path + os.path.basename(x))img_flip_left_right.save(saved_image_path + "left_right_" + os.path.basename(x))img_flip_top_bottom.save(saved_image_path + "top_bottom_" + os.path.basename(x))#img_rotate_90.save(saved_image_path + "rotate_90_" + os.path.basename(x))#img_rotate_180.save(saved_image_path + "rotate_180_" + os.path.basename(x))#img_rotate_270.save(saved_image_path + "rotate_270_" + os.path.basename(x))#img_rotate_90_left.save(saved_image_path + "rotate_90_left_" + os.path.basename(x))#img_rotate_270_left.save(saved_image_path + "rotate_270_left_" + os.path.basename(x))#image_brightened.save(saved_image_path + "brighted_" + os.path.basename(x))#image_colored.save(saved_image_path + "colored_" + os.path.basename(x))image_contrasted.save(saved_image_path + "contrasted_" + os.path.basename(x))#image_sharped.save(saved_image_path + "sharped_" + os.path.basename(x))image_number += 1print("convert pictur" "es :%s size:%s mode:%s" % (image_number, img.size, img.mode))

  加载数据的类（自定义继承）

• 与pytorch中的加载数据类差不多，只是多了自己的某些功能。

• from torch.utils.data import Dataset
  from torchvision import transforms as T
  from config import config
  from PIL import Image
  from itertools import chain
  from glob import glob
  from tqdm import tqdm
  import random
  import numpy as np
  import pandas as pd
  import os
  import cv2
  import torch #1.set random seed
  random.seed(config.seed)
  np.random.seed(config.seed)
  torch.manual_seed(config.seed)
  torch.cuda.manual_seed_all(config.seed)#2.define dataset
  class ZiyiDataset(Dataset):def __init__(self,label_list,transforms=None,train=True,test=False):self.test = test self.train = train imgs = []if self.test:for index,row in label_list.iterrows():imgs.append((row["filename"]))self.imgs = imgs else:for index,row in label_list.iterrows():imgs.append((row["filename"],row["label"]))self.imgs = imgsif transforms is None:if self.test or not train:self.transforms = T.Compose([T.Resize((config.img_weight,config.img_height)),T.ToTensor(),T.Normalize(mean = [0.485,0.456,0.406],std = [0.229,0.224,0.225])])else:self.transforms  = T.Compose([T.Resize((config.img_weight,config.img_height)),T.RandomRotation(30),T.RandomHorizontalFlip(),T.RandomVerticalFlip(),T.RandomAffine(45),T.ToTensor(),T.Normalize(mean = [0.485,0.456,0.406],std = [0.229,0.224,0.225])])else:self.transforms = transformsdef __getitem__(self,index):if self.test:filename = self.imgs[index]img = Image.open(filename)img = self.transforms(img)return img,filenameelse:filename,label = self.imgs[index] img = Image.open(filename)img = self.transforms(img)return img,labeldef __len__(self):return len(self.imgs)def collate_fn(batch):imgs = []label = []for sample in batch:imgs.append(sample[0])label.append(sample[1])return torch.stack(imgs, 0), \labeldef get_files(root,mode):#for testif mode == "test":files = []for img in os.listdir(root):files.append(root + img)files = pd.DataFrame({"filename":files})return fileselif mode != "test": #for train and val       all_data_path,labels = [],[]image_folders = list(map(lambda x:root+x,os.listdir(root)))jpg_image_1 = list(map(lambda x:glob(x+"/*.jpg"),image_folders))jpg_image_2 = list(map(lambda x:glob(x+"/*.JPG"),image_folders))all_images = list(chain.from_iterable(jpg_image_1 + jpg_image_2))print("loading train dataset")for file in tqdm(all_images):all_data_path.append(file)labels.append(int(file.split("/")[-2]))all_files = pd.DataFrame({"filename":all_data_path,"label":labels})return all_fileselse:print("check the mode please!")

  3.获取模型

• 获取模型较为简单，单一模型采取pytorch中的预训练模型，添加所需要的层，进行微调然后迁移学习新数据。

  import torchvision
  import torch.nn.functional as F
  from torch import nn
  from config import configdef generate_model():class DenseModel(nn.Module):def __init__(self, pretrained_model):super(DenseModel, self).__init__()self.classifier = nn.Linear(pretrained_model.classifier.in_features, config.num_classes)for m in self.modules():if isinstance(m, nn.Conv2d):nn.init.kaiming_normal(m.weight)elif isinstance(m, nn.BatchNorm2d):m.weight.data.fill_(1)m.bias.data.zero_()elif isinstance(m, nn.Linear):m.bias.data.zero_()self.features = pretrained_model.featuresself.layer1 = pretrained_model.features._modules['denseblock1']self.layer2 = pretrained_model.features._modules['denseblock2']self.layer3 = pretrained_model.features._modules['denseblock3']self.layer4 = pretrained_model.features._modules['denseblock4']def forward(self, x):features = self.features(x)out = F.relu(features, inplace=True)out = F.avg_pool2d(out, kernel_size=8).view(features.size(0), -1)out = F.sigmoid(self.classifier(out))return outreturn DenseModel(torchvision.models.densenet169(pretrained=True))def get_net():#return MyModel(torchvision.models.resnet101(pretrained = True))model = torchvision.models.resnet50(pretrained = True)    #for param in model.parameters():#    param.requires_grad = False# pytorch添加层的方式直接在Model.层名=层具体形式model.avgpool = nn.AdaptiveAvgPool2d(1)model.fc = nn.Linear(2048,config.num_classes)  #添加全连接层以作分类任务，num_classes为分类个数return model

  4.开始训练

• import os
  import random
  import time
  import json
  import torch
  import torchvision
  import numpy as np
  import pandas as pd
  import warnings
  from datetime import datetime
  from torch import nn,optim
  from config import config
  from collections import OrderedDict
  from torch.autograd import Variable
  from torch.utils.data import DataLoader
  from dataset.dataloader import *
  from sklearn.model_selection import train_test_split,StratifiedKFold
  from timeit import default_timer as timer
  from models.model import *
  from utils import *#1. 设置随机种子 and cudnn performance
  random.seed(config.seed)
  np.random.seed(config.seed)
  torch.manual_seed(config.seed)
  torch.cuda.manual_seed_all(config.seed)
  os.environ["CUDA_VISIBLE_DEVICES"] = config.gpus
  torch.backends.cudnn.benchmark = True
  warnings.filterwarnings('ignore')#2. 评估函数，通过Losses，topk的不断更新来评估模型
  def evaluate(val_loader,model,criterion):#2.1 AverageMeter类是Computes and stores the average and current value# 创建三个其对象，以用于评估losses = AverageMeter()top1 = AverageMeter()top2 = AverageMeter()#2.2 开启评估模式 and confirm model has been transfered to cudamodel.cuda()model.eval()with torch.no_grad():for i,(input,target) in enumerate(val_loader):input = Variable(input).cuda()target = Variable(torch.from_numpy(np.array(target)).long()).cuda()#target = Variable(target).cuda()#2.2.1 compute outputoutput = model(input)loss = criterion(output,target)#2.2.2 measure accuracy and record lossprecision1,precision2 = accuracy(output,target,topk=(1,2))losses.update(loss.item(),input.size(0))top1.update(precision1[0],input.size(0))top2.update(precision2[0],input.size(0))return [losses.avg,top1.avg,top2.avg]#3. test model on public dataset and save the probability matrixdef test(test_loader,model,folds):#3.1 confirm the model converted to cuda# 得出的结果是概率，再用softmax得出最终分类结果csv_map = OrderedDict({"filename":[],"probability":[]})model.cuda()model.eval()with open("./submit/baseline.json","w",encoding="utf-8") as f :submit_results = []for i,(input,filepath) in enumerate(tqdm(test_loader)):# filepath??????# 通过模型得到输出概率结果，再用softmax得出预测结果，写入文件。#3.2 change everything to cuda and get only basenamefilepath = [os.path.basename(x) for x in filepath]with torch.no_grad():image_var = Variable(input).cuda()#3.3.output#print(filepath)#print(input,input.shape)y_pred = model(image_var)#print(y_pred.shape)smax = nn.Softmax(1)smax_out = smax(y_pred)#3.4 save probability to csv filescsv_map["filename"].extend(filepath)for output in smax_out:prob = ";".join([str(i) for i in output.data.tolist()])csv_map["probability"].append(prob)result = pd.DataFrame(csv_map)result["probability"] = result["probability"].map(lambda x : [float(i) for i in x.split(";")])for index, row in result.iterrows():# 因为44，45类删除，所以预测结果加2pred_label = np.argmax(row['probability'])if pred_label > 43:pred_label = pred_label + 2submit_results.append({"image_id":row['filename'],"disease_class":pred_label})json.dump(submit_results,f,ensure_ascii=False,cls = MyEncoder)#4. more details to build main function
  def main():fold = 0#4.1 mkdirsif not os.path.exists(config.submit):os.mkdir(config.submit)if not os.path.exists(config.weights):os.mkdir(config.weights)if not os.path.exists(config.best_models):os.mkdir(config.best_models)if not os.path.exists(config.logs):os.mkdir(config.logs)if not os.path.exists(config.weights + config.model_name + os.sep +str(fold) + os.sep):os.makedirs(config.weights + config.model_name + os.sep +str(fold) + os.sep)if not os.path.exists(config.best_models + config.model_name + os.sep +str(fold) + os.sep):os.makedirs(config.best_models + config.model_name + os.sep +str(fold) + os.sep)       #4.2 get model and optimizermodel = get_net()#model = torch.nn.DataParallel(model)model.cuda()#optimizer = optim.SGD(model.parameters(),lr = config.lr,momentum=0.9,weight_decay=config.weight_decay)optimizer = optim.Adam(model.parameters(),lr = config.lr,amsgrad=True,weight_decay=config.weight_decay)criterion = nn.CrossEntropyLoss().cuda()#criterion = FocalLoss().cuda()log = Logger()log.open(config.logs + "log_train.txt",mode="a")log.write("\n----------------------------------------------- [START %s] %s\n\n" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'), '-' * 51))#4.3 some parameters for  K-fold and restart modelstart_epoch = 0best_precision1 = 0best_precision_save = 0resume = False#4.4 restart the training processif resume:checkpoint = torch.load(config.best_models + str(fold) + "/model_best.pth.tar")start_epoch = checkpoint["epoch"]fold = checkpoint["fold"]best_precision1 = checkpoint["best_precision1"]model.load_state_dict(checkpoint["state_dict"])optimizer.load_state_dict(checkpoint["optimizer"])#4.5 get files and split for K-fold dataset#4.5.1 read filestrain_ = get_files(config.train_data,"train")#val_data_list = get_files(config.val_data,"val")test_files = get_files(config.test_data,"test")""" #4.5.2 splitsplit_fold = StratifiedKFold(n_splits=3)folds_indexes = split_fold.split(X=origin_files["filename"],y=origin_files["label"])folds_indexes = np.array(list(folds_indexes))fold_index = folds_indexes[fold]#4.5.3 using fold index to split for train data and val datatrain_data_list = pd.concat([origin_files["filename"][fold_index[0]],origin_files["label"][fold_index[0]]],axis=1)val_data_list = pd.concat([origin_files["filename"][fold_index[1]],origin_files["label"][fold_index[1]]],axis=1)"""train_data_list,val_data_list = train_test_split(train_,test_size = 0.15,stratify=train_["label"])#4.5.4 load datasettrain_dataloader = DataLoader(ZiyiDataset(train_data_list),batch_size=config.batch_size,shuffle=True,collate_fn=collate_fn,pin_memory=True)val_dataloader = DataLoader(ZiyiDataset(val_data_list,train=False),batch_size=config.batch_size,shuffle=True,collate_fn=collate_fn,pin_memory=False)test_dataloader = DataLoader(ZiyiDataset(test_files,test=True),batch_size=1,shuffle=False,pin_memory=False)#scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,"max",verbose=1,patience=3)scheduler =  optim.lr_scheduler.StepLR(optimizer,step_size = 10,gamma=0.1)# optim.lr_scheduler 提供了基于多种epoch数目调整学习率的方法# step_size(整数类型): 调整学习率的步长,每过step_size次,更新一次学习率# gamma(float 类型):学习率下降的乘数因子#4.5.5.1 define metricstrain_losses = AverageMeter()train_top1 = AverageMeter()train_top2 = AverageMeter()valid_loss = [np.inf,0,0]model.train()#logslog.write('** start training here! **\n')log.write('                           |------------ VALID -------------|----------- TRAIN -------------|------Accuracy------|------------|\n')log.write('lr       iter     epoch    | loss   top-1  top-2            | loss   top-1  top-2           |    Current Best    | time       |\n')log.write('-------------------------------------------------------------------------------------------------------------------------------\n')#4.5.5 trainstart = timer()for epoch in range(start_epoch,config.epochs):# 一个epoch为所有数据迭代一次进入模型拟合的过程，其中又分为batch_size来分批次进行scheduler.step(epoch)# train#global iterfor iter,(input,target) in enumerate(train_dataloader):#4.5.5 switch to continue train processmodel.train()input = Variable(input).cuda()target = Variable(torch.from_numpy(np.array(target)).long()).cuda()#target = Variable(target).cuda()output = model(input)loss = criterion(output,target)precision1_train,precision2_train = accuracy(output,target,topk=(1,2))train_losses.update(loss.item(),input.size(0))train_top1.update(precision1_train[0],input.size(0))train_top2.update(precision2_train[0],input.size(0))#backwardoptimizer.zero_grad()loss.backward()optimizer.step()lr = get_learning_rate(optimizer)print('\r',end='',flush=True)print('%0.4f %5.1f %6.1f        | %0.3f  %0.3f  %0.3f         | %0.3f  %0.3f  %0.3f         |         %s         | %s' % (\lr, iter/len(train_dataloader) + epoch, epoch,valid_loss[0], valid_loss[1], valid_loss[2],train_losses.avg, train_top1.avg, train_top2.avg,str(best_precision_save),time_to_str((timer() - start),'min')), end='',flush=True)#evaluatelr = get_learning_rate(optimizer)#evaluate every half epochvalid_loss = evaluate(val_dataloader,model,criterion)is_best = valid_loss[1] > best_precision1best_precision1 = max(valid_loss[1],best_precision1)try:best_precision_save = best_precision1.cpu().data.numpy()except:passsave_checkpoint({"epoch":epoch + 1,"model_name":config.model_name,"state_dict":model.state_dict(),"best_precision1":best_precision1,"optimizer":optimizer.state_dict(),"fold":fold,"valid_loss":valid_loss,},is_best,fold)#adjust learning rate#scheduler.step(valid_loss[1])print("\r",end="",flush=True)log.write('%0.4f %5.1f %6.1f        | %0.3f  %0.3f  %0.3f          | %0.3f  %0.3f  %0.3f         |         %s         | %s' % (\lr, 0 + epoch, epoch,valid_loss[0], valid_loss[1], valid_loss[2],train_losses.avg,    train_top1.avg,    train_top2.avg, str(best_precision_save),time_to_str((timer() - start),'min')))log.write('\n')time.sleep(0.01)best_model = torch.load(config.best_models + os.sep+config.model_name+os.sep+ str(fold) +os.sep+ 'model_best.pth.tar')model.load_state_dict(best_model["state_dict"])test(test_dataloader,model,fold)if __name__ =="__main__":main()

【转载】使用Pytorch进行图像分类，AI challenger 农作物病害分类竞赛源码解读相关推荐

1. 世界首个！AI农作物病害检测竞赛火热进行中 | AI Challenger 全球AI挑战赛

   乾明 发自 凹非寺 量子位 出品 | 公众号 QbitAI 如果你用谷歌搜索"AI+农业"或者"人工智能+农业",就会发现与AI在其他领域的应用相比,农业依旧是 ...

2. 基于【EasyDL】【图像分类】实现农作物病害识别小程序

   内容.数据集来源:基于飞桨的农作物病害智能识别系统 - 飞桨AI Studio 项目背景 联合国粮食及农业组织的一份报告表明,每年农业生产的自然损失中有三分之一以上是由农业病虫害造成的,使这些成为当前 ...

3. PyTorch 源码解读之即时编译篇

   点击上方"AI遇见机器学习",选择"星标"公众号 重磅干货,第一时间送达 作者丨OpenMMLab 来源丨https://zhuanlan.zhihu.com/ ...

4. 小程序源码：AI微信小程序源码下载人脸照片AI转换动漫照片全新源码安装简单无需服务器域名-多玩法安装简单

   这是一款AI人脸转动漫的一款微信小程序源码 该款小程序源码无需服务器和域名 搭建安装简单 有多种风格转换模式可自选 安装教程: 首先解压源码然后把源码上传到微信开发者工具打开 另外设置一个合法域名 合 ...

5. 基于飞桨图像分类套件PaddleClas的柠檬分类竞赛实战

   前情提要   通过之前教程中的学习,相信大家对于如何搭建一个分类网络已经清晰了.那么我们不禁会想,有没有更快速的尝试模型及技巧的方法呢?因为我们在上一次课程中使用的代码都需要自己进行开发,自己写需要很 ...

6. 【 非线性回归 Logistics-Regression 模块实现与源码解读 深度学习 Pytorch笔记 B站刘二大人（5/10）】

   非线性回归 Logistics-Regression 模块实现与源码解读 深度学习 Pytorch笔记 B站刘二大人(5/10) 数学推导 什么是logistics函数 在定义上Logistic函数或 ...

7. Pytorch Bert源码解读

   Bert 源码解读 从 CV 转 NLP 有小半年了,一直在用 Bert 系列做一些自然语言理解任务,包括命名实体识别.关系抽取,文本结构化等信息抽取任务.之前开发任务紧,Bert 拿来就用了,很多代 ...

8. 基于改进PSGAN的AI自动美颜化妆系统（源码＆教程）

   1.研究背景 在计算机视觉领域,基于人脸图像的合成与分析一直是热门的研究课题.人脸表情合成在影视娱乐.人工智能产业应用广泛,自动化妆技术在美妆行业有着巨大的应用前景.虽然人脸图像合成技术在近几年得到了 ...

9. 伪原创php下载,PHP语言开发在线智能AI文章伪原创网站程序源码下载

   源码介绍 PHP语言开发在线智能AI文章伪原创网站程序源码,无需后台,直接将该程序源码上传到空间任意目录下即可以使用. 如果不想上传到网站根目录,则打开index.html文件,修改css和js文件地 ...

最新文章

1. C#Swagger使用
2. 【c语言】蓝桥杯基础练习 闰年判断
3. 给迷茫的Java员一些中肯建议，你还在虚度光阴吗？
4. python项目面试_Python面试中最常见的25个问题-结束
5. 手机移动端网站建设这些细节问题需重视
6. python网络爬虫工程师薪资-月薪2万的爬虫工程师，Python需要学到什么程度？
7. python基础-字典(8)
8. 数据库服务器网页,服务器 数据库 网页前端
9. SpringMVC 参数校验
10. 博文视点Open Party第8期
11. 武汉市房价数据挖掘与可视化分析（Python）
12. JAVA设计模式 — 生成器模式(Builder)
13. 守护进程杀手(测试)
14. pycharm interpreter invalid
15. Android Studio 解决数据库手机电脑不同步
16. android des 加密
17. python壁纸数据抓取_Python爬虫：爬取必应壁纸(可直接运行)
18. 一加11：新的赛场，“不温和”的答卷
19. 安装java进度条不动了_提示安装过程出错怎么处理啊？安装offi – 手机爱问
20. 我的PCB走线经验归纳

热门文章

1. 【codecs】JPEG、MPEG-1、MPEG-2和MPEG-4编解码流程对比
2. base64编码类------源代码（C#）
3. 需求跟踪系列 III - 建立需求关联最佳的方式和时间点
4. 深入浅出JS—11 ES6中类class和extends的使用
5. 关于射线检测与碰撞检测
6. 负数十进制转换二进制
7. C++中using的四大用法总结
8. 怎样在ipad安装ipa文件
9. [Android Input系统]MotionEvent的序列化传送
10. 旅游景区游玩管理系统