李沐《动手学深度学习》第二版比赛2-Classify Leaves

我的偶像，李沐大神主讲的《动手学深度学习》(使用Pytorch框架，第一版使用的是MXNet框架)目前已经进行到了双向循环神经网络。第二部分(卷积神经网络)的竞赛内容为树叶分类。

首先导入需要的包

# 首先导入包
import torch
import torch.nn as nn
import pandas as pd
import numpy as np
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms
from PIL import Image
import os
import matplotlib.pyplot as plt
import torchvision.models as models
# This is for the progress bar.
from tqdm import tqdm
import seaborn as sns

使用pd.read_csv将训练集表格读入，然后看看label文件长啥样，image栏是图片的名称，label是图片的分类标签。

labels_dataframe = pd.read_csv('./classify-leaves/train.csv')
labels_dataframe.head(10)

使用pd.describe()函数生成描述性统计数据，统计数据集的集中趋势，分散和行列的分布情况，不包括 NaN值。可以看到训练集总共有18353张图片，标签有176类。

labels_dataframe.describe()

用条形图可视化176类图片的分布（数目）。

# function to show bar lengthdef barw(ax): for p in ax.patches:val = p.get_width() # height of the barx = p.get_x()+ p.get_width() # x- position y = p.get_y() + p.get_height()/2 # y-positionax.annotate(round(val,2),(x,y))# finding top leavesplt.figure(figsize = (15,30))
# 类别特征的频数条形图（x轴是count数，y轴是类别。）
ax0 =sns.countplot(y=labels_dataframe['label'],order=labels_dataframe['label'].value_counts().index)
barw(ax0)
plt.show()

把label标签按字母排个序，这里仅显示前10个。

# 把label文件排个序
leaves_labels = sorted(list(set(labels_dataframe['label'])))
n_classes = len(leaves_labels)
print(n_classes)
leaves_labels[:10]

把label和176类zip一下再字典，把label转成对应的数字。

# 把label转成对应的数字
class_to_num = dict(zip(leaves_labels, range(n_classes)))
class_to_num

再将类别数转换回label，方便最后预测的时候使用。

# 再转换回来，方便最后预测的时候使用
num_to_class = {v : k for k, v in class_to_num.items()}

创建树叶数据集类LeavesData(Dataset)，用来批量管理训练集、验证集和测试集。

# 继承pytorch的dataset，创建自己的
class LeavesData(Dataset):def __init__(self, csv_path, file_path, mode='train', valid_ratio=0.2, resize_height=256, resize_width=256):"""Args:csv_path (string): csv 文件路径img_path (string): 图像文件所在路径mode (string): 训练模式还是测试模式valid_ratio (float): 验证集比例"""# 需要调整后的照片尺寸，我这里每张图片的大小尺寸不一致#self.resize_height = resize_heightself.resize_width = resize_widthself.file_path = file_pathself.mode = mode# 读取 csv 文件# 利用pandas读取csv文件self.data_info = pd.read_csv(csv_path, header=None)  #header=None是去掉表头部分# 计算 lengthself.data_len = len(self.data_info.index) - 1self.train_len = int(self.data_len * (1 - valid_ratio))if mode == 'train':# 第一列包含图像文件的名称self.train_image = np.asarray(self.data_info.iloc[1:self.train_len, 0])  #self.data_info.iloc[1:,0]表示读取第一列，从第二行开始到train_len# 第二列是图像的 labelself.train_label = np.asarray(self.data_info.iloc[1:self.train_len, 1])self.image_arr = self.train_image self.label_arr = self.train_labelelif mode == 'valid':self.valid_image = np.asarray(self.data_info.iloc[self.train_len:, 0])  self.valid_label = np.asarray(self.data_info.iloc[self.train_len:, 1])self.image_arr = self.valid_imageself.label_arr = self.valid_labelelif mode == 'test':self.test_image = np.asarray(self.data_info.iloc[1:, 0])self.image_arr = self.test_imageself.real_len = len(self.image_arr)print('Finished reading the {} set of Leaves Dataset ({} samples found)'.format(mode, self.real_len))def __getitem__(self, index):# 从 image_arr中得到索引对应的文件名single_image_name = self.image_arr[index]# 读取图像文件img_as_img = Image.open(self.file_path + single_image_name)#如果需要将RGB三通道的图片转换成灰度图片可参考下面两行
#         if img_as_img.mode != 'L':
#             img_as_img = img_as_img.convert('L')#设置好需要转换的变量，还可以包括一系列的nomarlize等等操作if self.mode == 'train':transform = transforms.Compose([transforms.Resize((224, 224)),transforms.RandomHorizontalFlip(p=0.5),   #随机水平翻转 选择一个概率transforms.ToTensor()])else:# valid和test不做数据增强transform = transforms.Compose([transforms.Resize((224, 224)),transforms.ToTensor()])img_as_img = transform(img_as_img)if self.mode == 'test':return img_as_imgelse:# 得到图像的 string labellabel = self.label_arr[index]# number labelnumber_label = class_to_num[label]return img_as_img, number_label  #返回每一个index对应的图片数据和对应的labeldef __len__(self):return self.real_len

定义一下不同数据集的csv_path，并通过更改mode修改数据集类的实例对象。

train_path = './classify-leaves/train.csv'
test_path = './classify-leaves/test.csv'
# csv文件中已经images的路径了，因此这里只到上一级目录
img_path = './classify-leaves/'train_dataset = LeavesData(train_path, img_path, mode='train')
val_dataset = LeavesData(train_path, img_path, mode='valid')
test_dataset = LeavesData(test_path, img_path, mode='test')
print(train_dataset)
print(val_dataset)
print(test_dataset)

定义data loader，设置batch_size。

# 定义data loader
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,batch_size=8, shuffle=False,num_workers=5)val_loader = torch.utils.data.DataLoader(dataset=val_dataset,batch_size=8, shuffle=False,num_workers=5)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,batch_size=8, shuffle=False,num_workers=5)

展示数据

# 给大家展示一下数据长啥样
def im_convert(tensor):""" 展示数据"""image = tensor.to("cpu").clone().detach()image = image.numpy().squeeze()image = image.transpose(1,2,0)image = image.clip(0, 1)return imagefig=plt.figure(figsize=(20, 12))
columns = 4
rows = 2dataiter = iter(val_loader)
inputs, classes = dataiter.next()for idx in range (columns*rows):ax = fig.add_subplot(rows, columns, idx+1, xticks=[], yticks=[])ax.set_title(num_to_class[int(classes[idx])])plt.imshow(im_convert(inputs[idx]))
plt.show()

# 看一下是在cpu还是GPU上
def get_device():return 'cuda' if torch.cuda.is_available() else 'cpu'device = get_device()
print(device)

# 是否要冻住模型的前面一些层
def set_parameter_requires_grad(model, feature_extracting):if feature_extracting:model = modelfor param in model.parameters():param.requires_grad = False
# resnet34模型
def res_model(num_classes, feature_extract = False, use_pretrained=True):model_ft = models.resnet34(pretrained=use_pretrained)set_parameter_requires_grad(model_ft, feature_extract)num_ftrs = model_ft.fc.in_featuresmodel_ft.fc = nn.Sequential(nn.Linear(num_ftrs, num_classes))return model_ft

# 超参数
learning_rate = 3e-4
weight_decay = 1e-3
num_epoch = 50
model_path = './pre_res_model.ckpt'

# Initialize a model, and put it on the device specified.
model = res_model(176)
model = model.to(device)
model.device = device
# For the classification task, we use cross-entropy as the measurement of performance.
criterion = nn.CrossEntropyLoss()# Initialize optimizer, you may fine-tune some hyperparameters such as learning rate on your own.
optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate, weight_decay=weight_decay)# The number of training epochs.
n_epochs = num_epochbest_acc = 0.0
for epoch in range(n_epochs):# ---------- Training ----------# Make sure the model is in train mode before training.model.train() # These are used to record information in training.train_loss = []train_accs = []# Iterate the training set by batches.for batch in tqdm(train_loader):# A batch consists of image data and corresponding labels.imgs, labels = batchimgs = imgs.to(device)labels = labels.to(device)# Forward the data. (Make sure data and model are on the same device.)logits = model(imgs)# Calculate the cross-entropy loss.# We don't need to apply softmax before computing cross-entropy as it is done automatically.loss = criterion(logits, labels)# Gradients stored in the parameters in the previous step should be cleared out first.optimizer.zero_grad()# Compute the gradients for parameters.loss.backward()# Update the parameters with computed gradients.optimizer.step()# Compute the accuracy for current batch.acc = (logits.argmax(dim=-1) == labels).float().mean()# Record the loss and accuracy.train_loss.append(loss.item())train_accs.append(acc)# The average loss and accuracy of the training set is the average of the recorded values.train_loss = sum(train_loss) / len(train_loss)train_acc = sum(train_accs) / len(train_accs)# Print the information.print(f"[ Train | {epoch + 1:03d}/{n_epochs:03d} ] loss = {train_loss:.5f}, acc = {train_acc:.5f}")# ---------- Validation ----------# Make sure the model is in eval mode so that some modules like dropout are disabled and work normally.model.eval()# These are used to record information in validation.valid_loss = []valid_accs = []# Iterate the validation set by batches.for batch in tqdm(val_loader):imgs, labels = batch# We don't need gradient in validation.# Using torch.no_grad() accelerates the forward process.with torch.no_grad():logits = model(imgs.to(device))# We can still compute the loss (but not the gradient).loss = criterion(logits, labels.to(device))# Compute the accuracy for current batch.acc = (logits.argmax(dim=-1) == labels.to(device)).float().mean()# Record the loss and accuracy.valid_loss.append(loss.item())valid_accs.append(acc)# The average loss and accuracy for entire validation set is the average of the recorded values.valid_loss = sum(valid_loss) / len(valid_loss)valid_acc = sum(valid_accs) / len(valid_accs)# Print the information.print(f"[ Valid | {epoch + 1:03d}/{n_epochs:03d} ] loss = {valid_loss:.5f}, acc = {valid_acc:.5f}")# if the model improves, save a checkpoint at this epochif valid_acc > best_acc:best_acc = valid_acctorch.save(model.state_dict(), model_path)print('saving model with acc {:.3f}'.format(best_acc))

saveFileName = './classify-leaves/submission.csv'

## predict
model = res_model(176)# create model and load weights from checkpoint
model = model.to(device)
model.load_state_dict(torch.load(model_path))# Make sure the model is in eval mode.
# Some modules like Dropout or BatchNorm affect if the model is in training mode.
model.eval()# Initialize a list to store the predictions.
predictions = []
# Iterate the testing set by batches.
for batch in tqdm(test_loader):imgs = batchwith torch.no_grad():logits = model(imgs.to(device))# Take the class with greatest logit as prediction and record it.predictions.extend(logits.argmax(dim=-1).cpu().numpy().tolist())preds = []
for i in predictions:preds.append(num_to_class[i])test_data = pd.read_csv(test_path)
test_data['label'] = pd.Series(preds)
submission = pd.concat([test_data['image'], test_data['label']], axis=1)
submission.to_csv(saveFileName, index=False)
print("Done!!!!!!!!!!!!!!!!!!!!!!!!!!!")

参考文献

https://www.kaggle.com/c/classify-leaves 比赛平台
https://www.cnblogs.com/zgqcn/p/14160093.html kaggle 训练操作
https://www.kaggle.com/nekokiku/simple-resnet-baseline 大神提供的baseline

李沐《动手学深度学习》第二版比赛2-Classify Leaves相关推荐

李沐-动手学深度学习-pytorch版-”d2lzh_pytorch”包的使用
github上直接的代码 github上需要导入时写的代码块 import torch import torchvision import numpy as np import sys sys.pat ...
李沐动手学深度学习V2-全卷积网络FCN和代码实现
一.全卷积网络FCN 1. 介绍语义分割是对图像中的每个像素分类,全卷积网络(fully convolutional network,FCN)采用卷积神经网络实现了从图像像素到像素类别的变换 ,与前 ...
李沐动手学深度学习v2/总结1
总结编码过程数据数据预处理模型参数,初始化参数超参数损失函数,先计算损失,清空梯度(防止有累积的梯度),再对损失后向传播计算损失关于参数的梯度优化算法,使用优化算法更新参数训练求参数 ...
李沐动手学深度学习(pytorch版本)d2lzh_pytorch包的缺少安装问题
学习深度学习时候,很多人参考的是李沐的动手学深度学习Pytorch版本(附上官方地址:https://tangshusen.me/Dive-into-DL-PyTorch/#/). 在学习3.5.1节 ...
关于李沐动手学深度学习（d2l）pytorch环境本地配置
本地安装d2l 由于之前试了很多次d2l课本的安装方法失败了,这里提供一种我可以成功安装d2l包的方法. pytorch安装首先安装cuda.cudnn.pytroch(gpu版本).可以参考这篇文 ...
【李沐动手学深度学习】读书笔记 01前言
虽然之前已经学过这部分内容和深度学习中的基础知识,但总觉得学的不够系统扎实,所以希望再通过沐神的课程以及书籍,系统条理的学习一遍.在读书过程中,利用导图做了一下梳理,形成了这个读书笔记.如有侵权,请联 ...
李沐动手学深度学习：08 线性回归（代码逐行理解）
目录一.相关资料连接 1.1 李沐视频 1.2 代码.PPT 二.代码及笔记(使用Jupyter Notebook) 2.1 线性回归从零开始实现 2.1.1 基本概念 2.1.2 基础优化算法 2 ...
windows上配置深度学习（李沐-动手学深度学习）
1.安装miniconda windows下安装,去清华大学开源镜像下载,速度比较快. 选中Miniconda3-latest-Windos-x86_64.exe下载安装包(目前最新的是py3.9) ...
李沐动手学深度学习第四章-4.9.环境和分布偏移
我们从来没有想过数据最初从哪里来?以及我们计划最终如何处理模型的输出? 根据测试集的精度衡量,模型表现得非常出色. 但是当数据分布突然改变时,模型在部署中会出现灾难性的失败. 解决方案很简单(要求&q ...
李沐动手学深度学习V2-图像增广和代码实现
图像增广大型数据集是成功应用深度神经网络的先决条件,因为解决了大型复杂网络的过拟合性. 图像增广在对训练图像进行一系列的随机变化之后,生成相似但不同的训练样本,从而扩大了训练集的规模. 此外,应用图 ...

李沐《动手学深度学习》第二版比赛2-Classify Leaves

李沐《动手学深度学习》第二版比赛2-Classify Leaves

参考文献

李沐《动手学深度学习》第二版比赛2-Classify Leaves相关推荐

最新文章

热门文章