手写字母数据集转换为.pickle文件

首先是数据集，我上传了相关的资源，https://download.csdn.net/download/fanzonghao/10566701

转换代码如下：

import numpy as np
import os
import matplotlib.pyplot as plt
import matplotlib.image as mpig
import imageio
import pickle
"""
函数功能：将notMNIST_large和notMNIST_small的图片生成对应的.pickle文件
"""
def load_letter(folder,min_num_images,image_size):image_files=os.listdir(folder)print(folder)#定义存放图片的numpy类型dataset=np.ndarray(shape=(len(image_files),image_size,image_size),dtype=np.float32)num_image=0for image in image_files:image_file=os.path.join(folder,image)try:image_data=(mpig.imread(image_file)-0.5)/1assert image_data.shape==(image_size,image_size)dataset[num_image,:,:]=image_datanum_image+=1except(IOError,ValueError)as e:print('could not read:',image_file,e,'skipping')#提示所需样本数少if num_image<min_num_images:raise Exception('samples is few {}<{}'.format(num_image,min_num_images))dataset=dataset[0:num_image,:,:]#去掉没能读取图片的列表print('full dataset tensor:',dataset.shape)print('Mean:',np.mean(dataset))print('Standard deviation:',np.std(dataset))return dataset
"""
将训练样本和测试样本图片输出为.pickle形式
"""
def deal_data(base_dir,min_num_images):data_folders=[os.path.join(base_dir, i) for i in sorted(os.listdir(base_dir))]dataset_names=[]for folder in data_folders:set_filename=folder+'.pickle'dataset_names.append(set_filename)dataset=load_letter(folder, min_num_images, image_size=28)try:with open(set_filename,'wb') as f:pickle.dump(dataset,f,pickle.HIGHEST_PROTOCOL)except Exception as e:print('unable to save data',set_filename,e)print(dataset_names)return dataset_names
"""
给定训练和测试样本路径
"""
def produce_train_test_pickle():train_dir = './data/notMNIST_large'deal_data(train_dir,min_num_images=45000)test_dir = './data/notMNIST_small'deal_data(test_dir,min_num_images=1800)# #测试 imageio模块和matploab image的区别
# def test():
#     image_path = os.path.join('./data/notMNIST_small', 'MDEtMDEtMDAudHRm.png')
#     # 用matploab image 读出来的像素处于0～1之间
#     image = mpig.imread(image_path)
#     print(image.shape)
#     print((image - 0.5) / 1)
#     plt.subplot(121)
#     plt.imshow(image)
#
#     # 用imageio模块 读出来的像素处于0～255之间
#     image = imageio.imread(image_path)
#     print(image.shape)
#     print((image - 255 / 2) / 255)
#     plt.subplot(122)
#     plt.imshow(image)
#
#     plt.show()
if __name__ == '__main__':# test()produce_train_test_pickle()# #结果保存输出路径
# output_path='./data/notMNIST_small/Pickles'
# if not os.path.exists(output_path):
#     os.makedirs(output_path)

打印生成的结果：

将两个数据集的手写字母生成的.pickle转换成整个.pickle数据集，这样在使用的时候方便直接调用,代码如下：

import numpy as np
import data_deal
import os
import pickle
"""
函数功能：功能1：调用把图片文件生成pickle文件的功能2：通过把生成的pickle文件调用生成train_dataset和valid_dataset和test_dataset
"""
#生成.pickle文件  没有的时候才执行
# data_deal.produce_train_test_pickle()"""
生成所需数据的np array
"""
def make_array(rows,img_size):if rows:dataset=np.ndarray(shape=(rows,img_size,img_size),dtype=np.float32)labels=np.ndarray(shape=(rows,),dtype=np.int32)else:dataset, labels=None,Nonereturn dataset,labels
"""
生成训练集和测试集 dataset
"""
def produce_train_test_datasets(pickle_files,train_size,valid_size=0):num_classes=len(pickle_files)valid_dataset,valid_lable=make_array(valid_size, img_size=28)train_dataset, train_lable = make_array(train_size, img_size=28)#小数据量存储近train_dataset和valid_datasetvalid_size_per_class = valid_size // num_classestrain_size_per_class = train_size // num_classesstart_v,start_t=0,0end_v,end_t=valid_size_per_class,train_size_per_classend_l=valid_size_per_class+train_size_per_classfor lable,pickle_file in enumerate(pickle_files):with open(pickle_file,'rb') as f:#载入每个字母的pickleevery_letter_samples=pickle.load(f)#打乱顺序 (7000,28,28)对下一层进行打乱操作 直接改变原有的顺序np.random.shuffle(every_letter_samples)#制作验证集if valid_dataset is not None:#放入test数据不需要valid_datasetvalid_letter=every_letter_samples[:valid_size_per_class,:,:]valid_dataset[start_v:end_v,:,:]=valid_lettervalid_lable[start_v:end_v]=lablestart_v+=valid_size_per_classend_v+=valid_size_per_class# 制作训练集train_letter = every_letter_samples[valid_size_per_class:end_l, :, :]train_dataset[start_t:end_t, :, :] = train_lettertrain_lable[start_t:end_t] = lablestart_t += train_size_per_classend_t += train_size_per_classreturn valid_dataset,valid_lable,train_dataset,train_lable
"""
实现训练样本 测试样本的A~j顺序打乱
"""
def random_letter(dataset,labels):#获取打乱的索引permutation=np.random.permutation(labels.shape[0])dataset=dataset[permutation,:,:]labels=labels[permutation]return dataset,labels
"""
生成最终的notMNIST.pickle 包含train valid test
"""
def notMNIST_pickle():train_size=200000valid_size=1000test_size=1000train_dir = './data/notMNIST_large/Pickles'train_pickle_dir=[os.path.join(train_dir,i) for i in sorted(os.listdir(train_dir))]valid_dataset,valid_lable,train_dataset,train_lable=produce_train_test_datasets(train_pickle_dir,train_size,valid_size)test_dir = './data/notMNIST_small/Pickles'test_pickle_dir=[os.path.join(test_dir,i) for i in sorted(os.listdir(test_dir))]_,_,test_dataset,test_lable=produce_train_test_datasets(test_pickle_dir,test_size)print('Training',train_dataset.shape,train_lable.shape)print('Validing',valid_dataset.shape,valid_lable.shape)print('Testing',test_dataset.shape,test_lable.shape)train_dataset,train_label=random_letter(train_dataset,train_lable)valid_dataset, valid_label = random_letter(valid_dataset, valid_lable)test_dataset, test_label = random_letter(test_dataset, test_lable)print('after shuffle training',train_dataset.shape,train_label.shape)print('after shuffle validing',valid_dataset.shape,valid_label.shape)print('after shuffle testing',test_dataset.shape,test_label.shape)all_pickle_file=os.path.join('./data','notMNIST.pickle')try:with open(all_pickle_file, 'wb') as f:save={'train_dataset':train_dataset,'train_label': train_label,'valid_dataset': valid_dataset,'valid_label': valid_label,'test_dataset': test_dataset,'test_label': test_label,}pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)except Exception as e:print('unable to save data', all_pickle_file, e)statinfo=os.stat(all_pickle_file)print('Compressed pickle size',statinfo.st_size)
if __name__ == '__main__':notMNIST_pickle()

读取.pickle

import tensorflow as tf
import numpy as np
import pickle
import matplotlib.pyplot as plt
#对于x变成(samles,pixs),y变成one_hot (samples,10)
"""
one-hot
"""
def reformat(dataset,labels,imgsize,C):dataset=dataset.reshape(-1,imgsize*imgsize).astype(np.float32)#one_hot两种写法#写法一labels=np.eye(C)[labels.reshape(-1)].astype(np.float32)#写法二#labels=(np.arange(10)==labels[:,None]).astype(np.float32)return dataset,labels
"""
读取.pickle文件
"""
def pickle_dataset():path='./data/notMNIST.pickle'with open(path,'rb') as f:restore=pickle.load(f)train_dataset=restore['train_dataset']train_label = restore['train_label']valid_dataset = restore['valid_dataset']valid_label = restore['valid_label']test_dataset = restore['test_dataset']test_label = restore['test_label']del restore# print('Training:', train_dataset.shape, train_label.shape)# print('Validing:', valid_dataset.shape, valid_label.shape)# print('Testing:', test_dataset.shape, test_label.shape)train_dataset,train_label=reformat(train_dataset,train_label,imgsize=28,C=10)valid_dataset,valid_label=reformat(valid_dataset,valid_label,imgsize=28,C=10)test_dataset,test_label=reformat(test_dataset,test_label,imgsize=28,C=10)# print('after Training:', train_dataset.shape, train_label.shape)# print('after Validing:', valid_dataset.shape, valid_label.shape)# print('after Testing:', test_dataset.shape, test_label.shape)return train_dataset,train_label,valid_dataset,valid_label,test_dataset,test_label# #测试生成的数据正确不
# def test(train_dataset,train_label):
#     print(train_label[:10])
#     #plt.figure(figsize=(50,20))
#     for i in range(10):
#         plt.subplot(5,2,i+1)
#         plt.imshow(train_dataset[i].reshape(28,28))
#     plt.show()# if __name__ == '__main__':
#     test(train_dataset,train_label)

手写字母数据集转换为.pickle文件相关推荐

NIN模块tensorflow实现和一个自己制作的手写字母数据集
NIN层简介: 我们提出了一种新型的深度网络结构,称为"Network In Network"(NIN),它可以增强模型在感受野(receptive field)内对局部区域(l ...
基于tensorflow、keras利用emnist数据集构建CNN卷积神经网络进行手写字母识别
EMNIST 数据集是一个包含手写字母,数字的数据集,它具有和MNIST相同的数据格式.The EMNIST Dataset | NIST 引用模块介绍: import tensorflow as t ...
python手写字母识别_机器学习--kNN算法识别手写字母
本文主要是用kNN算法对字母图片进行特征提取,分类识别.内容如下: kNN算法及相关Python模块介绍对字母图片进行特征提取 kNN算法实现 kNN算法分析一.kNN算法介绍 K近邻(kNN,k ...
Pytorch 实现全连接神经网络/卷积神经网络训练MNIST数据集，并将训练好的模型在制作自己的手写图片数据集上测试
使用教程代码下载地址:点我下载模型在训练过程中会自动显示训练进度,如果您的pytorch是CPU版本的,代码会自动选择CPU训练,如果有cuda,则会选择GPU训练. 项目目录说明: CNN文件夹 ...
使用MATLAB实现基于BP神经网络训练的手写字母识别程序
前言大三的时候利用MATLAB搭建了一个基于BP神经网络框架的手写字母识别程序,其中使用了EMNIST数据集进行训练和测试,可实时对手写输入样本进行识别,并返回两个最可能的结果,过程中可继续添加样本 ...
将MNIST手写数字数据集导入NumPy数组（《深度学习入门：基于Python的理论与实现》实践笔记）
将MNIST手写数字数据集导入NumPy数组(<深度学习入门:基于Python的理论与实现>实践笔记) 一.下载MNIST数据集(使用urllib.request.urlretrieve( ...
BP算法实现--minst手写数字数据集识别
实验步骤初始化网络架构网络层数,每层神经元数,连接神经元的突触权重,每个神经元的偏置构造bp算法函数对于一个输入数据,前向计算每层的输出值,保存未激活的输出和激活过的输出值,这里用的激活函数是 ...
教你使用TensorFlow2对阿拉伯语手写字符数据集进行识别
@Author:Runsen 在本教程中,我们将使用 TensorFlow (Keras API) 实现一个用于多分类任务的深度学习模型,该任务需要对阿拉伯语手写字符数据集进行识别. 数据集下载地址: ...
多种深度模型实现手写字母MNIST的识别(CNN,RNN,DNN,逻辑回归,CRNN,LSTM/Bi-LSTM,GRU/Bi-GRU)
多种深度模型实现手写字母MNIST的识别(CNN,RNN,DNN,逻辑回归,CRNN,LSTM/Bi-LSTM,GRU/Bi-GRU 1.CNN模型 1.1 代码 1.2 运行结果 2.RNN模型 2 ...

手写字母数据集转换为.pickle文件

手写字母数据集转换为.pickle文件相关推荐

最新文章

热门文章