Python 实现深度学习（2） 第一篇：基础知识简介

1.5 实践：mnist数据集解析

本章会撰写程序实现一下功能：

1.下载mnsit数据集，解析mnist数据放在numpy的array中;

2.将解析的数据先序列化，然后持久化

3.反序列化，读取mnist中的一样图像，用plt或者skimage显示。

  1 # -*- coding: utf-8 -*-
  2 # @File  : mnist.py
  3 # @Author: lizhen
  4 # @Date  : 2020/2/4
  5 # @Desc  : 工具类，datasets/mnist.py
  6 
  7 import urllib.request # python3
  8 import os.path
  9 import gzip
 10 import pickle
 11 import os
 12 import numpy as np
 13 
 14 # http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz
 15 # http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz
 16 # http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz
 17 # http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz
 18 
 19 
 20 url_base = "http://yann.lecun.com/exdb/mnist/"
 21 key_file = {
 22         'train_img':'train-images-idx3-ubyte.gz',
 23         'train_label':'train-labels-idx1-ubyte.gz',
 24         'test_img':'t10k-images-idx3-ubyte.gz',
 25         'test_label':'t10k-labels-idx1-ubyte.gz'
 26         }
 27 
 28 dataset_dir=os.path.dirname(os.path.abspath(__file__))
 29 save_file=dataset_dir + "/mnist.pkl"
 30 
 31 train_num = 60000;
 32 test_num  = 10000;
 33 img_dim   = (1, 28, 28)
 34 img_size  = 28*28;
 35 
 36 
 37 def _download(file_name):
 38     """
 39     :param file_name: 下载mnist的文件
 40     :return: null
 41     """
 42     file_path = os.path.join(dataset_dir, file_name)
 43 
 44     if os.path.exists(file_path):
 45         return
 46 
 47     print("downloading"+file_name+ "...")
 48     urllib.request.urlretrieve(url_base + file_name , file_path)
 49     print("Done.")
 50 
 51 def download_mnist():
 52     """
 53 
 54     :return:
 55     """
 56     for file_name in key_file.values():
 57         _download(file_name);
 58 
 59 def _load_label(file_name):
 60     """
 61     解析标签
 62     :param file_name:
 63     :return:
 64     """
 65     file_path = dataset_dir+'/'+ file_name
 66 
 67     print("converting "+file_name+" to numpy Array.")
 68     with gzip.open(file_path) as f:
 69         labels = np.frombuffer(f.read(), np.uint8, offset=8)
 70     print("Done")
 71 
 72     return labels
 73 
 74 def _load_img(file_name):
 75     """
 76     解析 压缩的图片
 77     :param file_name:
 78     :return:
 79     """
 80     file_path = dataset_dir +'/' + file_name
 81 
 82     print("converting "+ file_name + "to numpy Array")
 83     with gzip.open(file_path) as f:
 84         data = np.frombuffer(f.read(), np.uint8, offset=16) # 16*8=
 85     data = data.reshape(-1, img_size) # N, (W*H*C)=[N,28*28*1]
 86     print("Done")
 87 
 88     return data
 89 
 90 def _convert_numpy():
 91     """
 92      解析 image和label，将其转换为numpy
 93     """
 94     dataset = {}
 95     dataset['train_img'] = _load_img(key_file['train_img'])
 96     dataset['train_label'] = _load_label(key_file['train_label'])
 97     dataset['test_img'] = _load_img(key_file['test_img'])
 98     dataset['test_label'] = _load_label(key_file['test_label'])
 99 
100     return dataset
101 
102 def init_mnist():
103     """
104     初始化mnist数据集：
105     1. 下载mnist，
106     2. 以二进制的方式读取，并转换成numpy的ndarray对象
107     3. 将转换后的ndarray 序列化
108 
109     :return:
110     """
111     print("download mnist dataset...")
112     download_mnist()
113     print("convert to numpy array...")
114     dataset = _convert_numpy()
115     print("creating pickle file ...")
116     with open(save_file, 'wb') as f:
117         pickle.dump(dataset, f, -1)
118     print("Done!")
119 
120 def _change_one_hot_label(Y):
121     T = np.zeros((Y.size,10))
122     for idx,row in enumerate(T):
123         row[Y[idx]] = 1
124     return T
125 
126 def load_mnist(normalize=True, flatten=True, one_hot_label=False):
127     """
128 
129     :param normalize: 将数据标准化到0.0~1.0
130     :param flatten: 是否要将数据拉伸层1D数组的形式
131     :param one_hot_label:
132     :return: (训练数据, 训练标签), (测试数据, 测试label)
133     """
134 
135 
136     if not os.path.exists(save_file):
137         init_mnist()
138 
139     with open(save_file,'rb') as f:
140         dataset = pickle.load(f)
141 
142     if normalize:
143         for key in ('train_img','test_img'):
144             dataset[key] = dataset[key].astype(np.float32)
145             dataset[key] /=255.0
146     if one_hot_label:
147         dataset['train_label'] = _change_one_hot_label(dataset['train_label'])
148         dataset['test_label']  = _change_one_hot_label(dataset['test_label'])
149 
150     if not flatten:
151         for key in ('train_img', 'test_img'):
152             dataset[key] = dataset[key].reshape(-1,1,28,28) # NCHW
153 
154     return (dataset['train_img'],dataset['train_label']),(dataset['test_img'], dataset['test_label'])
155 
156 if __name__ == '__main__':
157     init_mnist()
158 

测试

  1 # -*- coding: utf-8 -*-
  2 # @File  : show_mnist.py
  3 # @Author: lizhen
  4 # @Date  : 2020/1/27
  5 # @Desc  : 显示图片
  6 
  7 from src.datasets.mnist import load_mnist
  8 
  9 from skimage import io
 10 
 11 
 12 def img_show(data):
 13     # pil_img = Image.fromarray(np.uint8(data))
 14     io.imshow(data)
 15     io.show()
 16 
 17 
 18 (x_train, t_train), (x_test, t_test) = load_mnist(flatten=True, normalize=False)
 19 img = x_train[0]
 20 label = t_train[0]
 21 print(label)
 22 
 23 print(img.shape)
 24 img = img.reshape(28,28)
 25 print(img.shape)
 26 
 27 img_show(img)
 28 

2020年2月11日 修改