如何将notMNIST转成MNIST格式

相信了解机器学习的对MNIST不会陌生，Google的工程师Yaroslav Bulatov 创建了notMNIST，它和MNIST类似，图像28x28，也有10个Label（A-J）。

在Tensorflow中已经封装好了读取MNIST数据集的函数 read_data_sets()，

from tensorflow.contrib.learn.python.learn.datasets.mnist import read_data_sets
mnist = read_data_sets("data", one_hot=True, reshape=False, validation_size=0)

但是由于notMNIST的格式和MNIST的格式不是完全相同，所以基于tensorflow创建的针对MNIST的模型并不能直接读取notMNIST的图片。

Github上有人编写了格式转换代码（https://github.com/davidflanagan/notMNIST-to-MNIST），转换后可直接使用read_data_sets()完成读取，这样模型代码的变动就不会很大。本文是对在阅览完代码后所做的注释。

 import numpy, imageio, glob, sys, os, random
 #Imageio 提供简单的用于读写图像数据的接口
 #glob 功能类似于文件搜索，查找文件只用到三个匹配符：”*”, “?”, “[]”。”*”匹配0个或多个字符；”?”匹配单个字符；”[]”匹配指定范围内的字符，如：[0-9]匹配数字。
 def get_labels_and_files(folder, number):
   # Make a list of lists of files for each label
   filelists = []
   for label in range(0,10):
     filelist = []
     filelists.append(filelist);
     dirname = os.path.join(folder, chr(ord('A') + label))
     #label实际为0-9，chr(ord('A') + label)返回A-J
     #拼接路径dirname=folder/[A-J]
     for file in os.listdir(dirname):
     #返回一个装满当前路径中文件名的list
       if (file.endswith('.png')):
         fullname = os.path.join(dirname, file)
         if (os.path.getsize(fullname) > 0):
           filelist.append(fullname)
         else:
           print('file ' + fullname + ' is empty')
     # sort each list of files so they start off in the same order
     # regardless of how the order the OS returns them in
     filelist.sort()

   # Take the specified number of items for each label and
   # build them into an array of (label, filename) pairs
   # Since we seeded the RNG, we should get the same sample each run
   labelsAndFiles = []
   for label in range(0,10):
     filelist = random.sample(filelists[label], number)
     #随机采样 设定个数的文件名
     for filename in filelist:
       labelsAndFiles.append((label, filename))
       #Python的元组与列表类似，不同之处在于元组的元素不能修改。元组使用小括号，列表使用方括号。
   return labelsAndFiles

 def make_arrays(labelsAndFiles):
   images = []
   labels = []
   for i in range(0, len(labelsAndFiles)):

     # display progress, since this can take a while
     if (i % 100 == 0):
       sys.stdout.write("\r%d%% complete" % ((i * 100)/len(labelsAndFiles)))
       #\r 返回第一个指针，覆盖前面的内容
       sys.stdout.flush()

     filename = labelsAndFiles[i][1]
     try:
       image = imageio.imread(filename)
       images.append(image)
       labels.append(labelsAndFiles[i][0])
     except:
       # If this happens we won't have the requested number
       print("\nCan't read image file " + filename)

   count = len(images)
   imagedata = numpy.zeros((count,28,28), dtype=numpy.uint8)
   labeldata = numpy.zeros(count, dtype=numpy.uint8)
   for i in range(0, len(labelsAndFiles)):
     imagedata[i] = images[i]
     labeldata[i] = labels[i]
   print("\n")
   return imagedata, labeldata

 def write_labeldata(labeldata, outputfile):
   header = numpy.array([0x0801, len(labeldata)], dtype='>i4')
   with open(outputfile, "wb") as f:
   #以二进制写模式打开
   #这里使用了 with 语句，不管在处理文件过程中是否发生异常，都能保证 with 语句执行完毕后已经关闭了打开的文件句柄
     f.write(header.tobytes())
     #写入二进制数
     f.write(labeldata.tobytes())

 def write_imagedata(imagedata, outputfile):
   header = numpy.array([0x0803, len(imagedata), 28, 28], dtype='>i4')
   with open(outputfile, "wb") as f:
     f.write(header.tobytes())
     f.write(imagedata.tobytes())

 def main(argv):
   # Uncomment the line below if you want to seed the random
   # number generator in the same way I did to produce the
   # specific data files in this repo.
   # random.seed(int("notMNIST", 36))
   #当我们设置相同的seed，每次生成的随机数相同。如果不设置seed，则每次会生成不同的随机数

   labelsAndFiles = get_labels_and_files(argv[1], int(argv[2]))
   #随机排序
   random.shuffle(labelsAndFiles)

   imagedata, labeldata = make_arrays(labelsAndFiles)
   write_labeldata(labeldata, argv[3])
   write_imagedata(imagedata, argv[4])

 if __name__=='__main__':
 #Make a script both importable and executable
 #如果我们是直接执行某个.py文件的时候，该文件中那么”__name__ == '__main__'“是True
 #如果被别的模块import，__name__！='__main__'，这样main()就不会执行

   main(sys.argv)

使用方法

下载解压notMNIST：

curl -o notMNIST_small.tar.gz http://yaroslavvb.com/upload/notMNIST/notMNIST_small.tar.gz
curl -o notMNIST_large.tar.gz http://yaroslavvb.com/upload/notMNIST/notMNIST_large.tar.gz
tar xzf notMNIST_small.tar.gz
tar xzf notMNIST_large.tar.gz

运行转换代码：

python convert_to_mnist_format.py notMNIST_small  data/t10k-labels-idx1-ubyte data/t10k-images-idx3-ubyte
python convert_to_mnist_format.py notMNIST_large  data/train-labels-idx1-ubyte data/train-images-idx3-ubyte
gzip data/*ubyte