pytorch 读数据接口 制作数据集 data.dataset

【吐槽】

啊，代码，你这个大猪蹄子

自己写了cifar10的数据接口，跟官方接口load的数据一样，

沾沾自喜，以为自己会写数据接口了

几天之后，突然想，自己的代码为啥有点慢呢，这数据集不大啊

用了官方接口，真快啊。。。

啊啊啊啊啊啊啊啊

但这是好事，至少我明白了一点知识对吧

【lesson】

看了cifar10的接口，发现自己在数据集初始化的地方写的太少了，应该在初始化的时候就把所有数据读进来，这样的话在__getitem__的时候才能快。

人家的初始化：

 if self.train:
            self.train_data = []
            self.train_labels = []
            for fentry in self.train_list:
                f = fentry[0]
                file = os.path.join(self.root, self.base_folder, f)
                fo = open(file, 'rb')
                if sys.version_info[0] == 2:
                    entry = pickle.load(fo)
                else:
                    entry = pickle.load(fo, encoding='latin1')
                self.train_data.append(entry['data'])
                if 'labels' in entry:
                    self.train_labels += entry['labels']
                else:
                    self.train_labels += entry['fine_labels']
                fo.close()

            self.train_data = np.concatenate(self.train_data)
            self.train_data = self.train_data.reshape((50000, 3, 32, 32))
            self.train_data = self.train_data.transpose((0, 2, 3, 1))  # convert to HWC

人家的getitem

    def __getitem__(self, index):
        """
        Args:
            index (int): Index

        Returns:
            tuple: (image, target) where target is index of the target class.
        """
        if self.train:
            img, target = self.train_data[index], self.train_labels[index]
        else:
            img, target = self.test_data[index], self.test_labels[index]

        # doing this so that it is consistent with all other datasets
        # to return a PIL Image
        img = Image.fromarray(img)

        if self.transform is not None:
            img = self.transform(img)

        if self.target_transform is not None:
            target = self.target_transform(target)

        return img, target

自己：（都写到getitem里面了）

 def __init__(self, root, transforms=transform(), train=True, test=False):
        self.root = root
        self.transform = transforms
        self.train = train
        self.test = test
        if self.test:
            self.train = False

    def __getitem__(self, item):
        x = math.floor(item / 10000) + 1
        y = item % 10000
        if not self.train and not self.test:
            x = 5
            y = 5000+item

        imgpath = os.path.join(self.root, "data_batch_"+str(x))
        with open(imgpath, 'rb') as fo:
            dict = pickle.load(fo, encoding='bytes')
            d_decode = {}
            for k,v in dict.items():
                d_decode[k.decode('utf8')] = v
            dict = d_decode
            data = dict['data'][y]  # 3*32*32==3072
            data = np.reshape(data,(3,32,32))
            data = data.transpose(1,2,0)
            data = self.transform(data)
            label = dict['labels'][y]
            # label = torch.from_numpy(label)

            return data, label

附自己的代码和人家的代码全部

人家：

 base_folder = 'cifar-10-batches-py'
     url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
     filename = "cifar-10-python.tar.gz"
     tgz_md5 = 'c58f30108f718f92721af3b95e74349a'
     train_list = [
         ['data_batch_1', 'c99cafc152244af753f735de768cd75f'],
         ['data_batch_2', 'd4bba439e000b95fd0a9bffe97cbabec'],
         ['data_batch_3', '54ebc095f3ab1f0389bbae665268c751'],
         ['data_batch_4', '634d18415352ddfa80567beed471001a'],
         ['data_batch_5', '482c414d41f54cd18b22e5b47cb7c3cb'],
     ]

     test_list = [
         ['test_batch', '40351d587109b95175f43aff81a1287e'],
     ]

     def __init__(self, root, train=True,
                  transform=None, target_transform=None,
                  download=False):
         self.root = os.path.expanduser(root)
         self.transform = transform
         self.target_transform = target_transform
         self.train = train  # training set or test set

         if download:
             self.download()

         if not self._check_integrity():
             raise RuntimeError('Dataset not found or corrupted.' +
                                ' You can use download=True to download it')

         # now load the picked numpy arrays
         if self.train:
             self.train_data = []
             self.train_labels = []
             for fentry in self.train_list:
                 f = fentry[0]
                 file = os.path.join(self.root, self.base_folder, f)
                 fo = open(file, 'rb')
                 if sys.version_info[0] == 2:
                     entry = pickle.load(fo)
                 else:
                     entry = pickle.load(fo, encoding='latin1')
                 self.train_data.append(entry['data'])
                 if 'labels' in entry:
                     self.train_labels += entry['labels']
                 else:
                     self.train_labels += entry['fine_labels']
                 fo.close()

             self.train_data = np.concatenate(self.train_data)
             self.train_data = self.train_data.reshape((50000, 3, 32, 32))
             self.train_data = self.train_data.transpose((0, 2, 3, 1))  # convert to HWC
         else:
             f = self.test_list[0][0]
             file = os.path.join(self.root, self.base_folder, f)
             fo = open(file, 'rb')
             if sys.version_info[0] == 2:
                 entry = pickle.load(fo)
             else:
                 entry = pickle.load(fo, encoding='latin1')
             self.test_data = entry['data']
             if 'labels' in entry:
                 self.test_labels = entry['labels']
             else:
                 self.test_labels = entry['fine_labels']
             fo.close()
             self.test_data = self.test_data.reshape((10000, 3, 32, 32))
             self.test_data = self.test_data.transpose((0, 2, 3, 1))  # convert to HWC

     def __getitem__(self, index):
         """
         Args:
             index (int): Index

         Returns:
             tuple: (image, target) where target is index of the target class.
         """
         if self.train:
             img, target = self.train_data[index], self.train_labels[index]
         else:
             img, target = self.test_data[index], self.test_labels[index]

         # doing this so that it is consistent with all other datasets
         # to return a PIL Image
         img = Image.fromarray(img)

         if self.transform is not None:
             img = self.transform(img)

         if self.target_transform is not None:
             target = self.target_transform(target)

         return img, target