download cifar to datasets dir

This commit is contained in:
2023-03-29 12:25:42 +04:00
parent e5f430d8c6
commit 1240c12ac5
2 changed files with 5 additions and 3 deletions
+1
View File
@@ -17,3 +17,4 @@ recognize*
.idea
disassemblers/applegpu
*.prof
datasets/cifar-10-python.tar.gz
+4 -3
View File
@@ -1,10 +1,9 @@
import os
import io
import numpy as np
import gzip
import tarfile
import pickle
from extra.utils import fetch
from extra.utils import download_file
def fetch_mnist():
parse = lambda file: np.frombuffer(gzip.open(file).read(), dtype=np.uint8).copy()
@@ -17,7 +16,9 @@ def fetch_mnist():
def fetch_cifar(train=True):
cifar10_mean = np.array([0.4913997551666284, 0.48215855929893703, 0.4465309133731618], dtype=np.float32).reshape(1,3,1,1)
cifar10_std = np.array([0.24703225141799082, 0.24348516474564, 0.26158783926049628], dtype=np.float32).reshape(1,3,1,1)
tt = tarfile.open(fileobj=io.BytesIO(fetch('https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz')), mode='r:gz')
fn = os.path.dirname(__file__)+"/cifar-10-python.tar.gz"
download_file('https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz', fn, True)
tt = tarfile.open(fn, mode='r:gz')
if train:
db = [pickle.load(tt.extractfile(f'cifar-10-batches-py/data_batch_{i}'), encoding="bytes") for i in range(1,6)]
else: