diff options
Diffstat (limited to 'MultiLayerPerceptron')
| -rw-r--r-- | MultiLayerPerceptron/__init__.py | 2 | ||||
| -rw-r--r-- | MultiLayerPerceptron/download.py | 109 | ||||
| -rw-r--r-- | MultiLayerPerceptron/main.py | 17 |
3 files changed, 120 insertions, 8 deletions
diff --git a/MultiLayerPerceptron/__init__.py b/MultiLayerPerceptron/__init__.py index 0b2f4ea..63c9c50 100644 --- a/MultiLayerPerceptron/__init__.py +++ b/MultiLayerPerceptron/__init__.py @@ -1 +1 @@ -name="multilayer-perceptron"
\ No newline at end of file +name="multilayer-perceptron" diff --git a/MultiLayerPerceptron/download.py b/MultiLayerPerceptron/download.py new file mode 100644 index 0000000..6a0e23a --- /dev/null +++ b/MultiLayerPerceptron/download.py @@ -0,0 +1,109 @@ +"""Download MNIST and prepare .npy files expected by main.py.""" + +import argparse +import gzip +import shutil +import urllib.request +from pathlib import Path + +import idx2numpy +import numpy as np + +BASE_URL = "https://storage.googleapis.com/cvdf-datasets/mnist" +FILES = { + "train-images-idx3-ubyte.gz": f"{BASE_URL}/train-images-idx3-ubyte.gz", + "train-labels-idx1-ubyte.gz": f"{BASE_URL}/train-labels-idx1-ubyte.gz", + "t10k-images-idx3-ubyte.gz": f"{BASE_URL}/t10k-images-idx3-ubyte.gz", + "t10k-labels-idx1-ubyte.gz": f"{BASE_URL}/t10k-labels-idx1-ubyte.gz", +} + + +def download(url: str, dest: Path) -> None: + if dest.exists(): + print(f"[skip] {dest} already exists") + return + dest.parent.mkdir(parents=True, exist_ok=True) + print(f"[download] {url} -> {dest}", end="", flush=True) + urllib.request.urlretrieve(url, dest) + print(" OK") + + +def gunzip(src: Path, dst: Path) -> None: + if dst.exists(): + print(f"[skip] {dst} already exists") + return + print(f"[gunzip] {src} -> {dst}", end="", flush=True) + with gzip.open(src, "rb") as f_in, open(dst, "wb") as f_out: + shutil.copyfileobj(f_in, f_out) + print(" OK") + + +def convert_dataset(images_path: Path, labels_path: Path, out_path: Path) -> None: + print(f"[convert] {images_path.name} + {labels_path.name} -> {out_path}") + images = idx2numpy.convert_from_file(str(images_path)) / 255.0 + labels = idx2numpy.convert_from_file(str(labels_path)) + new_labels = np.zeros((len(labels), 10)) + new_labels[np.arange(len(labels)), labels] = 1 + dataset = list(zip(images, new_labels, strict=False)) + out_path.parent.mkdir(parents=True, exist_ok=True) + with open(out_path, "wb") as f: + np.save(f, np.array(dataset, dtype=object)) + print(f" saved {len(dataset)} samples to {out_path}") + + +def ensure_mnist_weights(path: Path, layer_sizes=(784, 32, 16, 10)) -> None: + if path.exists(): + print(f"[skip] {path} already exists") + return + print(f"[init] creating {path} with layers {layer_sizes}") + from MultiLayerPerceptron.main import MultiLayerPerceptron + + mlp = MultiLayerPerceptron(list(layer_sizes)) + mlp.save(str(path)) + print(" OK") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Download MNIST and prepare npy files") + parser.add_argument("--raw-dir", default="data/raw", help="directory for raw idx files") + parser.add_argument("--force", action="store_true", help="re-download even if exists") + args = parser.parse_args() + + raw_dir = Path(args.raw_dir) + if args.force: + for name in FILES: + p = raw_dir / name + if p.exists(): + p.unlink() + ungz = raw_dir / name.replace(".gz", "") + if ungz.exists(): + ungz.unlink() + + # 1. download + for name, url in FILES.items(): + download(url, raw_dir / name) + + # 2. gunzip + for name in FILES: + gunzip(raw_dir / name, raw_dir / name.replace(".gz", "")) + + # 3. convert to npy expected by main.py + convert_dataset( + raw_dir / "train-images-idx3-ubyte", + raw_dir / "train-labels-idx1-ubyte", + Path("data/train.npy"), + ) + convert_dataset( + raw_dir / "t10k-images-idx3-ubyte", + raw_dir / "t10k-labels-idx1-ubyte", + Path("data/test.npy"), + ) + + # 4. ensure initial weights + ensure_mnist_weights(Path("mnist.npy")) + + print("[done] all files ready") + + +if __name__ == "__main__": + main() diff --git a/MultiLayerPerceptron/main.py b/MultiLayerPerceptron/main.py index 2dbfa3d..d7a3138 100644 --- a/MultiLayerPerceptron/main.py +++ b/MultiLayerPerceptron/main.py @@ -49,9 +49,8 @@ class Layer(): self.grad_weights = np.outer(self.grad_inter, input) class MultiLayerPerceptron(): - layers = [] - def __init__(self, layer_sizes): + self.layers = [] for k in range(len(layer_sizes)-1): self.layers.append(Layer(layer_sizes[k], layer_sizes[k+1])) @@ -60,7 +59,8 @@ class MultiLayerPerceptron(): @staticmethod def load(filename): - array = np.load(open(filename, 'rb')) + with open(filename, 'rb') as f: + array = np.load(f, allow_pickle=True) sizes = [array[0].shape[1]] for layer in array: sizes.append(layer.shape[0]) @@ -70,8 +70,9 @@ class MultiLayerPerceptron(): return mlp def save(self, filename): - array = [layer.weights for layer in self.layers] - np.save(open(filename, 'wb'), array) + array = np.array([layer.weights for layer in self.layers], dtype=object) + with open(filename, 'wb') as f: + np.save(f, array, allow_pickle=True) def error(self, output, expected): return 1/2 * np.sum((expected - output)**2) @@ -171,10 +172,12 @@ if __name__ == "__main__": mlp = MultiLayerPerceptron.load('mnist.npy') print("OK") print("[Loading] Training set: ", end="") - space = np.load(open("data/train.npy", "rb")) + with open("data/train.npy", "rb") as f: + space = np.load(f, allow_pickle=True) space = [(input.reshape(28*28), expected) for input, expected in space] print("OK Testing set: ", end="") - test_space = np.load(open("data/test.npy", "rb")) + with open("data/test.npy", "rb") as f: + test_space = np.load(f, allow_pickle=True) test_space = [(input.reshape(28*28), expected) for input, expected in test_space] print("OK") |