move

1 files changed, 196 insertions, 0 deletions

diff --git a/MultiLayerPerceptron/main.py b/MultiLayerPerceptron/main.py
new file mode 100644
index 0000000..2dbfa3d
--- /dev/null
+++ b/MultiLayerPerceptron/main.py
@@ -0,0 +1,196 @@
+from math import exp
+import numpy as np
+import random
+import matplotlib.pyplot as plt
+import time
+from timeit import timeit
+
+def sigmoid(x):
+    return 1 / (1 + exp(-x))
+sigmoid = np.vectorize(sigmoid)
+
+def d_sigmoid(x):
+    return sigmoid(x) * (1 - sigmoid(x))
+d_sigmoid = np.vectorize(d_sigmoid)
+
+learning_rate = 0.001
+verbose = False
+
+class Layer():
+    
+    def __init__(self, input_size, output_size, weights=None, bias=None):
+        self.weights = np.random.randn(output_size, input_size)
+        # self.bias = np.random.randn(output_size)
+        self.input = np.zeros(input_size)
+        self.output = np.zeros(output_size)
+        self.inter = np.zeros(output_size)
+        self.grad_output = np.zeros(output_size)
+        self.grad_inter = np.zeros(output_size)
+        self.grad_weights = np.zeros((output_size, input_size))
+        # Bias
+
+    def __repr__(self):
+        return "<Layer {} -> {}>".format(self.weights.shape[1], self.weights.shape[0])
+
+    def compute(self, input):
+        self.input = input
+        self.inter = self.weights @ input #+ self.bias
+        self.output = sigmoid(self.inter)
+        return self.output
+
+    def compute_gradient(self, input=None, next_layer=None, expected=None):
+        if expected is not None:
+            self.grad_output = self.output - expected
+        else:
+            self.grad_output = next_layer.weights.T @ next_layer.grad_inter
+        self.grad_inter = d_sigmoid(self.inter) * self.grad_output
+        if input is None:
+            input = self.input
+        self.grad_weights =  np.outer(self.grad_inter, input)
+
+class MultiLayerPerceptron():
+    layers = []
+
+    def __init__(self, layer_sizes):
+        for k in range(len(layer_sizes)-1):
+            self.layers.append(Layer(layer_sizes[k], layer_sizes[k+1]))
+
+    def __repr__(self):
+        return "<MultiLayerPerceptron {}>".format(self.layers)
+
+    @staticmethod
+    def load(filename):
+        array = np.load(open(filename, 'rb'))
+        sizes = [array[0].shape[1]]
+        for layer in array:
+            sizes.append(layer.shape[0])
+        mlp = MultiLayerPerceptron(sizes)
+        for k, layer in enumerate(array):
+            mlp.layers[k].weights = layer
+        return mlp
+
+    def save(self, filename):
+        array = [layer.weights for layer in self.layers]
+        np.save(open(filename, 'wb'), array)
+
+    def error(self, output, expected):
+        return 1/2 * np.sum((expected - output)**2)
+
+    def frontprop(self, input) -> np.ndarray:
+        for layer in self.layers:
+            input = layer.compute(input)
+        return input
+
+    def backprop(self, input: np.ndarray, expected):
+        self.frontprop(input)
+        self.layers[-1].compute_gradient(expected=expected)
+        for i in range(len(self.layers)-2, 0, -1):
+            self.layers[i].compute_gradient(next_layer=self.layers[i+1])
+        self.layers[0].compute_gradient(input=input, next_layer=self.layers[1])
+
+    def fit(self):
+        for layer in self.layers:
+            layer.weights -= learning_rate * layer.grad_weights
+
+    def test(self, test_space, test_iterations=100):
+        errors = []
+        success = 0
+        for j in range(test_iterations):
+            input, expected = random.choice(test_space)
+            output = self.frontprop(input)
+            errors.append(self.error(output, expected))
+            if np.argmax(output) == np.argmax(expected):
+                success += 1
+        return sum(errors)/100, success/100
+
+    def learn(self, space, test_space, iterations=100, test_iterations=100, epochs=100, save=None):
+        mean_errors = []
+        mean_accuracy = []
+        begin = time.time()
+        for i in range(epochs):
+            if save: self.save(save)
+            # Print ETA
+            elapsed = time.time() - begin
+            total = int(epochs/i * elapsed) if i > 0 else 0
+            remaining = int(total - elapsed)
+            text = "\r[Learning] {:3.2f}% ~ ert ".format(i*100/epochs)
+            # if remaining//3600 > 0:
+            text += "{:2d} h ".format(remaining//3600)
+            remaining = remaining % 3600
+            # if remaining//60 > 0:
+            text += "{:2d} m ".format(remaining//60)
+            remaining = remaining % 60
+            text += "{:2d} s".format(remaining)
+            if verbose: print(text, end="")
+            # Train
+            for j in range(iterations):
+                input, expected = random.choice(space)
+                self.backprop(input, expected)
+                self.fit()
+            # Test
+            error, accuracy = self.test(test_space, test_iterations)
+            mean_errors.append(error)
+            mean_accuracy.append(accuracy)
+        return mean_errors, mean_accuracy
+
+            
+
+
+if __name__ == "__main__":
+    # XOR
+    # mlp = MultiLayerPerceptron([2, 4, 4, 1])
+    # space = [
+    #     (np.array([0, 0]), np.array([0])),
+    #     (np.array([0, 1]), np.array([1])),
+    #     (np.array([1, 0]), np.array([1])),
+    #     (np.array([1, 1]), np.array([0])),
+    # ]
+    # mean_errors = []
+    # begin = time.time()
+    # for i in range(100):
+    #     errors = []
+    #     elapsed = time.time() - begin
+    #     remaining = int(100/i * elapsed - elapsed) if i> 0 else 0
+    #     print(i, "{}% ~ {} min {} sec left".format(i, remaining//60, remaining%60), end='\r')
+    #     for j in range(1600):
+    #         input, expected = random.choice(space)
+    #         mlp.backprop(input, expected)
+    #         mlp.fit()
+    #     for input, expected in space:
+    #         errors.append(mlp.error(mlp.frontprop(input), expected))
+    #     mean_errors.append(sum(errors, 0)/4)
+    # for input, expected in space:
+    #     print(mlp.frontprop(input), expected)
+            
+    # plt.plot(mean_errors)
+    # plt.show()
+
+    # MNIST
+    print("[Initializing] Perceptron: ", end="")
+    # mlp = MultiLayerPerceptron([28*28, 32, 16, 10])
+    mlp = MultiLayerPerceptron.load('mnist.npy')
+    print("OK")
+    print("[Loading] Training set: ", end="")
+    space = np.load(open("data/train.npy", "rb"))
+    space = [(input.reshape(28*28), expected) for input, expected in space]
+    print("OK    Testing set: ", end="")
+    test_space = np.load(open("data/test.npy", "rb"))
+    test_space = [(input.reshape(28*28), expected) for input, expected in test_space]
+    print("OK")
+
+    mean_errors, mean_accuracy = mlp.learn(space, test_space, iterations=1000, epochs=100, save='mnist.npy')
+            
+    image, expected = random.choice(space)
+    input = np.reshape(image, 28*28)
+    print(mlp.frontprop(input), expected)
+    print(np.argmax(mlp.frontprop(input)), "=>", np.argmax(expected))
+
+    plt.figure(1)
+    plt.subplot(211)
+    plt.plot(mean_errors)
+    plt.ylabel("error")
+    plt.subplot(212)
+    plt.plot(mean_accuracy)
+    plt.ylabel("accuracy")
+    plt.axis([0, len(mean_accuracy), 0, 1])
+    plt.show()
\ No newline at end of file