initial commit

5 files changed, 197 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..887670a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+env
+.vscode
+data
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..944948f
--- /dev/null
+++ b/README.md
@@ -0,0 +1,8 @@
+## TODO
+* [ ] Print stages
+* [ ] Learning samples method
+* [ ] Save mlp
+* [ ] Choose error
+* [ ] Choose function
+* [ ] CLI
+* [ ] Multithread / Multiprocess
\ No newline at end of file
diff --git a/data.py b/data.py
new file mode 100644
index 0000000..ae41bb7
--- /dev/null
+++ b/data.py
@@ -0,0 +1,18 @@
+import idx2numpy
+import numpy as np
+
+images = list(idx2numpy.convert_from_file('t10k-images-idx3-ubyte')/255)
+labels = idx2numpy.convert_from_file('t10k-labels-idx1-ubyte')
+new_labels = np.zeros((len(labels), 10))
+new_labels[np.arange(len(labels)), labels] = 1
+# new_labels = [np.zeros(10, dtype=int)]*len(labels)
+# for k in range(len(labels)):
+#     new_labels[k][labels[k]] = 1
+train_set = list(zip(images, new_labels))
+# print(train_set[0])
+
+np.save(open('test.npy', 'wb'), train_set)
+
+# arr = np.load(open('train.npy', 'rb'))
+
+# print(arr[0])
\ No newline at end of file
diff --git a/main.py b/main.py
new file mode 100644
index 0000000..845fc6e
--- /dev/null
+++ b/main.py
@@ -0,0 +1,164 @@
+from math import exp
+import numpy as np
+import random
+import matplotlib.pyplot as plt
+import time
+
+def sigmoid(x):
+    return 1 / (1 + exp(-x))
+sigmoid = np.vectorize(sigmoid)
+
+def d_sigmoid(x):
+    return sigmoid(x) * (1 - sigmoid(x))
+d_sigmoid = np.vectorize(d_sigmoid)
+
+learning_rate = 0.001
+
+class Layer():
+    
+    def __init__(self, input_size, output_size, weights=None, bias=None):
+        self.weights = np.random.randn(output_size, input_size)
+        # self.bias = np.random.randn(output_size)
+        self.input = np.zeros(input_size)
+        self.output = np.zeros(output_size)
+        self.inter = np.zeros(output_size)
+        self.grad_output = np.zeros(output_size)
+        self.grad_inter = np.zeros(output_size)
+        self.grad_weights = np.zeros((output_size, input_size))
+        # Bias
+
+    # def __repr__(self):
+    #     return "(Layer {} -> {})".format(len(self.weights), len(self.bias))
+
+    def compute(self, input):
+        self.input = input
+        self.inter = self.weights @ input #+ self.bias
+        self.output = sigmoid(self.inter)
+        return self.output
+
+    def compute_gradient(self, input=None, next_layer=None, expected=None):
+        if expected is not None:
+            self.grad_output = self.output - expected
+        else:
+            self.grad_output = next_layer.weights.T @ next_layer.grad_inter
+        self.grad_inter = d_sigmoid(self.inter) * self.grad_output
+        if input is None:
+            input = self.input
+        self.grad_weights =  np.outer(self.grad_inter, input)
+
+class MultiLayerPerceptron():
+    layers = []
+
+    def __init__(self, layer_sizes):
+        for k in range(len(layer_sizes)-1):
+            self.layers.append(Layer(layer_sizes[k], layer_sizes[k+1]))
+
+    def error(self, output, expected):
+        return 1/2 * np.sum((expected - output)**2)
+
+    def frontprop(self, input) -> np.ndarray:
+        for layer in self.layers:
+            input = layer.compute(input)
+        return input
+
+    def backprop(self, input: np.ndarray, expected):
+        self.frontprop(input)
+        self.layers[-1].compute_gradient(expected=expected)
+        for i in range(len(self.layers)-2, 0, -1):
+            self.layers[i].compute_gradient(next_layer=self.layers[i+1])
+        self.layers[0].compute_gradient(input=input, next_layer=self.layers[1])
+
+    def fit(self):
+        for layer in self.layers:
+            layer.weights -= learning_rate * layer.grad_weights
+
+    def learn(self, space, test_space, iterations=100, test_iterations=100, epochs=100):
+        mean_errors = []
+        mean_accuracy = []
+        begin = time.time()
+        for i in range(epochs):
+            # Print ETA
+            elapsed = time.time() - begin
+            total = int(epochs/i * elapsed) if i > 0 else 0
+            remaining = int(total - elapsed)
+            text = "\r[Learning] {:3.2f}% ~ ert ".format(i*epochs/100)
+            if remaining//3600 > 0:
+                text += "{:2d} h ".format(remaining//3600)
+                remaining = remaining % 3600
+            if remaining//60 > 0:
+                text += "{:2d} m ".format(remaining//60)
+                remaining = remaining % 60
+            text += "{:2d} s".format(remaining)
+            print(text, end="")
+            # Train
+            for j in range(iterations):
+                input, excpected = random.choice(space)
+                self.backprop(input, excpected)
+                self.fit()
+            # Test
+            errors = []
+            success = 0
+            for j in range(test_iterations):
+                input, expected = random.choice(test_space)
+                errors.append(self.error(self.frontprop(input), expected))
+                # Only for classifier
+                if np.argmax(self.frontprop(input)) == np.argmax(expected):
+                    success += 1
+            mean_errors.append(sum(errors, 0)/100)
+            mean_accuracy.append(success/100)
+        return mean_errors, mean_accuracy
+
+            
+
+
+if __name__ == "__main__":
+    # XOR
+    # mlp = MultiLayerPerceptron([2, 4, 4, 1])
+    # space = [
+    #     (np.array([0, 0]), np.array([0])),
+    #     (np.array([0, 1]), np.array([1])),
+    #     (np.array([1, 0]), np.array([1])),
+    #     (np.array([1, 1]), np.array([0])),
+    # ]
+    # mean_errors = []
+    # begin = time.time()
+    # for i in range(100):
+    #     errors = []
+    #     elapsed = time.time() - begin
+    #     remaining = int(100/i * elapsed - elapsed) if i> 0 else 0
+    #     print(i, "{}% ~ {} min {} sec left".format(i, remaining//60, remaining%60), end='\r')
+    #     for j in range(1600):
+    #         input, expected = random.choice(space)
+    #         mlp.backprop(input, expected)
+    #         mlp.fit()
+    #     for input, expected in space:
+    #         errors.append(mlp.error(mlp.frontprop(input), expected))
+    #     mean_errors.append(sum(errors, 0)/4)
+    # for input, expected in space:
+    #     print(mlp.frontprop(input), expected)
+            
+    # plt.plot(mean_errors)
+    # plt.show()
+
+    # MNIST
+    print("[Initializing] Perceptron: ", end="")
+    mlp = MultiLayerPerceptron([28*28, 32, 16, 10])
+    print("OK")
+    print("[Loading] Training set: ", end="")
+    space = np.load(open("data/train.npy", "rb"))
+    space = [(input.reshape(28*28), expected) for input, expected in space]
+    print("OK    Testing set: ", end="")
+    test_space = np.load(open("data/test.npy", "rb"))
+    test_space = [(input.reshape(28*28), expected) for input, expected in test_space]
+    print("OK")
+
+    mean_errors, mean_accuracy = mlp.learn(space, test_space)
+            
+    image, expected = random.choice(space)
+    input = np.reshape(image, 28*28)
+    print(mlp.frontprop(input), expected)
+    print(np.argmax(mlp.frontprop(input)), np.argmax(expected))
+
+    plt.plot(mean_errors)
+    plt.plot(mean_accuracy)
+    plt.show()
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..3953040
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,4 @@
+numpy
+idx2numpy
+matplotlib
+pylint
\ No newline at end of file