perceptron

git clone git://git.codymlewis.com/perceptron.git
Log | Files | Refs | README

Perceptron.py (7229B)


      1 #!/usr/bin/env python3
      2 
      3 
      4 import argparse
      5 
      6 import numpy as np
      7 import pandas as pd
      8 
      9 
     10 '''
     11 An implementation of a multi-layer perceptron.
     12 
     13 Author: Cody Lewis
     14 Date: 2019-09-21
     15 '''
     16 
     17 
     18 class Neuron:
     19     '''A neuron of the perceptron'''
     20     def __init__(self, input_layer=False):
     21         self.activation_strength = 0
     22         self.connections = []
     23         self.is_input_layer = input_layer
     24 
     25     def connect(self, other):
     26         '''Connect a neuron to this'''
     27         self.connections.append(Connection(self, other))
     28 
     29     def input(self, value):
     30         '''Input a value into this neuron'''
     31         self.activation_strength = value
     32 
     33     def output(self):
     34         '''Recieve a final output from this neuron with softmax activation'''
     35         val = np.exp(self.activation_strength)
     36         self.activation_strength = 0
     37         return val
     38 
     39     def mutate(self, step_size):
     40         '''Mutate the weights of the connections on this neuron'''
     41         for connection in self.connections:
     42             connection.mutate_weight(step_size)
     43 
     44     def revert(self):
     45         '''Reset the weights of the connections on this neuron'''
     46         for connection in self.connections:
     47             connection.revert_weight()
     48 
     49     def activate(self):
     50         '''Activate this neuron'''
     51         for connection in self.connections:
     52             if self.is_input_layer:
     53                 connection.activate(self.activation_strength)
     54             else:
     55                 connection.activate(activate(self.activation_strength))
     56         self.activation_strength = 0
     57 
     58     def activated(self, value):
     59         '''Get activated with the value'''
     60         self.activation_strength += value
     61 
     62 
     63 class Connection:
     64     '''A connection between 2 neurons'''
     65     def __init__(self, from_node, to_node):
     66         self.from_node = from_node
     67         self.to_node = to_node
     68         self.weight = np.random.normal()
     69         self.cached_weight = 0
     70 
     71     def activate(self, value):
     72         '''Activate the neuron that this connects to with the value'''
     73         self.to_node.activated(value * self.weight)
     74 
     75     def mutate_weight(self, step_size):
     76         '''Mutate the weight of this'''
     77         self.cached_weight = self.weight
     78         self.weight += step_size * np.random.normal()
     79 
     80     def revert_weight(self):
     81         '''Reset the weight mutation'''
     82         self.weight = self.cached_weight
     83 
     84 
     85 class Perceptron:
     86     '''A multi layer perceptron implementation'''
     87     def __init__(self, structure):
     88         self.layers = [
     89             [Neuron(i == 0) for _ in range(layer_size)]
     90             for i, layer_size in enumerate(structure)
     91         ]
     92 
     93         for i in range(len(self.layers) - 1):
     94             for neuron in self.layers[i]:
     95                 for next_neuron in self.layers[i + 1]:
     96                     neuron.connect(next_neuron)
     97 
     98     def summary(self):
     99         print(f"{'-' * 5} [Perceptron Summary] {'-' * 5}")
    100         num_layers = len(self.layers)
    101         for i, layer in enumerate(self.layers):
    102             if i == 0:
    103                 layer_type = "Input Layer"
    104             elif i == num_layers - 1:
    105                 layer_type = "Output Layer"
    106             else:
    107                 layer_type = f"Hidden Layer {i}"
    108             print(f"{layer_type}:\t{len(layer)}")
    109         print("-" * 32)
    110 
    111     def predict(self, inputs):
    112         '''Predict a value given the inputs'''
    113         for i, inp in enumerate(inputs):
    114             self.layers[0][i + 1].input(inp)
    115         # Set up bias
    116         self.layers[0][0].input(1)
    117 
    118         for layer in self.layers[:-1]:
    119             for neuron in layer:
    120                 neuron.activate()
    121 
    122         outputs = []
    123         for neuron in self.layers[-1]:
    124             outputs.append(neuron.output())
    125         denominator = np.sum(outputs)
    126         activations = []
    127         for output in outputs:
    128             activations.append(output / denominator)
    129 
    130         return activations
    131 
    132     def classify(self, inputs):
    133         '''
    134         Find the classification given a prediction for the prediction of the
    135         inputs
    136         '''
    137         return np.argmax(self.predict(inputs))
    138 
    139     def learn(self, error_goal, inputs, target_responses, verbose=False):
    140         '''Learn the weights for the network using evolutionary hill descent'''
    141         counter = 0
    142         n_epochs = 10_000
    143         error_champ = find_error(inputs, target_responses, self)
    144         errors = [error_champ]
    145         while(error_goal < error_champ) and (counter < n_epochs):
    146             if verbose:
    147                 print(f"\rEpoch {counter + 1}, error: {error_champ}", end="")
    148             step_size = 0.02 * np.random.normal()
    149             for layer in self.layers[:-1]:
    150                 for neuron in layer:
    151                     neuron.mutate(step_size)
    152 
    153             error_mutant = find_error(inputs, target_responses, self)
    154 
    155             if error_mutant < error_champ:
    156                 error_champ = error_mutant
    157             else:
    158                 for layer in self.layers[:-1]:
    159                     for neuron in layer:
    160                         neuron.revert()
    161             counter += 1
    162             errors.append(error_champ)
    163         if verbose:
    164             print()
    165         return error_champ, errors
    166 
    167 
    168 def activate(x_value, coefficient=1, constant=0):
    169     '''
    170     Perform the sigmoid function to determine whether a neuron is activated.
    171     '''
    172     return 1 / (1 + np.exp(-coefficient * x_value - constant))
    173 
    174 
    175 def find_error(inputs, target_responses, perceptron):
    176     '''
    177     Find the error of the perceptron.
    178     '''
    179     error = 0
    180     for i, target_response in zip(inputs, target_responses):
    181         predictions = perceptron.predict(i)
    182         error += log_loss(1, predictions[target_response])
    183     return -error / len(inputs)
    184 
    185 
    186 def log_loss(label, prediction, eps=1e-15):
    187     '''Find the log loss'''
    188     # clip performs a minmax
    189     np.clip(prediction, eps, 1 - eps)
    190     return label * np.log2(prediction)
    191 
    192 
    193 def eval_predictions(model, data, labels):
    194     '''Evaluate the accuracy of predictions on the data made by the model'''
    195     print("Input data\tPrediction\tTrue Label")
    196     accuracy = 0
    197     for row, label in zip(data, labels):
    198         prediction = model.classify(row)
    199         print(f"{row}\t\t{prediction}\t\t{label}")
    200         accuracy += prediction == label
    201     print(f"Accuracy: {100 * accuracy / len(labels)}%")
    202     print()
    203 
    204 
    205 if __name__ == "__main__":
    206     PARSER = argparse.ArgumentParser(description="A multi-layer perceptron")
    207     PARSER.add_argument("-f", "--file", dest="file", type=str, action="store",
    208                         default="XOR2.csv",
    209                         help="csv file cotaining the training data and labels")
    210     PARSER.add_argument("-l", "--layers", dest="layers", type=int,
    211                         action="store", default=[4], nargs="+",
    212                         help="Number of neurons in the hidden layers")
    213     ARGS = PARSER.parse_args()
    214     DF = pd.read_csv(ARGS.file, header=None)
    215     DATA = DF[DF.columns[:-1]].to_numpy()
    216     LABELS = DF[DF.columns[-1]].to_numpy()
    217     MODEL = Perceptron(
    218         [np.shape(DATA)[1] + 1] + ARGS.layers + [len(np.unique(LABELS))]
    219     )
    220     print(f"Constructed Perceptron:")
    221     MODEL.summary()
    222     print()
    223     eval_predictions(MODEL, DATA, LABELS)
    224     MODEL.learn(0.01, DATA, LABELS, True)
    225     print()
    226     eval_predictions(MODEL, DATA, LABELS)