#!/usr/bin/env python
# coding: utf-8

# # Backpropagation Neural Network
# 
# [Question](http://stats.stackexchange.com/questions/184483/backpropagation-neural-network-training)
# 
# 

# In[33]:


#!/usr/bin/env python

"""
    This module consits of a Back Propogation Network.

    Author : Goutam - TAGA Labs
    Date : 2015-11-27 18:54:33
"""

import numpy as np          # For matrix and other linear algebra calcuations 
from scipy.special import expit
import sys

def expit_prime(x):
    return expit(x)*(1-expit(x))
def heaviside(x):
    return np.array([0 if xx < 0 else 1 for xx in x])

class BackpropagationNetwork(object):
    """    
    A back Propogation Neural Network.
    """

    def __init__(self,shape):
        """ 
        Constructs  the network of given ``shape``

        It initializes the weights of the network
        """

        self.shape = shape              
        self.weights = list()


        for i in range(1,len(shape)-1):
            # Weights of input & hidden layer
            weight = np.ones((shape[i-1] + 1,shape[i] + 1))
            #weight = np.random.random((shape[i-1] + 1,shape[i] + 1))  # Initializing weights with bias unit
            self.weights.append(weight)
        else:
            # Weights of output layer
            weight = np.ones(((shape[i-1] + 1,shape[i] + 1)))
            #weight = np.random.random((shape[i] + 1,shape[i+1]))      # No bias unit for output layer
            self.weights.append(weight)

    def train(self,input_data_set,output_data_set,epochs=10000):
        """
        Trains the network with given training data set


        Arguements:
        input_data_set -- input training data set
        output_data_set -- output training  data set
        """
        ones = np.atleast_2d(np.ones(input_data_set.shape[0]))
        input_data_set = np.concatenate((ones.T, input_data_set), axis=1)
        for i in range(epochs):

            for inputs, outputs in zip(input_data_set, output_data_set):
                activations = list(self.__feed_forward(inputs))
                deltas = self.__backpropagate(activations,outputs)
                self.__update_weights(deltas,activations)

        self.activations = activations
        self.deltas = deltas

    def __feed_forward(self,inputs):
        # Uses feed forward mechanism to calculate activations
        activation = inputs
        yield activation
        for weight in self.weights:
            activation = heaviside(activation.dot(weight))
            yield activation

    def __backpropagate(self,activations,outputs):
        # Uses back propagation algorithm to calculate deltas

        error = outputs - activations[-1]                           # Error of the last layer
        delta = error * expit_prime(activations[-1])   # Error delta 

        deltas = [delta]                                            # List to store generated deltas

        for (weight,activation) in zip(reversed(self.weights[1:]),reversed(activations[1:-1])):
            delta = delta.dot(weight.T) * expit_prime(activation)
            deltas.append(delta)

        deltas.reverse()
        return deltas

    def __update_weights(self,deltas,activations):
        # Uses gradient descent to update the weights of the network
        for (weight,delta,activation) in zip(self.weights,deltas,activations[:-1]):
            activation = np.atleast_2d(activation)
            delta = np.atleast_2d(delta)
            weight += activation.T.dot(delta)

    def predict(self,inputs):
        """
        Predicts output for given ``inputs``

        Returns list  of outputs 
        """
        inputs = np.concatenate((np.ones(1).T, np.array(inputs)))
        activations = self.__feed_forward(inputs)
        return list(activations)

# ---------------------------------------------------------------------------

np.random.seed(0)

bpn = BackpropagationNetwork((4,1,2,3,2,1))

input_data_set = np.array((
            (0,0),
            (0,1),
            (1,0),
            (1,1)
            ))
output_data_set = np.array((0,1,1,0))
print("untrained\nweights")
print(bpn.weights)
print("return\n",bpn.predict(input_data_set[1]))

bpn.train(input_data_set,output_data_set, epochs=1000)

print("trained\nweights\n",bpn.weights)

print(bpn.predict(input_data_set[1]))