%matplotlib inline

import numpy as np
import matplotlib.pyplot as plt
import time
from IPython import display
from IPython.display import Latex
from pylab import axis

plt.rcParams['figure.figsize'] = (9, 6)

num_samples = 40 # datapoints

mean = [0,0]
cov = [[1,5],
       [0,1]]
X, T = np.random.multivariate_normal(mean,cov,num_samples).T

plt.scatter(X, T, color='red', label='samples')
plt.ylabel('target')
plt.xlabel('x')
plt.legend(loc='upper left')
axis('equal')

w = 1

Y = X*w

plt.scatter(X, T, color='red', label='samples')
plt.scatter(X, Y, color='blue', label='prediction')
plt.ylabel('y')
plt.xlabel('x')
plt.legend(loc='upper left')
axis('equal')

Latex(r""" 
        \begin{equation}
            MSE = \frac{1}{n} \sum_{i=1}^n (Y_i - T_i)^2
        \end{equation}
      """)

def mean_squared_error(y,t):
    return np.sum((y-t)**2)/np.size(y)

def loss_function(y,t):
    return mean_squared_error(y,t)

num_parametrizations = 100
w_values = np.linspace(-5,5,num_parametrizations)

error = np.zeros(num_parametrizations)
for i, w in enumerate(w_values):
    Y = X*w
    error[i] = loss_function(Y,T)

optimum = np.argmin(error)
w = w_values[optimum]
error_value = error[optimum]
plt.annotate('optimal w = {0:.{1}f}'.format(w,2), 
             xy=(w, error_value),  xycoords='data',
             xytext=(-50, 50), textcoords='offset points',
             arrowprops=dict(arrowstyle="->"))
plt.plot(w_values, error, color='red')
plt.ylabel('error')
plt.xlabel('w')

num_minibatches = 4
size_minibatches = num_samples/num_minibatches

mbatch_error = np.zeros((num_minibatches,num_parametrizations))

for i, w in enumerate(w_values):
    Y = X*w
    for j in range(num_minibatches):
        mbatch_error[j,i] = loss_function(Y[j*size_minibatches:(j+1)*size_minibatches],
                                       T[j*size_minibatches:(j+1)*size_minibatches])

plt.rcParams['figure.figsize'] = (12, 8)

for j in range(num_minibatches):
    plt.subplot(2, 2, j+1)
    plt.plot(w_values, error, color='red', label='All samples')
    plt.plot(w_values, mbatch_error[j,:], color='blue', label='minibatch {0}'.format(j+1))
    
    optimum = np.argmin(mbatch_error[j,:])
    w = w_values[optimum]
    error_value = mbatch_error[j,optimum]
    plt.annotate('optimal w = {0:.{1}f}'.format(w,2), xy=(w, error_value),  xycoords='data',
                xytext=(-50, 50), textcoords='offset points',
                arrowprops=dict(arrowstyle="->"))
                
    plt.ylabel('error')
    plt.xlabel('w')
    plt.legend(loc='upper right')