#!/usr/bin/env python
# coding: utf-8

# In[1]:


get_ipython().run_line_magic('matplotlib', 'inline')
import numpy as np
import matplotlib.pyplot as plt


# In[2]:


rng = np.random.RandomState(0)
n_samples, n_features = 1000, 2
X = rng.uniform(low=(-10, -10), high=(10, 10), size=(n_samples, 2))
print("data shape: (%d, %d)" % (n_samples, n_features))


# In[3]:


from scipy.stats import multivariate_normal

def target_func(X, noise=0, seed=0):
    rng = np.random.RandomState(seed)
    y = multivariate_normal(mean=(5, 0), cov=5).pdf(X)
    y += multivariate_normal(mean=(-5, 0), cov=5).pdf(X)
    y -= multivariate_normal(mean=(0, 5), cov=3).pdf(X)
    y -= multivariate_normal(mean=(0, -5), cov=3).pdf(X)
    if noise > 0:
        y += rng.normal(scale=noise, size=y.shape)
    return y


y = target_func(X, noise=0.01)
print(y.shape)


# In[4]:


plt.figure(figsize=(10, 10))

def plot_background_2d(func, low=-10, high=10, resolution=100):
    x1 = np.linspace(low, high, resolution)
    x2 = np.linspace(low, high, resolution)
    xx1, xx2 = np.meshgrid(x1, x2)
    zz = func(np.array([xx1.ravel(), xx2.ravel()]).T).reshape(resolution, resolution)
    plt.contourf(xx1, xx2, zz, 30, cmap=plt.cm.RdBu_r)
                     
plot_background_2d(target_func)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolors='none', cmap=plt.cm.RdBu_r);


# In[5]:


plt.figure(figsize=(10, 2))
plt.hist(y);


# In[6]:


from sklearn.gaussian_process import GaussianProcessRegressor

plt.figure(figsize=(10, 10))
gpr = GaussianProcessRegressor(alpha=1).fit(X, y)
plot_background_2d(gpr.predict)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolors='none', cmap=plt.cm.RdBu_r);


# In[7]:


plt.figure(figsize=(10, 10))
gpr = GaussianProcessRegressor(alpha=1e-3).fit(X, y)
plot_background_2d(gpr.predict)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolors='none', cmap=plt.cm.RdBu_r);


# In[8]:


plt.figure(figsize=(10, 10))
gpr = GaussianProcessRegressor(alpha=1e6).fit(X, y)
plot_background_2d(gpr.predict)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolors='none', cmap=plt.cm.RdBu_r);


# In[9]:


from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import learning_curve


# In[10]:


def plot_learning_curve(estimator, X, y, title=None, ylim=None, cv=None,
                        n_jobs=1, train_sizes=np.logspace(-2, 0, 6),
                        scoring=None):
    plt.figure()
    if title is not None:
        plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel("Training examples")
    plt.ylabel("Score")
    plt.xscale("log")
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes,
        scoring=scoring)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="Cross-validation score")

    plt.legend(loc="best")
    print("CV scores:\n%s" % test_scores_mean)
    return plt


# In[11]:


cv = ShuffleSplit(n_iter=200)
ylim = (-1e-3, 0)
scoring = 'mean_squared_error'

plot_learning_curve(GaussianProcessRegressor(alpha=1e-10), X, y, n_jobs=-1,
                    ylim=ylim, scoring=scoring);


# In[12]:


plot_learning_curve(GaussianProcessRegressor(alpha=1e-3), X, y, n_jobs=-1,
                    ylim=ylim, scoring=scoring);


# In[13]:


plot_learning_curve(GaussianProcessRegressor(alpha=1), X, y, n_jobs=-1,
                   ylim=ylim, scoring=scoring);


# In[14]:


plot_learning_curve(GaussianProcessRegressor(alpha=10), X, y, n_jobs=-1,
                   ylim=ylim, scoring=scoring);


# In[15]:


from sklearn.kernel_ridge import KernelRidge

plt.figure(figsize=(10, 10))
kr = KernelRidge(kernel='rbf', gamma=0.1, alpha=1).fit(X, y)
plot_background_2d(kr.predict)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolors='none', cmap=plt.cm.RdBu_r);


# In[16]:


plot_learning_curve(KernelRidge(kernel='rbf', gamma=1e-1, alpha=1), X, y, n_jobs=-1,
                    ylim=ylim, scoring=scoring);


# In[17]:


plot_learning_curve(KernelRidge(kernel='rbf', gamma=1e-1, alpha=1e-2), X, y, n_jobs=-1,
                    ylim=ylim, scoring=scoring);


# In[18]:


plot_learning_curve(KernelRidge(kernel='rbf', gamma=1e-2, alpha=1e-3), X, y, n_jobs=-1,
                    ylim=ylim, scoring=scoring);


# In[27]:


from sklearn.ensemble import ExtraTreesRegressor

plt.figure(figsize=(10, 10))
et = ExtraTreesRegressor(n_estimators=500, n_jobs=-1, max_depth=5).fit(X, y)
plot_background_2d(et.predict)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolors='none', cmap=plt.cm.RdBu_r);


# In[20]:


plot_learning_curve(ExtraTreesRegressor(n_estimators=500, max_depth=8), X, y, n_jobs=-1,
                    ylim=ylim, scoring=scoring);


# In[23]:


from sklearn.neural_network import MLPRegressor

plt.figure(figsize=(10, 10))
mlp = MLPRegressor(hidden_layer_sizes=100, algorithm='l-bfgs', alpha=1e-2, activation='tanh').fit(X, y)
plot_background_2d(mlp.predict)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, edgecolors='none', cmap=plt.cm.RdBu_r);


# In[22]:


mlp = MLPRegressor(hidden_layer_sizes=100, algorithm='l-bfgs', alpha=1e-2, activation='tanh')
plot_learning_curve(mlp, X, y, n_jobs=-1, ylim=ylim, scoring=scoring);


# In[ ]: