7. Dimensionality Reduction with PCA(Principal Component Analysis)¶

import numpy as np
X = [[2, 0, -1.4],
    [2.2, 0.2, -1.5],
    [2.4, 0.1, -1],
    [1.9, 0, -1.2]]
print np.cov(np.array(X).T)

import numpy as np
w, v = np.linalg.eig(np.array([[1, -2], [2, -3]]))
w; v
print 'eigenvalues(w) : \n',w,'\n\neigenvectors(v) :\n', v

%matplotlib inline

import numpy as np
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.datasets import load_iris

# Origin value
x1 = [0.9, 2.4, 1.2, 0.5, 0.3, 1.8, 0.5, 0.3, 2.5, 1.3]
x2 = [1, 2.6, 1.7, 0.7, 0.7, 1.4, 0.6, 0.6, 2.6, 1.1]
c = np.cov(x1, x2)
print "COV\n", c

w, v = np.linalg.eig(np.array(c))
print "\nEigenvalue\n",w, "\n\nEigenVector\n", v

#  Origin value - Mean
x, y = [],[]
x_1, y_1 = [],[]

print v[0][0], v[0][1],v[1][0],v[1][1]
for i in range(10):
    x.append((x1[i]-1.17))
    y.append((x2[i]-1.3))
    x_1.append((x1[i]-1.17)*v[0][0]+(x2[i]-1.3)*v[1][0])
    y_1.append((x1[i]-1.17)*v[0][1]+(x2[i]-1.3)*v[1][1])        

# PCA transform * eigenvalue
z1 = [-0.40200434,1.78596968,0.29427599,-0.89923557,-1.04573848,
   0.52955,-0.96731071,-1.11381362,1.85922113,-0.04092339]
z2 = [0,0,0,0,0,0,0,0,0,0]

plt.scatter(x, y, c='b', marker='x', label='origin-mean')
plt.scatter(x_1, y_1, c='r', marker='x', label='rotate')
plt.scatter(z1, z2, c='g', marker='.', label='reduction with PCA')
plt.legend()

plt.show()

%matplotlib inline
print(__doc__)


# Code source: Gaël Varoquaux
# Modified for documentation by Jaques Grobler
# License: BSD 3 clause

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn import datasets
from sklearn.decomposition import PCA

# import some data to play with
iris = datasets.load_iris()
X = iris.data[:, :2]  # we only take the first two features.
Y = iris.target

x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5

plt.figure(2, figsize=(8, 6))
plt.clf()

# Plot the training points
plt.scatter(X[:, 0], X[:, 1], c=Y, cmap=plt.cm.Paired)
plt.xlabel('Sepal length')
plt.ylabel('Sepal width')

plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)
plt.xticks(())
plt.yticks(())

# To getter a better understanding of interaction of the dimensions
# plot the first three PCA dimensions
fig = plt.figure(1, figsize=(8, 6))
ax = Axes3D(fig, elev=-150, azim=110)
X_reduced = PCA(n_components=3).fit_transform(iris.data)
ax.scatter(X_reduced[:, 0], X_reduced[:, 1], X_reduced[:, 2], c=Y,
           cmap=plt.cm.Paired)
ax.set_title("First three PCA directions")
ax.set_xlabel("1st eigenvector")
ax.w_xaxis.set_ticklabels([])
ax.set_ylabel("2nd eigenvector")
ax.w_yaxis.set_ticklabels([])
ax.set_zlabel("3rd eigenvector")
ax.w_zaxis.set_ticklabels([])

plt.show()

%matplotlib inline 

import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.datasets import load_iris

data = load_iris()
y=data.target
x=data.data
target_names = data.target_names
pca=PCA(n_components=2)
reduced_X=pca.fit_transform(x)

#Mastering Machine Learning with scikit-learn 예제
red_x, red_y = [], []
blue_x, blue_y = [], []
green_x, green_y = [], []
for i in range(len(reduced_X)):
    if y[i] == 0:
        red_x.append(reduced_X[i][0])
        red_y.append(reduced_X[i][1])
    elif y[i] == 1:
        blue_x.append(reduced_X[i][0])
        blue_y.append(reduced_X[i][1])
    else:
        green_x.append(reduced_X[i][0])
        green_y.append(reduced_X[i][1])
plt.figure(1)
plt.scatter(red_x, red_y, c='r', marker='x', label=target_names[0])
plt.scatter(blue_x, blue_y, c='b', marker='D', label=target_names[1])
plt.scatter(green_x, green_y, c='g', marker='.', label=target_names[2])
plt.legend()
plt.title('PCA of IRIS dataset(book example)')

#scikit-learn 예제
plt.figure(2)
for c, m, i, target_name in zip("rbg", "xD.", [0, 1, 2], target_names):
    plt.scatter(reduced_X[y == i, 0], reduced_X[y == i, 1],marker=m, c=c, label=target_name)

plt.legend()
plt.title('PCA of IRIS dataset(scikit-learn.org example)')
plt.show()

# We then randomly split the images into training and test sets, and fit the PCA object on the training set:
X_train, X_test, y_train, y_test = train_test_split(X, y)
pca = PCA(n_components=150)

print len(X_train), len(X_test), len(y_train), len(y_test)

#We reduce all of the instances to 150 dimensions and train a logistic regression 
#classifier. The data set contains forty classes; scikit-learn automatically creates  
#binary classifiers using the one versus all strategy behind the scenes:
X_train_reduced = pca.fit_transform(X_train)
X_test_reduced = pca.transform(X_test)
print 'The original dimensions of the training data were', X_train.shape
print 'The reduced dimensions of the training data are', X_train_reduced.shape


classifier = LogisticRegression()
accuracies = cross_val_score(classifier, X_train_reduced, y_train)
#Finally, we evaluate the performance of the classifier using cross-validation and a 
#test set. The average per-class F1 score of the classifier trained on the full data was 
#0.94, but required significantly more time to train and could be prohibitively slow in 
#an application with more training instances:
print 'Cross validation accuracy:', np.mean(accuracies), accuracies


classifier.fit(X_train_reduced, y_train)
predictions = classifier.predict(X_test_reduced)
print classification_report(y_test, predictions)