from sklearn.base import BaseEstimator
from sklearn.preprocessing import MinMaxScaler, StandardScaler
import numpy as np
def load_dataset(fname='data/ex1data2.txt'):
"""
load a (X,y) tuple while X is a n_samples * n_features numpy array and y is the label.
"""
X = []
y = []
for line in open(fname):
rows = [int(x) for x in line.split(',')]
X.append(rows[0:2])
y.append(rows[-1])
return np.array(X, float32), np.array(y, int)
class LinearRegression(BaseEstimator):
"""
implement the linear regression with gradient descent.
"""
def __init__(self, alpha=0.3, max_iters=100, tols=0.001):
# learning rate
self.alpha = alpha
self.max_iters = max_iters
self.tols = tols
self.histories_ = []
def fit(self, X, y):
i = 0
prep_J = 0
m, n = X.shape
self.theta = np.zeros(n)
while i < self.max_iters:
self.theta = self.theta - self.alpha * self._compute_gradient(X, y)
J = self._compute_cost(X, y)
if abs(J - prep_J) <= self.tols:
print 'converged'
break
else:
# print 'iteration %s cost function %s' %(i, J)
prep_J = J
self.histories_.append(J)
i += 1
def predict(self, X):
return np.dot(X, self.theta)
def _compute_cost(self, X, y):
m = X.shape[0]
predicted = self.predict(X)
error = predicted - y
return np.dot(error.T, error) / (2 * m)
def _compute_gradient(self, X, y):
predicted = self.predict(X)
error = predicted - y
m = X.shape[0]
return np.dot(X.T, error) / m
X, y = load_dataset()
min_max_scalar = MinMaxScaler()
lr = LinearRegression()
X_scaled = min_max_scalar.fit_transform(X)
X = np.concatenate((np.ones((X.shape[0], 1)), X_scaled), axis=1)
lr.fit(X, y)
X, y = load_dataset()
lr2 = LinearRegression()
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
X = np.concatenate((np.ones((X.shape[0], 1)), X_scaled), axis=1)
lr2.fit(X, y)
import matplotlib.pyplot as plt
x1 = [i for i in range(len(lr.histories_))]
y1 = lr.histories_
x2 = [i for i in range(len(lr2.histories_))]
y2 = lr2.histories_
plt.xlabel('# iterations')
plt.ylabel('cost function')
plt.plot(x1, y1)
plt.plot(x2, y2, 'r')
plt.show()
test = np.array([1650, 3], float32)
print 'predicted', np.dot(np.hstack((1, min_max_scalar.transform(test))), lr.theta)
print 'predicted', np.dot(np.hstack((1, scaler.transform(test))), lr2.theta)