train = 'data/train.csv'
test = 'data/test.csv'
train_data = pd.read_csv(train)
test_data = pd.read_csv(test)

#一列目と最終列(ROLE_CODE)以外のtest, trainデータを合わせる
all_data = np.vstack((train_data.ix[:,1:-1], test_data.ix[:,1:-1]))   
num_train = np.shape(train_data)[0]         

print "train_data.shape:", train_data.shape
print "train_data:"
print train_data.ix[:1,]

print "\nTransforming data…"
dp = group_data(all_data, degree=2)   #2種類ずつグルーピングした2つのデータ列
dt = group_data(all_data, degree=3)   #3種類ずつグルーピングしたデータ列

y = array(train_data.ACTION)

X = all_data[:num_train]    # ~学習用データ行まで
X_2 = dp[:num_train]
X_3 = dt[:num_train]

X_test = all_data[num_train:]   #テスト用データ行 ~ 
X_test_2 = dp[num_train:]
X_test_3 = dt[num_train:]

X_train_all = np.hstack((X, X_2, X_3))    
X_test_all = np.hstack((X_test, X_test_2, X_test_3))

print "X_train_all[0,5:10]: "
print X_train_all[0,5:11]
    

num_features = X_train_all.shape[1]          
print "num_features:", num_features     #特徴量 = (8 + 8C2 + 8C3)

model = linear_model.LogisticRegression()     #ロジスティックモデルを"model"で呼び出し
Xts = [OneHotEncoder(X_train_all[:,[i]])[0] for i in range(num_features)] 

print "Performing greedy feature selection..."
score_hist = []
N = 10
good_features = set([])
#最低2回以上、最新のスコアが一個前よりも悪くなるまで続ける
#3-4時間かかる
while len(score_hist) < 2 or score_hist[-1][0] > score_hist[-2][0]:
    scores = []
    
    #good説明変数以外の変数を足して交差検定してみる
    for f in range(len(Xts)):
        if f not in good_features:
            feats = list(good_features) + [f]                    #新しく説明変数を追加
            Xt = sparse.hstack([Xts[j] for j in feats]).tocsr()  
            score = cv_loop(Xt, y, model, N)
            scores.append((score, f))
            print "Feature: %i Mean AUC: %f" % (f, score)   #i番目の変数を説明変数に加えたときのAUC
    good_features.add(sorted(scores)[-1][1])      #一番よかった変数をgood説明変数に追加
    score_hist.append(sorted(scores)[-1])         #よかったスコアと変数を記録
    print "Current features: %s" % sorted(list(good_features))
    
     # Remove last added feature from good_features
good_features.remove(score_hist[-1][1])
good_features = sorted(list(good_features))
print "Selected features %s" % good_features   #特徴量

print "Training full model..."
model.fit(X_train, y)

print "Making prediction and saving results..."
preds = model.predict_proba(X_test)[:,1]
create_test_submission(submit, preds)

hash((81625,21375))

data = X_train_all[:,[7]]
keymap = None

if keymap is None:
          keymap = []
          for col in data.T:
               uniques = set(list(col))
               keymap.append(dict((key, i) for i, key in enumerate(uniques)))
                
total_pts = data.shape[0]
outdat = []    
for i, col in enumerate(data.T):
     km = keymap[i]
     num_labels = len(km)
     spmat = sparse.lil_matrix((total_pts, num_labels)) #要素がすべて0の疎行列作成
     for j, val in enumerate(col):
          if val in km:
               spmat[j, km[val]] = 1
     outdat.append(spmat)                
print spmat.toarray()                
print "output = "
print sparse.hstack(outdat).tocsr()[:3]
print keymap[0]

    mean_auc = 0.
    X = Xt
    N = 10
    for i in range(N):
        X_train, X_cv, y_train, y_cv = cross_validation.train_test_split(
                                       X, y, test_size=.20, 
                                       random_state = i*SEED)
        model.fit(X_train, y_train)
        preds = model.predict_proba(X_cv)[:,1]
        auc = metrics.auc_score(y_cv, preds)
        print "AUC (fold %d/%d): %f" % (i + 1, N, auc)
        mean_auc += auc

x1 = np.logspace( -4, 4, N, base = 2)
print x1
y = np.zeros(N)
plt.plot(x1, y, 'o')

np.logspace(-4, 4, 15, base = 2)