%pylab inline
Populating the interactive namespace from numpy and matplotlib
import pandas as pd
import joblib
column_names = ['Square_id',
'Time_interval',
'Country_code',
'SMS_in',
'SMS_out',
'Call_in',
'Call_out',
'Internet_traffic']
dtypes = {'Square_id': int32,
'Time_interval': uint64,
'Country_code': int32,
'SMS_in': float32,
'SMS_out': float32,
'Call_in': float32,
'Call_out': float32,
'Internet_traffic': float32}
file_pattern = 'data/telco/sms-call-internet-mi-2013-{:02d}-{:02d}.txt'
df_list = []
for day in range(1,31):
print 'loading', file_pattern.format(11, day)
df = pd.read_csv(file_pattern.format(11, day),
sep='\t', header=None,
names=column_names, dtype=dtypes)
df_list.append(df)
for day in range(1,32):
print 'loading', file_pattern.format(12, day)
df = pd.read_csv(file_pattern.format(12, day),
sep='\t', header=None,
names=column_names, dtype=dtypes)
df_list.append(df)
df = pd.concat(df_list)
loading data/telco/sms-call-internet-mi-2013-11-01.txt loading data/telco/sms-call-internet-mi-2013-11-02.txt loading data/telco/sms-call-internet-mi-2013-11-03.txt loading data/telco/sms-call-internet-mi-2013-11-04.txt loading data/telco/sms-call-internet-mi-2013-11-05.txt loading data/telco/sms-call-internet-mi-2013-11-06.txt loading data/telco/sms-call-internet-mi-2013-11-07.txt loading data/telco/sms-call-internet-mi-2013-11-08.txt loading data/telco/sms-call-internet-mi-2013-11-09.txt loading data/telco/sms-call-internet-mi-2013-11-10.txt loading data/telco/sms-call-internet-mi-2013-11-11.txt loading data/telco/sms-call-internet-mi-2013-11-12.txt loading data/telco/sms-call-internet-mi-2013-11-13.txt loading data/telco/sms-call-internet-mi-2013-11-14.txt loading data/telco/sms-call-internet-mi-2013-11-15.txt loading data/telco/sms-call-internet-mi-2013-11-16.txt loading data/telco/sms-call-internet-mi-2013-11-17.txt loading data/telco/sms-call-internet-mi-2013-11-18.txt loading data/telco/sms-call-internet-mi-2013-11-19.txt loading data/telco/sms-call-internet-mi-2013-11-20.txt loading data/telco/sms-call-internet-mi-2013-11-21.txt loading data/telco/sms-call-internet-mi-2013-11-22.txt loading data/telco/sms-call-internet-mi-2013-11-23.txt loading data/telco/sms-call-internet-mi-2013-11-24.txt loading data/telco/sms-call-internet-mi-2013-11-25.txt loading data/telco/sms-call-internet-mi-2013-11-26.txt loading data/telco/sms-call-internet-mi-2013-11-27.txt loading data/telco/sms-call-internet-mi-2013-11-28.txt loading data/telco/sms-call-internet-mi-2013-11-29.txt loading data/telco/sms-call-internet-mi-2013-11-30.txt loading data/telco/sms-call-internet-mi-2013-12-01.txt loading data/telco/sms-call-internet-mi-2013-12-02.txt loading data/telco/sms-call-internet-mi-2013-12-03.txt loading data/telco/sms-call-internet-mi-2013-12-04.txt loading data/telco/sms-call-internet-mi-2013-12-05.txt loading data/telco/sms-call-internet-mi-2013-12-06.txt loading data/telco/sms-call-internet-mi-2013-12-07.txt loading data/telco/sms-call-internet-mi-2013-12-08.txt loading data/telco/sms-call-internet-mi-2013-12-09.txt loading data/telco/sms-call-internet-mi-2013-12-10.txt loading data/telco/sms-call-internet-mi-2013-12-11.txt loading data/telco/sms-call-internet-mi-2013-12-12.txt loading data/telco/sms-call-internet-mi-2013-12-13.txt loading data/telco/sms-call-internet-mi-2013-12-14.txt loading data/telco/sms-call-internet-mi-2013-12-15.txt loading data/telco/sms-call-internet-mi-2013-12-16.txt loading data/telco/sms-call-internet-mi-2013-12-17.txt loading data/telco/sms-call-internet-mi-2013-12-18.txt loading data/telco/sms-call-internet-mi-2013-12-19.txt loading data/telco/sms-call-internet-mi-2013-12-20.txt loading data/telco/sms-call-internet-mi-2013-12-21.txt loading data/telco/sms-call-internet-mi-2013-12-22.txt loading data/telco/sms-call-internet-mi-2013-12-23.txt loading data/telco/sms-call-internet-mi-2013-12-24.txt loading data/telco/sms-call-internet-mi-2013-12-25.txt loading data/telco/sms-call-internet-mi-2013-12-26.txt loading data/telco/sms-call-internet-mi-2013-12-27.txt loading data/telco/sms-call-internet-mi-2013-12-28.txt loading data/telco/sms-call-internet-mi-2013-12-29.txt loading data/telco/sms-call-internet-mi-2013-12-30.txt loading data/telco/sms-call-internet-mi-2013-12-31.txt
df.index = pd.to_datetime(df.Time_interval.values, unit='ms').tz_localize('utc').tz_convert('Europe/Rome')
df.drop('Time_interval', axis=1, inplace=True)
df.sort_index(inplace=True)
STORE_FILE = 'stores/sms-call-internet-mi-table-blosc.h5'
store = pd.HDFStore(STORE_FILE, 'w', complib='blosc')
store.append('telco_data', df, format='t', data_columns=['Square_id', 'Country_code'])
store.append('telco_codes', df.Country_code.drop_duplicates(), format='t')
store.close()