Multimodal Dialogue Data Analysis¶

f = xiofile(path_to_xiofile, 'r',indexing = False)

kinect[0:1]

kinect.ix[60000:60100]

skeletonframe = kinect.ix[:,['ElbowLeft', 'ElbowRight','HandLeft', 'HandRight', 'Head', 'ShoulderCenter','ShoulderLeft', 'ShoulderRight','WristLeft', 'WristRight']]
skeletonframe.ix[65000:65010]

point = skeletonframe['Head'].ix[65008]
print point.v # as vector
print point.x #a specific coordinate

plot(list(skeletonframe['Head'][0:100].map(lambda x: x.x)))

frame_time = 65019
fig = figure(figsize=(8,8))
ax = fig.add_subplot(111, projection='3d')
xs ,ys, zs = [], [], []
for joint in skeletonframe.columns:
    xs.append(skeletonframe[joint].ix[frame_time].x)
    ys.append(skeletonframe[joint].ix[frame_time].y)
    zs.append(skeletonframe[joint].ix[frame_time].z)
ax.scatter(xs, ys, zs, s=150,c='b')
#draw lines between points
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')

start_time = 62000
end_time = 63000
fig = figure(figsize = (10,8))
ax = fig.add_subplot(111, projection='3d')
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
for frame_time in skeletonframe.ix[start_time:end_time].index:
    xs,ys, zs = [], [], []
    for joint in ['HandLeft', 'HandRight']:
        try:
            xs.append(skeletonframe[joint].ix[frame_time].x)
            ys.append(skeletonframe[joint].ix[frame_time].y)
            zs.append(skeletonframe[joint].ix[frame_time].z)
            a = (frame_time - start_time)/ (end_time-start_time)
            #cs = cmap((frame_time - start_time)/ (end_time-start_time))
        except AttributeError:
            pass
        ax.scatter(xs, ys, zs, s= 50, c = ['r','b'], alpha = a)

start_time = 62500
end_time = 64000
joint = 'Head'
fig = figure(figsize = (10,8))
ax = fig.add_subplot(111, projection='3d')
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')
xs,ys, zs, cs = [], [], [], []
for frame_time in skeletonframe.ix[start_time:end_time].index:
    try:
        xs.append(skeletonframe[joint].ix[frame_time].x)
        ys.append(skeletonframe[joint].ix[frame_time].y)
        zs.append(skeletonframe[joint].ix[frame_time].z)
        #cs.append((frame_time - start_time)/ (end_time-start_time))
        cs.append(frame_time)
    except AttributeError:
        pass
cax = ax.scatter(xs, ys, zs, zdir = 'z', s= 50, c = cs, cmap = cm.jet)
fig.colorbar(cax)

myannotations = pd.Panel(mr.Textgrid_to_Pandas("example_files/GEO-TXT-LI-00001-72_ch1.TextGrid"))
myannotations #show the tiers

fig=figure(figsize = (24,2))
colormap = { 'p' : 'white', 'o' : 'red', 's1' : 'blue', 's2' : 'green','n': 'black'}
ax1 = subplot2grid((1,7), (0,0), colspan=5)
ax1.set_xlabel('time(s)',fontsize = 15)
for i in mytier[0:20].index:
    try:
        ax1.axvspan(mytier['start_time'][i], mytier['end_time'][i], facecolor = colormap[mytier['text'][i]])
    except KeyError:
        ax1.axvspan(mytier['start_time'][i], mytier['end_time'][i], facecolor = colormap['n'])
ax2 = subplot2grid((1,7), (0,6))    
ax2.pos = arange(5)+.1    # the bar centers on the y axis
ax2.barh(ax2.pos, [50*x for x in np.ones(5)], align='center',height=0.5, color = (colormap['p'], colormap['o'], colormap['s1'], colormap['s2'], colormap['n'])) 
ax2.get_xaxis().set_visible(False)
ax2.set_yticks(range(5))
ax2.set_xticks(range(100))
ax2.set_yticklabels(('silence', 'overlap', 'speaker1', 'speaker2' , 'non-speech'), fontsize = 15)

myannotations = pd.Panel(mr.Textgrid_to_Pandas("example_files/r1-20120704-cam1-head-sk.TextGrid"))
myannotations #show the tiers

mytier[20:30]

plot(f.index,f['time']-f['time_real'])
print np.mean(f['time']-f['time_real'])

f.ix[0:10]

print "quaternion:", f['headRotation'][0].v #as vector
print "Pitch (or bank):",f['headRotation'][0].bank #Bank or pitch Euler angle
print "Yaw (or heading):",f['headRotation'][0].heading #Yaw or heading Euler angle
print "Roll (or attitude):" ,f['headRotation'][0].attitude #Roll or attitude Euler angle

f['bank'].index

#plot_gesture
axisID = "bank"
start = 13500
end = 13800
data = []
for x in f.ix[start:end][axisID]:
    data.append(x)
print len(data),start,end
y = savitzky_golay(data, window_size = 7, order = 3, deriv =0)
dy = savitzky_golay(data, window_size =7, order = 3, deriv =1)
subplot(211)
plot(f.ix[start:end].index,data,'b',f.ix[start:end].index,y,'r')
subplot(212)
plot(f.ix[start:end].index,dy)

myannotations

mytier = myannotations['A (en)']
mytier