from __future__ import division import tgt import pandas as pd from mumodo.mumodo import * import mumodo.reader as mr from scipy.signal import medfilt import math from sgolayscipyorg import * from mpl_toolkits.mplot3d import Axes3D pd.set_option('display.column_space',20) pd.set_option('display.max_columns',40) pd.set_option('display.max_rows',400) path_to_xiofile='example_files/r1_12_15/r1_12_15.xio.gz' f = xiofile(path_to_xiofile, 'r',indexing = False) facelab = pd.DataFrame(quantize(f.xioquicklinegen(0,0,True),"FaceLabNode")) # Head, eye, and gaze tracking from seeingmachines FaceLab kinect = pd.DataFrame(quantize(f.xioquicklinegen(0,0,True),"irioKinect")) # motion capture from Microsoft Kinect #facelab.to_csv("/Users/spyroskousidis/Desktop/facelab.csv") #kinect.to_csv("/Users/spyroskousidis/Desktop/kinect.csv") #facelab = pd.DataFrame.from_csv("/Users/spyroskousidis/Desktop/facelab.csv") #kinect = pd.DataFrame.from_csv("/Users/spyroskousidis/Desktop/kinect.csv") kinect[0:1] kinect.index = kinect['time'] - 1341393414830 kinect.ix[60000:60100] skeletonframe = kinect.ix[:,['ElbowLeft', 'ElbowRight','HandLeft', 'HandRight', 'Head', 'ShoulderCenter','ShoulderLeft', 'ShoulderRight','WristLeft', 'WristRight']] skeletonframe.ix[65000:65010] point = skeletonframe['Head'].ix[65008] print point.v # as vector print point.x #a specific coordinate plot(list(skeletonframe['Head'][0:100].map(lambda x: x.x))) frame_time = 65019 fig = figure(figsize=(8,8)) ax = fig.add_subplot(111, projection='3d') xs ,ys, zs = [], [], [] for joint in skeletonframe.columns: xs.append(skeletonframe[joint].ix[frame_time].x) ys.append(skeletonframe[joint].ix[frame_time].y) zs.append(skeletonframe[joint].ix[frame_time].z) ax.scatter(xs, ys, zs, s=150,c='b') #draw lines between points ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Z') start_time = 62000 end_time = 63000 fig = figure(figsize = (10,8)) ax = fig.add_subplot(111, projection='3d') ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Z') for frame_time in skeletonframe.ix[start_time:end_time].index: xs,ys, zs = [], [], [] for joint in ['HandLeft', 'HandRight']: try: xs.append(skeletonframe[joint].ix[frame_time].x) ys.append(skeletonframe[joint].ix[frame_time].y) zs.append(skeletonframe[joint].ix[frame_time].z) a = (frame_time - start_time)/ (end_time-start_time) #cs = cmap((frame_time - start_time)/ (end_time-start_time)) except AttributeError: pass ax.scatter(xs, ys, zs, s= 50, c = ['r','b'], alpha = a) start_time = 62500 end_time = 64000 joint = 'Head' fig = figure(figsize = (10,8)) ax = fig.add_subplot(111, projection='3d') ax.set_xlabel('X') ax.set_ylabel('Y') ax.set_zlabel('Z') xs,ys, zs, cs = [], [], [], [] for frame_time in skeletonframe.ix[start_time:end_time].index: try: xs.append(skeletonframe[joint].ix[frame_time].x) ys.append(skeletonframe[joint].ix[frame_time].y) zs.append(skeletonframe[joint].ix[frame_time].z) #cs.append((frame_time - start_time)/ (end_time-start_time)) cs.append(frame_time) except AttributeError: pass cax = ax.scatter(xs, ys, zs, zdir = 'z', s= 50, c = cs, cmap = cm.jet) fig.colorbar(cax) myannotations = pd.Panel(mr.Textgrid_to_Pandas("example_files/GEO-TXT-LI-00001-72_ch1.TextGrid")) myannotations #show the tiers mytier = myannotations['silences'] fig=figure(figsize = (24,2)) colormap = { 'p' : 'white', 'o' : 'red', 's1' : 'blue', 's2' : 'green','n': 'black'} ax1 = subplot2grid((1,7), (0,0), colspan=5) ax1.set_xlabel('time(s)',fontsize = 15) for i in mytier[0:20].index: try: ax1.axvspan(mytier['start_time'][i], mytier['end_time'][i], facecolor = colormap[mytier['text'][i]]) except KeyError: ax1.axvspan(mytier['start_time'][i], mytier['end_time'][i], facecolor = colormap['n']) ax2 = subplot2grid((1,7), (0,6)) ax2.pos = arange(5)+.1 # the bar centers on the y axis ax2.barh(ax2.pos, [50*x for x in np.ones(5)], align='center',height=0.5, color = (colormap['p'], colormap['o'], colormap['s1'], colormap['s2'], colormap['n'])) ax2.get_xaxis().set_visible(False) ax2.set_yticks(range(5)) ax2.set_xticks(range(100)) ax2.set_yticklabels(('silence', 'overlap', 'speaker1', 'speaker2' , 'non-speech'), fontsize = 15) This annotation file contains head gestures myannotations = pd.Panel(mr.Textgrid_to_Pandas("example_files/r1-20120704-cam1-head-sk.TextGrid")) myannotations #show the tiers mytier = myannotations['HeadSK'] mytier = mytier[mytier['text']>''] mytier[20:30] f = facelab.dropna() f['time_real'] = 1000 * f['frameTimeSeconds'] + f['frameTimeMilliSeconds'] plot(f.index,f['time']-f['time_real']) print np.mean(f['time']-f['time_real']) f.drop_duplicates(cols=['time_real']) f.index = f['time_real'].map(lambda x: int(x) - 1341393414826 ) f = f.sort_index() f.ix[0:10] print "quaternion:", f['headRotation'][0].v #as vector print "Pitch (or bank):",f['headRotation'][0].bank #Bank or pitch Euler angle print "Yaw (or heading):",f['headRotation'][0].heading #Yaw or heading Euler angle print "Roll (or attitude):" ,f['headRotation'][0].attitude #Roll or attitude Euler angle f["bank"] = f["headRotation"].map(lambda x: x.bank) f['bank'].index Plot gestures and calculate velocity using a Savisky-Golay filter to smooth the data. Here we see a nod with two cycles #plot_gesture axisID = "bank" start = 13500 end = 13800 data = [] for x in f.ix[start:end][axisID]: data.append(x) print len(data),start,end y = savitzky_golay(data, window_size = 7, order = 3, deriv =0) dy = savitzky_golay(data, window_size =7, order = 3, deriv =1) subplot(211) plot(f.ix[start:end].index,data,'b',f.ix[start:end].index,y,'r') subplot(212) plot(f.ix[start:end].index,dy) myannotations = pd.Panel(mr.Textgrid_to_Pandas("example_files/r1_12_15/r1215en.TextGrid")) myannotations mytier = myannotations['A (en)'] mytier fig = figure(figsize