#!/usr/bin/env python
# coding: utf-8

# # Run the benchmark

# In[ ]:


get_ipython().system('python benchmark.py --frameworks tensorflow gensim originalc --file data/text8 --epochs 4 --batch_size 16 --workers 7 --size 100')


# ## Load Report file

# In[54]:


from plotly.offline import download_plotlyjs, init_notebook_mode, iplot
from plotly.graph_objs import *
import json
init_notebook_mode()
with open('./report.json','r') as f:
    report = json.loads(f.read())


# ## System Information

# In[55]:


print report['systeminfo']


# ## Training Parameters

# In[56]:


print report['trainingparams']


# # Generate graphics

# Time to train(in seconds) and peak memory(in MiB) results

# In[57]:


X = list(report['time'].keys())
Y = list(report['time'].values())
data = [Bar(
            x = X,
            y = Y,
            orientation = 'v',
)]
layout = Layout(
    title = 'Time Report',
    xaxis = dict(title = 'Framework'),
    yaxis = dict(title = 'Training time (in seconds)'),
    annotations=[
        dict(x=xi,y=yi,
             text=str(yi),
             xanchor='center',
             yanchor='bottom',
             showarrow=False,
        ) for xi, yi in zip(X, Y)]
)
fig = dict(data=data, layout=layout)
iplot(fig)  


# In[58]:


X = list(report['memory'].keys())
Y = list(report['memory'].values())
data = [Bar(
            x = X,
            y = Y,
            orientation = 'v',
)]
layout = Layout(
    title = 'Memory Report',
    xaxis = dict(title = 'Framework'),
    yaxis = dict(title = 'Peak memory (in MB)'),
    annotations=[
        dict(x=xi,y=yi,
             text=str(yi),
             xanchor='center',
             yanchor='bottom',
             showarrow=False,
        ) for xi, yi in zip(X, Y)]
)
fig = dict(data=data, layout=layout)
iplot(fig)  


# Results of evaluation on the popular **Word Similarities** task. This task measures how well the notion of word similarity according to humans is captured by the word vector representations. Two lists are obtained by sorting the word pairs according to human similarity and vector-space similarity. Spearman’s correlation/rho between these
# ranked lists is the used to signify how well the vector space agrees with human judgement. 

# In[59]:


data = []
for framework in report['frameworks']:
    X = [x[0] for x in report['wordpairs'][framework]]
    Y = [x[1] for x in report['wordpairs'][framework]]
    trace = Bar(
        x = X,
        y = Y,
        name =framework
    )
    data.append(trace)
layout = Layout(
    title = 'Word Pairs Evaluation Report',
    xaxis = dict(title = 'Dataset', tickangle = -45),
    yaxis = dict(title = 'Spearman\'s Rho'),
    barmode = 'group'
)
fig = dict(data=data, layout=layout)
iplot(fig) 


# Results of evaluation on the popular **Word Analogy** task. The aim of this task is to find the missing word b' in the relation: a is to a' as b is to b'. In other words we look at the most similar word vector to b' (= a' + b - a) and compare it with the human analogy and report the accuracy. 

# In[60]:


data = []
for framework in report['frameworks']:
    X = [x[0] for x in report['qa'][framework]]
    Y = [x[1] for x in report['qa'][framework]]
    trace = Bar(
        x = X,
        y = Y,
        name =framework
    )
    data.append(trace)
layout = Layout(
    title = 'Analogies Task(Questions&Answers) Report',
    xaxis = dict(tickangle = -45),
    yaxis = dict(title = 'Accuracy (in %)'),
    barmode = 'group'
)
fig = dict(data=data, layout=layout)
iplot(fig)  


# In[ ]: