import pandas as pd
from IPython.parallel import Client, error
Relevant part of the class. The error happens when trying to execute the code on the engines
class Load_balanced_view(object):
def __init__(self, profile='default'):
#initialise a direct and a load ballanced view
self.c = Client(profile=profile)
self.dview = self.c[:]
self.lbview = self.c.load_balanced_view()
def exec_on_engine(self, code, block=True):
"""
Execute the given code on all engines
Parameters
----------
to_execute: string or list of strings
command(s) to execute on all the nodes. Thought for short tasks,
like importing modules
block: bool
whether or not to wait until done to return. default: True
"""
#Six: Python 2 and 3 Compatibility Library
from six import string_types #appropriate string type
if isinstance(code, string_types): # if it's a string
code = [code,] # convert to list
# execute the required commands
# (better to do in block mode, avoids errors if command is slow)
for te in code:
try:
self.dview.execute(te, block=block)
except error.CompositeError as e: # if an error occurs, print a single one, not one per engine
e.raise_exception()
My default profile (set up for ipy v0.13).
In ipython_config.py
:
c.InteractiveShellApp.pylab = 'qt'
c.InteractiveShellApp.pylab_import_all = False
In ipython_notebook_config.py
:
c.IPKernelApp.pylab = 'qt'
c.IPKernelApp.pylab_import_all = False
In startup/01-interactive.py
#enable interactive plotting
plt.ion()
print("Interactive plot on")
Motivation:
I mostlt use numpy and matplotlib and don't want to import them every time.
Plus I don't like from pylab import *
and I want interactive plotting (I guess that this affects mostly the console)
parallel_env = Load_balanced_view(profile='default')
I've create an "empty" profile with
ipython profile create empty --reset
If I start the engines with profile=empty
and execute the cell below, the whole code runs without problem
#parallel_env = Load_balanced_view(profile='empty')
If I download this notebook as .py
and run it, I get the error in the same position as here, but the error message is less helpful:
File "/home/montefra/.local/lib/python2.7/site-packages/IPython/parallel/error.py", line 199, in raise_exception
raise RemoteError(en, ev, etb, ei)
RemoteError: NameError(name 'plt' is not defined)
imports = ['import numpy as np',]
parallel_env.exec_on_engine(imports)
--------------------------------------------------------------------------- NameError Traceback (most recent call last)/home/montefra/.local/lib/python2.7/site-packages/IPython/utils/py3compat.pyc in execfile(fname, *where) 202 else: 203 filename = fname --> 204 __builtin__.execfile(filename, *where) /home/montefra/.config/ipython/profile_default/startup/01-interactive.py in <module>() 1 #enable interactive plotting ----> 2 plt.ion() 3 4 print("Interactive plot on") 5 NameError: name 'plt' is not defined
imports = ['import numpy as np',]
parallel_env.exec_on_engine(imports)
imports = ['import pandas as pd',]
parallel_env.exec_on_engine(imports)
imports = ['import pandas as pd',]
parallel_env.exec_on_engine(imports)