For SO Question 14184621
Connect to the cluster
from IPython.parallel import Client
rc = Client()
lview = rc.load_balanced_view()
dv = rc[:]
Define a magic that runs a cell locally and remotely
def pxlocal(line, cell):
ip = get_ipython()
ip.run_cell_magic("px", line, cell)
ip.run_cell(cell)
ip.register_magic_function(pxlocal, "cell")
Use the magic to define a class interactively everywhere (including local)
%%pxlocal
class MyClass(object):
def __init__(self, parameter):
self.parameter = parameter
def update_something(self, some_data):
# do something smart here with some_data & internal state
self.parameter = some_data
def compute_something(self, other_data):
# do something smart here with other data & internal state
return self.parameter * other_data
Run some tasks
def process(obj, some_data, other_data):
obj.update_something(some_data)
return obj.compute_something(other_data)
tasks = []
some_instances = [MyClass(i) for i in range(5)]
data_source_1 = range(1,4)
data_source_2 = range(4,1,-1)
for obj in some_instances:
for some_data in data_source_1:
for other_data in data_source_2:
ar = lview.apply_async(process, obj, some_data, other_data)
tasks.append(ar)
# wait for computation to end
results = [ar.get() for ar in tasks]
results[:18]
[4, 3, 2, 8, 6, 4, 12, 9, 6, 4, 3, 2, 8, 6, 4, 12, 9, 6]