For SO Question 14184621

Connect to the cluster

In [1]:
from IPython.parallel import Client
rc = Client()
lview = rc.load_balanced_view()
dv = rc[:]

Define a magic that runs a cell locally and remotely

In [2]:
def pxlocal(line, cell):
    ip = get_ipython()
    ip.run_cell_magic("px", line, cell)
    ip.run_cell(cell)
ip.register_magic_function(pxlocal, "cell")    

Use the magic to define a class interactively everywhere (including local)

In [3]:
%%pxlocal
class MyClass(object):
    def __init__(self, parameter):
        self.parameter = parameter

    def update_something(self, some_data):
       # do something smart here with some_data & internal state
        self.parameter = some_data
    
    def compute_something(self, other_data):
        # do something smart here with other data & internal state
        return self.parameter * other_data

Run some tasks

In [4]:
def process(obj, some_data, other_data):
    obj.update_something(some_data)
    return obj.compute_something(other_data)

tasks = []
some_instances = [MyClass(i) for i in range(5)]
data_source_1 = range(1,4)
data_source_2 = range(4,1,-1)
for obj in some_instances:
   for some_data in data_source_1:
        for other_data in data_source_2:
            ar = lview.apply_async(process, obj, some_data, other_data)
            tasks.append(ar)

# wait for computation to end
results = [ar.get() for ar in tasks] 
In [5]:
results[:18]
Out[5]:
[4, 3, 2, 8, 6, 4, 12, 9, 6, 4, 3, 2, 8, 6, 4, 12, 9, 6]
Back to top