Working with dicts in IPython.parallel
def scatter_dict(view, name, d):
"""partition a dictionary across the engines of a view"""
ntargets = len(view)
keys = d.keys() # list(d.keys()) in Python 3
for i, target in enumerate(view.targets):
subd = {}
for key in keys[i::ntargets]:
subd[key] = d[key]
view.client[target][name] = subd
from IPython import parallel
rc = parallel.Client()
dv = rc[:]
dv
<DirectView [0, 1, 2, 3]>
test_dict = {'43':"lion", '34':"tiger", '343':"duck"}
scatter_dict(dv, 'test', test_dict)
dv['test']
[{'343': 'duck'}, {'43': 'lion'}, {'34': 'tiger'}, {}]
def dostuff():
d2 = {}
for k,v in test.items():
print k, v
d2[k[::-1]] = v[::-1]
return d2
ar = dv.apply_async(dostuff)
ar.wait()
ar.display_outputs()
ar.get()
[stdout:0] 343 duck [stdout:1] 43 lion [stdout:2] 34 tiger
[{'343': 'kcud'}, {'34': 'noil'}, {'43': 'regit'}, {}]
def gather_dict(view, name):
"""gather dictionaries from a DirectView"""
merged = {}
for d in view.pull(name):
merged.update(d)
return merged
gather_dict(dv, 'test')
{'34': 'tiger', '343': 'duck', '43': 'lion'}