from IPython import parallel
rc = parallel.Client()
ref = parallel.Reference
%precision 2
u'%.2f'
rc.ids
dview = rc[:]
dview
<DirectView [0, 1, 2, 3,...]>
dview.scatter('rank', rc.ids, flatten=True)
<AsyncResult: scatter>
%%px
import numpy as np
A = np.random.random((rank+1,100))
B = np.random.random((100,rank+1))
lview = rc.load_balanced_view()
ref = parallel.Reference
You can use parallel.Reference
to refer to the task function and/or any of its arguments
that should be loaded from the remote namespace.
C_async = lview.apply(ref('A.dot'), ref('B'))
C_async
<AsyncResult: A.dot>
C_async.get()
array([[ 23.86, 26.7 , 27.13, 23.23, 26.71, 25.79], [ 23.68, 25.63, 26.76, 21.04, 26.23, 23.78], [ 23.53, 24.25, 26.79, 21.83, 24.83, 23.12], [ 24.2 , 27.3 , 24.49, 20.01, 24.84, 23.64], [ 25.97, 28.96, 26. , 24.07, 27.9 , 25.87], [ 25.77, 25.53, 25.47, 22.08, 26.1 , 25.75]])
from IPython.display import display
async_dots = [ lview.apply(ref('A.dot'), ref('B')) for id in rc.ids ]
for ar in async_dots:
C = ar.get()
print "A.dot(B) on engine %s:" % ar.engine_id
print C.shape
print C
A.dot(B) on engine 6: (7, 7) [[ 25.08 24.8 29.35 27.23 24.49 24.09 26.03] [ 26.54 26.62 29.57 29.18 26.74 26.19 25.74] [ 25.34 24.96 27.09 26.84 25.74 23.97 23.97] [ 23. 23.16 25.79 25.14 24.54 23.96 23.52] [ 25.53 26.05 27.65 27.44 27.65 25.58 24.86] [ 21.61 23.5 24.21 25.27 23.6 21.76 21.81] [ 23.72 23.13 25.99 22.32 23.04 21.49 22.98]] A.dot(B) on engine 4: (5, 5) [[ 20.04 21.75 23.82 18.53 23.31] [ 25. 25.96 27.3 23.83 26.82] [ 20.85 24.51 24.28 21.6 22.52] [ 21.82 22.95 23.72 21.93 23.16] [ 23.79 24.13 26.42 24.55 26.46]] A.dot(B) on engine 3: (4, 4) [[ 21.25 24.62 24.64 25.93] [ 25.83 28. 24.42 27.74] [ 24.81 24.13 23.41 26.73] [ 23.04 24.79 22.56 27.36]] A.dot(B) on engine 2: (3, 3) [[ 25.91 26.31 24.97] [ 24.06 24.77 24.61] [ 25.55 25.38 25.31]] A.dot(B) on engine 1: (2, 2) [[ 23.97 27.08] [ 25.86 25.33]] A.dot(B) on engine 0: (1, 1) [[ 23.53]] A.dot(B) on engine 7: (8, 8) [[ 23.05 24.24 24.83 25.14 23.69 27.05 23.99 23.09] [ 27.54 25.62 29.51 27.88 27.66 31.78 29.02 26.88] [ 27.86 24.5 27.1 25.37 25.34 30.25 25.3 24.9 ] [ 22.33 21.18 24.85 22.48 23.36 24.74 24.24 23.68] [ 27.11 26.53 26.65 26.48 24.81 28.17 27.07 25.46] [ 24.47 23.89 25.77 25.08 24.72 30.19 24.37 25.16] [ 26. 24.8 26.19 27.6 25.6 29.89 25.47 24.5 ] [ 21.97 21.59 24.88 22.72 22.52 24.59 24.3 22.05]] A.dot(B) on engine 5: (6, 6) [[ 23.86 26.7 27.13 23.23 26.71 25.79] [ 23.68 25.63 26.76 21.04 26.23 23.78] [ 23.53 24.25 26.79 21.83 24.83 23.12] [ 24.2 27.3 24.49 20.01 24.84 23.64] [ 25.97 28.96 26. 24.07 27.9 25.87] [ 25.77 25.53 25.47 22.08 26.1 25.75]]
Or you can just rely on module globals, which resolves to __main__
,
the user namespace.
async_dots = [ lview.apply_async(lambda : A.dot(B)) for id in rc.ids ]
for ar in async_dots:
C = ar.get()
print "A.dot(B) on engine %s:" % ar.engine_id
print C.shape
print C
A.dot(B) on engine 6: (7, 7) [[ 25.08 24.8 29.35 27.23 24.49 24.09 26.03] [ 26.54 26.62 29.57 29.18 26.74 26.19 25.74] [ 25.34 24.96 27.09 26.84 25.74 23.97 23.97] [ 23. 23.16 25.79 25.14 24.54 23.96 23.52] [ 25.53 26.05 27.65 27.44 27.65 25.58 24.86] [ 21.61 23.5 24.21 25.27 23.6 21.76 21.81] [ 23.72 23.13 25.99 22.32 23.04 21.49 22.98]] A.dot(B) on engine 4: (5, 5) [[ 20.04 21.75 23.82 18.53 23.31] [ 25. 25.96 27.3 23.83 26.82] [ 20.85 24.51 24.28 21.6 22.52] [ 21.82 22.95 23.72 21.93 23.16] [ 23.79 24.13 26.42 24.55 26.46]] A.dot(B) on engine 3: (4, 4) [[ 21.25 24.62 24.64 25.93] [ 25.83 28. 24.42 27.74] [ 24.81 24.13 23.41 26.73] [ 23.04 24.79 22.56 27.36]] A.dot(B) on engine 2: (3, 3) [[ 25.91 26.31 24.97] [ 24.06 24.77 24.61] [ 25.55 25.38 25.31]] A.dot(B) on engine 1: (2, 2) [[ 23.97 27.08] [ 25.86 25.33]] A.dot(B) on engine 0: (1, 1) [[ 23.53]] A.dot(B) on engine 7: (8, 8) [[ 23.05 24.24 24.83 25.14 23.69 27.05 23.99 23.09] [ 27.54 25.62 29.51 27.88 27.66 31.78 29.02 26.88] [ 27.86 24.5 27.1 25.37 25.34 30.25 25.3 24.9 ] [ 22.33 21.18 24.85 22.48 23.36 24.74 24.24 23.68] [ 27.11 26.53 26.65 26.48 24.81 28.17 27.07 25.46] [ 24.47 23.89 25.77 25.08 24.72 30.19 24.37 25.16] [ 26. 24.8 26.19 27.6 25.6 29.89 25.47 24.5 ] [ 21.97 21.59 24.88 22.72 22.52 24.59 24.3 22.05]] A.dot(B) on engine 5: (6, 6) [[ 23.86 26.7 27.13 23.23 26.71 25.79] [ 23.68 25.63 26.76 21.04 26.23 23.78] [ 23.53 24.25 26.79 21.83 24.83 23.12] [ 24.2 27.3 24.49 20.01 24.84 23.64] [ 25.97 28.96 26. 24.07 27.9 25.87] [ 25.77 25.53 25.47 22.08 26.1 25.75]]