Watching stdout of an AsyncResult as it arrives

In [1]:
from IPython import parallel
rc = parallel.Client()
In [2]:
def print_stuff(n):
    """dummy task that prints stuff"""
    import os, sys
    for i in range(n):
        print i, os.getpid()
        # stdout.flush is important, to ensure that stdout messages are
        # published in a timely manner
        sys.stdout.flush()
        time.sleep(1)
In [3]:
print_stuff(3)
0 99629
1 99629
2 99629

Create our view

In [4]:
dview = rc[:]
dview
Out[4]:
<DirectView [0, 1, 2, 3]>

Define a function that watches stdout as we wait for an AsyncResult to complete

In [13]:
import sys
import time
from IPython.display import clear_output

def wait_watching_stdout(ar, dt=1, truncate=1000):
    while not ar.ready():
        stdouts = ar.stdout
        if not any(stdouts):
            continue
        # clear_output doesn't work in plain terminal / script environments
        clear_output()
        print '-' * 30
        print "%.3fs elapsed" % ar.elapsed
        print ""
        for eid, stdout in zip(ar._targets, ar.stdout):
            if stdout:
                print "[ stdout %2i ]\n%s" % (eid, stdout[-truncate:])
        sys.stdout.flush()
        time.sleep(dt)
        
In [12]:
ar = dview.apply_async(print_stuff, 5)
wait_watching_stdout(ar)
------------------------------
4.024s elapsed

[ stdout 0 ]
0 99588
1 99588
2 99588
3 99588
4 99588

[ stdout 1 ]
0 99587
1 99587
2 99587
3 99587
4 99587

[ stdout 2 ]
0 99589
1 99589
2 99589
3 99589
4 99589

[ stdout 3 ]
0 99594
1 99594
2 99594
3 99594
4 99594