Set up a logger, so we can actually see the output (this is unnecessary, but helpful for debugging):
import logging
logger = logging.Logger('ipcluster')
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler(sys.stdout))
Get some config that the launchers expect from ipcluster:
ip = get_ipython()
config = ip.config
profile_dir = ip.profile_dir.location # aka ipython_dir/profile_{name}
Import the Launcher classes used for starting controller/engine subprocesses
from IPython.parallel.apps.launcher import LocalControllerLauncher, LocalEngineLauncher
There is also a LocalEngineSetLauncher class that allows you to start/stop a group of engines at a time
Define some functions for simple start/stop of engines
import time
engines = []
def add_engines(n, delay=0.1):
"""start n engines"""
for i in range(n):
e = LocalEngineLauncher(config=config, log=logger, profile_dir=profile_dir)
engines.append(e)
e.start()
if i + 1 < n:
time.sleep(delay)
def cleanup_engines():
[ e.stop() for e in engines ]
Start the Controller:
controller = LocalControllerLauncher(config=config, log=logger, profile_dir=profile_dir)
controller.start()
Starting LocalControllerLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipcontrollerapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20'] Process '/usr/bin/python' started: 44636 2012-09-27 16:38:23,506.506 [IPControllerApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default' 2012-09-27 16:38:23.830 [scheduler] Scheduler started [leastload]
Start 5 engines:
add_engines(5)
Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20'] Process '/usr/bin/python' started: 44641 Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20'] Process '/usr/bin/python' started: 44642 Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20'] Process '/usr/bin/python' started: 44643 Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20'] Process '/usr/bin/python' started: 44644 Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20'] Process '/usr/bin/python' started: 44645 2012-09-27 16:38:27,935.935 [IPEngineApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default' 2012-09-27 16:38:27,932.932 [IPEngineApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default' 2012-09-27 16:38:27,997.997 [IPEngineApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default' 2012-09-27 16:38:27,998.998 [IPEngineApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default' 2012-09-27 16:38:28,017.017 [IPEngineApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default'
Now we can work with the engines in the usual way:
from IPython import parallel
rc = parallel.Client()
rc.ids
[0, 1, 2, 3, 4]
rc[:]['a'] = 5
rc[:]['a']
[5, 5, 5, 5, 5]
And we can stop them as well:
engines[0].stop()
Process '/usr/bin/python' stopped: {'pid': 44641, 'exit_code': 0}
rc.ids
[0, 1, 3, 4]
And add some more
add_engines(3)
Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20'] Process '/usr/bin/python' started: 44676 Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20'] Process '/usr/bin/python' started: 44677 Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20'] Process '/usr/bin/python' started: 44678
rc.ids
[0, 1, 3, 4, 5, 6, 7]
And cleanup the whole cluster:
rc.shutdown(hub=True)
2012-09-27 16:39:05.590 [scheduler] Unhandled message type: u'shutdown_notice' Process '/usr/bin/python' stopped: {'pid': 44645, 'exit_code': 0} Process '/usr/bin/python' stopped: {'pid': 44642, 'exit_code': 0} Process '/usr/bin/python' stopped: {'pid': 44676, 'exit_code': 0} Process '/usr/bin/python' stopped: {'pid': 44678, 'exit_code': 0} Process '/usr/bin/python' stopped: {'pid': 44643, 'exit_code': 0} Process '/usr/bin/python' stopped: {'pid': 44677, 'exit_code': 0} Process '/usr/bin/python' stopped: {'pid': 44644, 'exit_code': 0} Process '/usr/bin/python' stopped: {'pid': 44636, 'exit_code': 0}