To get a sense of what IPython.parallel might be used for, we start with an example of some batch processing of image files with scikit-image. We will revisit pieces of this example as we learn about the different components of IPython.
You can download images with this notebook, or get a zip here, or find any images on your computer.
%matplotlib inline
import matplotlib.pyplot as plt
import sys,os,re,time
import urllib
import numpy as np
from IPython import parallel
from skimage.io import imread
from skimage import measure
Define a function to
def find_contours(path, low=0.1, high=0.8):
"""Find contours in an image at path
Returns the image and the contour lists.
"""
img = imread(path, flatten=True)
# Find contours at a constant value of 0.1 and 0.8
dark = measure.find_contours(img, low)
light = measure.find_contours(img, high)
return img, dark, light
def plot_contours(img, dark, light, show=True):
"""Display the image and plot all contours found"""
plt.imshow(img, cmap='gray')
for n, contour in enumerate(dark):
plt.plot(contour[:, 1], contour[:, 0], c='r', linewidth=1)
for n, contour in enumerate(light):
plt.plot(contour[:, 1], contour[:, 0], c='b', linewidth=1)
plt.axis('image')
plt.xticks([])
plt.yticks([])
if show:
plt.show()
def get_contours_image(path):
"""Given a path, return a PNG of the image with contour lines
Calls both find_contours and plot_contours
"""
from IPython.core.pylabtools import print_figure
img, dark, light = find_contours(path)
plot_contours(img, dark, light, show=False)
fig = plt.gcf()
pngdata = print_figure(fig)
plt.close(fig)
return pngdata
import os
pictures_dir = os.path.join('..', 'images', 'castle')
pictures = []
for directory, subdirs, files in os.walk(pictures_dir):
for fname in files:
if fname.lower().endswith(('.jpg', '.png')):
pictures.append(os.path.join(directory, fname))
Let's test our function locally, to see what it does.
for p in pictures[:3]:
img, dark, light = find_contours(p)
plot_contours(img, dark, light)
First, we connect our parallel Client
rc = parallel.Client()
all_engines = rc[:]
view = rc.load_balanced_view()
Then we initialize the namespace on all of the engines with imports
%px import os; os.chdir("{os.getcwd()}")
%%px
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from skimage.io import imread
from skimage import measure
and make sure some functions are defined everywhere (this is only necessary for the contours_in_url
case)
all_engines.push(dict(
plot_contours=plot_contours,
find_contours=find_contours,
))
Now we can iterate through all of our pictures, and detect and display any faces we find
from IPython.display import display, Image
amr = view.map_async(get_contours_image, pictures[:20], ordered=False)
for pngdata in amr:
display(Image(data=pngdata))