#!/usr/bin/env python
# coding: utf-8

# > This is one of the 100 recipes of the [IPython Cookbook](http://ipython-books.github.io/), the definitive guide to high-performance scientific computing and data science in Python.
# 

# # 4.9. Processing huge NumPy arrays with memory mapping

# In[ ]:


import numpy as np


# ## Writing a memory-mapped array

# We create a memory-mapped array with a specific shape.

# In[ ]:


nrows, ncols = 1000000, 100


# In[ ]:


f = np.memmap('memmapped.dat', dtype=np.float32, 
              mode='w+', shape=(nrows, ncols))


# Let's feed the array with random values, one column at a time because our system memory is limited!

# In[ ]:


for i in range(ncols):
    f[:,i] = np.random.rand(nrows)


# We save the last column of the array.

# In[ ]:


x = f[:,-1]


# Now, we flush memory changes to disk by removing the object.

# In[ ]:


del f


# ## Reading a memory-mapped file

# Reading a memory-mapped array from disk involves the same memmap function but with a different file mode. The data type and the shape need to be specified again, as this information is not stored in the file.

# In[ ]:


f = np.memmap('memmapped.dat', dtype=np.float32, shape=(nrows, ncols))


# In[ ]:


np.array_equal(f[:,-1], x)


# In[ ]:


del f


# > You'll find all the explanations, figures, references, and much more in the book (to be released later this summer).
# 
# > [IPython Cookbook](http://ipython-books.github.io/), by [Cyrille Rossant](http://cyrille.rossant.net), Packt Publishing, 2014 (500 pages).