#!/usr/bin/env python
# coding: utf-8

# In[1]:


import numpy as np
import pandas as pd


# In[2]:


from pandas import Series, DataFrame


# In[3]:


import numpy as np
np.random.seed(12345)
import matplotlib.pyplot as plt
plt.rc("figure", figsize=(10, 6))
PREVIOUS_MAX_ROWS = pd.options.display.max_rows
pd.options.display.max_rows = 20
pd.options.display.max_columns = 20
pd.options.display.max_colwidth = 80
np.set_printoptions(precision=4, suppress=True)


# In[4]:


obj = pd.Series([4, 7, -5, 3])
obj


# In[5]:


obj.array
obj.index


# In[6]:


obj2 = pd.Series([4, 7, -5, 3], index=["d", "b", "a", "c"])
obj2
obj2.index


# In[7]:


obj2["a"]
obj2["d"] = 6
obj2[["c", "a", "d"]]


# In[8]:


obj2[obj2 > 0]
obj2 * 2
import numpy as np
np.exp(obj2)


# In[9]:


"b" in obj2
"e" in obj2


# In[10]:


sdata = {"Ohio": 35000, "Texas": 71000, "Oregon": 16000, "Utah": 5000}
obj3 = pd.Series(sdata)
obj3


# In[11]:


obj3.to_dict()


# In[12]:


states = ["California", "Ohio", "Oregon", "Texas"]
obj4 = pd.Series(sdata, index=states)
obj4


# In[13]:


pd.isna(obj4)
pd.notna(obj4)


# In[14]:


obj4.isna()


# In[15]:


obj3
obj4
obj3 + obj4


# In[16]:


obj4.name = "population"
obj4.index.name = "state"
obj4


# In[17]:


obj
obj.index = ["Bob", "Steve", "Jeff", "Ryan"]
obj


# In[18]:


data = {"state": ["Ohio", "Ohio", "Ohio", "Nevada", "Nevada", "Nevada"],
        "year": [2000, 2001, 2002, 2001, 2002, 2003],
        "pop": [1.5, 1.7, 3.6, 2.4, 2.9, 3.2]}
frame = pd.DataFrame(data)


# In[19]:


frame


# In[20]:


frame.head()


# In[21]:


frame.tail()


# In[22]:


pd.DataFrame(data, columns=["year", "state", "pop"])


# In[23]:


frame2 = pd.DataFrame(data, columns=["year", "state", "pop", "debt"])
frame2
frame2.columns


# In[24]:


frame2["state"]
frame2.year


# In[25]:


frame2.loc[1]
frame2.iloc[2]


# In[26]:


frame2["debt"] = 16.5
frame2
frame2["debt"] = np.arange(6.)
frame2


# In[27]:


val = pd.Series([-1.2, -1.5, -1.7], index=["two", "four", "five"])
frame2["debt"] = val
frame2


# In[28]:


frame2["eastern"] = frame2["state"] == "Ohio"
frame2


# In[29]:


del frame2["eastern"]
frame2.columns


# In[30]:


populations = {"Ohio": {2000: 1.5, 2001: 1.7, 2002: 3.6},
               "Nevada": {2001: 2.4, 2002: 2.9}}


# In[31]:


frame3 = pd.DataFrame(populations)
frame3


# In[32]:


frame3.T


# In[33]:


pd.DataFrame(populations, index=[2001, 2002, 2003])


# In[34]:


pdata = {"Ohio": frame3["Ohio"][:-1],
         "Nevada": frame3["Nevada"][:2]}
pd.DataFrame(pdata)


# In[35]:


frame3.index.name = "year"
frame3.columns.name = "state"
frame3


# In[36]:


frame3.to_numpy()


# In[37]:


frame2.to_numpy()


# In[38]:


obj = pd.Series(np.arange(3), index=["a", "b", "c"])
index = obj.index
index
index[1:]


# In[39]:


labels = pd.Index(np.arange(3))
labels
obj2 = pd.Series([1.5, -2.5, 0], index=labels)
obj2
obj2.index is labels


# In[40]:


frame3
frame3.columns
"Ohio" in frame3.columns
2003 in frame3.index


# In[41]:


pd.Index(["foo", "foo", "bar", "bar"])


# In[42]:


obj = pd.Series([4.5, 7.2, -5.3, 3.6], index=["d", "b", "a", "c"])
obj


# In[43]:


obj2 = obj.reindex(["a", "b", "c", "d", "e"])
obj2


# In[44]:


obj3 = pd.Series(["blue", "purple", "yellow"], index=[0, 2, 4])
obj3
obj3.reindex(np.arange(6), method="ffill")


# In[45]:


frame = pd.DataFrame(np.arange(9).reshape((3, 3)),
                     index=["a", "c", "d"],
                     columns=["Ohio", "Texas", "California"])
frame
frame2 = frame.reindex(index=["a", "b", "c", "d"])
frame2


# In[46]:


states = ["Texas", "Utah", "California"]
frame.reindex(columns=states)


# In[47]:


frame.reindex(states, axis="columns")


# In[48]:


frame.loc[["a", "d", "c"], ["California", "Texas"]]


# In[49]:


obj = pd.Series(np.arange(5.), index=["a", "b", "c", "d", "e"])
obj
new_obj = obj.drop("c")
new_obj
obj.drop(["d", "c"])


# In[50]:


data = pd.DataFrame(np.arange(16).reshape((4, 4)),
                    index=["Ohio", "Colorado", "Utah", "New York"],
                    columns=["one", "two", "three", "four"])
data


# In[51]:


data.drop(index=["Colorado", "Ohio"])


# In[52]:


data.drop(columns=["two"])


# In[53]:


data.drop("two", axis=1)
data.drop(["two", "four"], axis="columns")


# In[54]:


obj = pd.Series(np.arange(4.), index=["a", "b", "c", "d"])
obj
obj["b"]
obj[1]
obj[2:4]
obj[["b", "a", "d"]]
obj[[1, 3]]
obj[obj < 2]


# In[55]:


obj.loc[["b", "a", "d"]]


# In[56]:


obj1 = pd.Series([1, 2, 3], index=[2, 0, 1])
obj2 = pd.Series([1, 2, 3], index=["a", "b", "c"])
obj1
obj2
obj1[[0, 1, 2]]
obj2[[0, 1, 2]]


# In[57]:


obj1.iloc[[0, 1, 2]]
obj2.iloc[[0, 1, 2]]


# In[58]:


obj2.loc["b":"c"]


# In[59]:


obj2.loc["b":"c"] = 5
obj2


# In[60]:


data = pd.DataFrame(np.arange(16).reshape((4, 4)),
                    index=["Ohio", "Colorado", "Utah", "New York"],
                    columns=["one", "two", "three", "four"])
data
data["two"]
data[["three", "one"]]


# In[61]:


data[:2]
data[data["three"] > 5]


# In[62]:


data < 5


# In[63]:


data[data < 5] = 0
data


# In[64]:


data
data.loc["Colorado"]


# In[65]:


data.loc[["Colorado", "New York"]]


# In[66]:


data.loc["Colorado", ["two", "three"]]


# In[67]:


data.iloc[2]
data.iloc[[2, 1]]
data.iloc[2, [3, 0, 1]]
data.iloc[[1, 2], [3, 0, 1]]


# In[68]:


data.loc[:"Utah", "two"]
data.iloc[:, :3][data.three > 5]


# In[69]:


data.loc[data.three >= 2]


# In[70]:


ser = pd.Series(np.arange(3.))
ser
ser[-1]


# In[71]:


ser


# In[72]:


ser2 = pd.Series(np.arange(3.), index=["a", "b", "c"])
ser2[-1]


# In[73]:


ser.iloc[-1]


# In[74]:


ser[:2]


# In[75]:


data.loc[:, "one"] = 1
data
data.iloc[2] = 5
data
data.loc[data["four"] > 5] = 3
data


# In[76]:


data.loc[data.three == 5]["three"] = 6


# In[77]:


data


# In[78]:


data.loc[data.three == 5, "three"] = 6
data


# In[79]:


s1 = pd.Series([7.3, -2.5, 3.4, 1.5], index=["a", "c", "d", "e"])
s2 = pd.Series([-2.1, 3.6, -1.5, 4, 3.1],
               index=["a", "c", "e", "f", "g"])
s1
s2


# In[80]:


s1 + s2


# In[81]:


df1 = pd.DataFrame(np.arange(9.).reshape((3, 3)), columns=list("bcd"),
                   index=["Ohio", "Texas", "Colorado"])
df2 = pd.DataFrame(np.arange(12.).reshape((4, 3)), columns=list("bde"),
                   index=["Utah", "Ohio", "Texas", "Oregon"])
df1
df2


# In[82]:


df1 + df2


# In[83]:


df1 = pd.DataFrame({"A": [1, 2]})
df2 = pd.DataFrame({"B": [3, 4]})
df1
df2
df1 + df2


# In[84]:


df1 = pd.DataFrame(np.arange(12.).reshape((3, 4)),
                   columns=list("abcd"))
df2 = pd.DataFrame(np.arange(20.).reshape((4, 5)),
                   columns=list("abcde"))
df2.loc[1, "b"] = np.nan
df1
df2


# In[85]:


df1 + df2


# In[86]:


df1.add(df2, fill_value=0)


# In[87]:


1 / df1
df1.rdiv(1)


# In[88]:


df1.reindex(columns=df2.columns, fill_value=0)


# In[89]:


arr = np.arange(12.).reshape((3, 4))
arr
arr[0]
arr - arr[0]


# In[90]:


frame = pd.DataFrame(np.arange(12.).reshape((4, 3)),
                     columns=list("bde"),
                     index=["Utah", "Ohio", "Texas", "Oregon"])
series = frame.iloc[0]
frame
series


# In[91]:


frame - series


# In[92]:


series2 = pd.Series(np.arange(3), index=["b", "e", "f"])
series2
frame + series2


# In[93]:


series3 = frame["d"]
frame
series3
frame.sub(series3, axis="index")


# In[94]:


frame = pd.DataFrame(np.random.standard_normal((4, 3)),
                     columns=list("bde"),
                     index=["Utah", "Ohio", "Texas", "Oregon"])
frame
np.abs(frame)


# In[95]:


def f1(x):
    return x.max() - x.min()

frame.apply(f1)


# In[96]:


frame.apply(f1, axis="columns")


# In[97]:


def f2(x):
    return pd.Series([x.min(), x.max()], index=["min", "max"])
frame.apply(f2)


# In[98]:


def my_format(x):
    return f"{x:.2f}"

frame.applymap(my_format)


# In[99]:


frame["e"].map(my_format)


# In[100]:


obj = pd.Series(np.arange(4), index=["d", "a", "b", "c"])
obj
obj.sort_index()


# In[101]:


frame = pd.DataFrame(np.arange(8).reshape((2, 4)),
                     index=["three", "one"],
                     columns=["d", "a", "b", "c"])
frame
frame.sort_index()
frame.sort_index(axis="columns")


# In[102]:


frame.sort_index(axis="columns", ascending=False)


# In[103]:


obj = pd.Series([4, 7, -3, 2])
obj.sort_values()


# In[104]:


obj = pd.Series([4, np.nan, 7, np.nan, -3, 2])
obj.sort_values()


# In[105]:


obj.sort_values(na_position="first")


# In[106]:


frame = pd.DataFrame({"b": [4, 7, -3, 2], "a": [0, 1, 0, 1]})
frame
frame.sort_values("b")


# In[107]:


frame.sort_values(["a", "b"])


# In[108]:


obj = pd.Series([7, -5, 7, 4, 2, 0, 4])
obj.rank()


# In[109]:


obj.rank(method="first")


# In[110]:


obj.rank(ascending=False)


# In[111]:


frame = pd.DataFrame({"b": [4.3, 7, -3, 2], "a": [0, 1, 0, 1],
                      "c": [-2, 5, 8, -2.5]})
frame
frame.rank(axis="columns")


# In[112]:


obj = pd.Series(np.arange(5), index=["a", "a", "b", "b", "c"])
obj


# In[113]:


obj.index.is_unique


# In[114]:


obj["a"]
obj["c"]


# In[115]:


df = pd.DataFrame(np.random.standard_normal((5, 3)),
                  index=["a", "a", "b", "b", "c"])
df
df.loc["b"]
df.loc["c"]


# In[116]:


df = pd.DataFrame([[1.4, np.nan], [7.1, -4.5],
                   [np.nan, np.nan], [0.75, -1.3]],
                  index=["a", "b", "c", "d"],
                  columns=["one", "two"])
df


# In[117]:


df.sum()


# In[118]:


df.sum(axis="columns")


# In[119]:


df.sum(axis="index", skipna=False)
df.sum(axis="columns", skipna=False)


# In[120]:


df.mean(axis="columns")


# In[121]:


df.idxmax()


# In[122]:


df.cumsum()


# In[123]:


df.describe()


# In[124]:


obj = pd.Series(["a", "a", "b", "c"] * 4)
obj.describe()


# In[125]:


price = pd.read_pickle("examples/yahoo_price.pkl")
volume = pd.read_pickle("examples/yahoo_volume.pkl")


# In[126]:


returns = price.pct_change()
returns.tail()


# In[127]:


returns["MSFT"].corr(returns["IBM"])
returns["MSFT"].cov(returns["IBM"])


# In[128]:


returns.corr()
returns.cov()


# In[129]:


returns.corrwith(returns["IBM"])


# In[130]:


returns.corrwith(volume)


# In[131]:


obj = pd.Series(["c", "a", "d", "a", "a", "b", "b", "c", "c"])


# In[132]:


uniques = obj.unique()
uniques


# In[133]:


obj.value_counts()


# In[134]:


pd.value_counts(obj.to_numpy(), sort=False)


# In[135]:


obj
mask = obj.isin(["b", "c"])
mask
obj[mask]


# In[136]:


to_match = pd.Series(["c", "a", "b", "b", "c", "a"])
unique_vals = pd.Series(["c", "b", "a"])
indices = pd.Index(unique_vals).get_indexer(to_match)
indices


# In[137]:


data = pd.DataFrame({"Qu1": [1, 3, 4, 3, 4],
                     "Qu2": [2, 3, 1, 2, 3],
                     "Qu3": [1, 5, 2, 4, 4]})
data


# In[138]:


data["Qu1"].value_counts().sort_index()


# In[139]:


result = data.apply(pd.value_counts).fillna(0)
result


# In[140]:


data = pd.DataFrame({"a": [1, 1, 1, 2, 2], "b": [0, 0, 1, 0, 0]})
data
data.value_counts()


# In[141]:





# In[142]:


pd.options.display.max_rows = PREVIOUS_MAX_ROWS