import urllib

html_str = urllib.urlopen("https://raw.githubusercontent.com/ledeprogram/courses/master/databases/data/kittens.html").read()

print html_str

from bs4 import BeautifulSoup

document = BeautifulSoup(html_str)
print type(document)

h1_tag = document.find('h1')
print type(h1_tag)

print h1_tag.string

img_tag = document.find('img')
print img_tag['src']

h2_tags = document.find_all('h2')
print type(h2_tags)
[tag.string for tag in h2_tags]

checkup_tags = document.find_all('span', attrs={'class': 'lastcheckup'})
[tag.string for tag in checkup_tags]

kitten_tags = document.find_all("div", attrs={"class": "kitten"})

for kitten_tag in kitten_tags:
    h2_tag = kitten_tag.find('h2')
    print h2_tag.string

for kitten_tag in kitten_tags:
    h2_tag = kitten_tag.find('h2')
    a_tags = kitten_tag.find_all('a')
    a_tag_strings = [tag.string for tag in a_tags]
    a_tag_strings_joined = ", ".join(a_tag_strings)
    print h2_tag.string + ": " + a_tag_strings_joined

cheese_html = """
<h2>Camembert</h2>
<p>A soft cheese made in the Camembert region of France.</p>

<h2>Cheddar</h2>
<p>A yellow cheese made in the Cheddar region of... France, probably, idk whatevs.</p>
"""

document = BeautifulSoup(cheese_html)
cheese_dict = {}
for h2_tag in document.find_all('h2'):
    cheese_name = h2_tag.string
    cheese_desc_tag = h2_tag.find_next_sibling('p')
    cheese_dict[cheese_name] = cheese_desc_tag.string

cheese_dict

footer_tag = document.find("footer")

print footer_tag

footer_tag.find("p")

footer_tag['title']

footer_tags = document.find_all("footer")
print footer_tags

print footer_tags[0].string

html_str = urllib.urlopen("http://www.journalism.columbia.edu/page/10/10?category_ids%5B%5D=2&category_ids%5B%5D=3&category_ids%5B%5D=37").read()
document = BeautifulSoup(html_str)
faculty_list = []
for faculty_tag in document.find_all('li'):
    # create empty dictionary to store this faculty member
    faculty_dict = {}
    # faculty name
    h4_tag = faculty_tag.find('h4')
    a_tag = h4_tag.find('a')
    faculty_dict['name'] = a_tag.string
    # image URL
    img_tag = faculty_tag.find('img')
    faculty_dict['img_src'] = img_tag['src']
    # title
    p_tag = faculty_tag.find('p', attrs={'class': 'description'})
    faculty_dict['title'] = p_tag.string
    # append to list
    faculty_list.append(faculty_dict)
faculty_list

html_str = urllib.urlopen("http://www.journalism.columbia.edu/page/10/10?category_ids%5B%5D=2&category_ids%5B%5D=3&category_ids%5B%5D=37").read()
document = BeautifulSoup(html_str)
for faculty_tag in document.find_all('li')[:5]:
    print faculty_tag

experts_ul_tag = document.find('ul', attrs={'class': 'experts-list'})
for faculty_tag in experts_ul_tag.find_all('li')[:5]:
    print faculty_tag

experts_ul_tag = document.find('ul', attrs={'class': 'experts-list'})
for faculty_tag in experts_ul_tag.find_all('li')[:5]:
    h4_tag = faculty_tag.find('h4')
    if h4_tag is None:
        continue
    print h4_tag

html_str = urllib.urlopen("http://www.journalism.columbia.edu/page/10/10?category_ids%5B%5D=2&category_ids%5B%5D=3&category_ids%5B%5D=37").read()
document = BeautifulSoup(html_str)
faculty_list = []
experts_ul_tag = document.find('ul', attrs={'class': 'experts-list'})
for faculty_tag in experts_ul_tag.find_all('li'):
    # create empty dictionary to store this faculty member
    faculty_dict = {}
    # faculty name
    h4_tag = faculty_tag.find('h4')
    if h4_tag is None:
        continue
    a_tag = h4_tag.find('a')
    faculty_dict['name'] = a_tag.string
    # image URL
    img_tag = faculty_tag.find('img')
    faculty_dict['img_src'] = img_tag['src']
    # title
    p_tag = faculty_tag.find('p', attrs={'class': 'description'})
    faculty_dict['title'] = p_tag.string
    # append to list
    faculty_list.append(faculty_dict)
faculty_list

html_str = urllib.urlopen("http://www.journalism.columbia.edu/page/10/10?category_ids%5B%5D=2&category_ids%5B%5D=3&category_ids%5B%5D=37").read()
document = BeautifulSoup(html_str)
faculty_list = []
experts_ul_tag = document.find('ul', attrs={'class': 'experts-list'})
for faculty_tag in experts_ul_tag.find_all('li'):
    # create empty dictionary to store this faculty member
    faculty_dict = {}
    # faculty name
    h4_tag = faculty_tag.find('h4')
    if h4_tag is None:
        continue
    a_tag = h4_tag.find('a')
    faculty_dict['name'] = a_tag.string
    # image URL: if <img> tag found, grab its src. if not, use None
    img_tag = faculty_tag.find('img')
    if img_tag is None:
        faculty_dict['img_src'] = None
    else:
        faculty_dict['img_src'] = img_tag['src']
    # title
    p_tag = faculty_tag.find('p', attrs={'class': 'description'})
    faculty_dict['title'] = p_tag.string
    # append to list
    faculty_list.append(faculty_dict)
faculty_list

import pandas as pd

faculty_frame = pd.DataFrame(faculty_list)
faculty_frame[faculty_frame["title"]=="Adjunct Faculty"]

import feedparser

!sudo pip install feedparser

import feedparser

feed = feedparser.parse("http://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml")
print type(feed.entries)

feed.entries[0]

[article['title'] for article in feed.entries]