requests is a python library for dealing with web pages.
http://docs.python-requests.org/en/v2.0-0/user/quickstart/
import requests
from pattern import web
from BeautifulSoup import BeautifulSoup
url = 'http://www.imdb.com/search/title?sort=num_votes,desc&start=1&title_type=feature&year=1950,2015'
r = requests.get(url)
print r.url
http://www.imdb.com/search/title?sort=num_votes,desc&start=1&title_type=feature&year=1950,2015
params is way to specify the added features while getting a url
url = 'http://www.imdb.com/search/title'
params = dict(sort='num_votes,desc', start=1, title_type='feature', year='1950,2015')
r = requests.get(url, params=params)
print r.url # notice it constructs the full url for you
http://www.imdb.com/search/title?sort=num_votes%2Cdesc&start=1&title_type=feature&year=1950%2C2015
check this out
http://www.clips.ua.ac.be/pages/pattern-web
r.text has the source code for the entire webpage. dom will now have all the tags from the source code.
dom = web.Element(r.text)
for movie in dom.by_tag('td.title'):
title = movie.by_tag('a')[0].content #content of a tag is the stuff between the opening and closing tags
runtime = movie.by_tag('span.runtime')[0].content
rating = movie.by_tag('span.value')[0].content
genres = movie.by_tag('span.genre')[0].by_tag('a')
genre = [g.content for g in genres]
#could have as well done
#genre = []
#for g in genres:
# genre.append(g.content)
print title, runtime, rating, genre
The Shawshank Redemption 142 mins. 9.3 [u'Crime', u'Drama'] The Dark Knight 152 mins. 9.0 [u'Action', u'Crime', u'Drama'] Inception 148 mins. 8.8 [u'Action', u'Mystery', u'Sci-Fi', u'Thriller'] Fight Club 139 mins. 8.9 [u'Drama'] Pulp Fiction 154 mins. 8.9 [u'Crime', u'Drama'] The Lord of the Rings: The Fellowship of the Ring 178 mins. 8.8 [u'Adventure', u'Fantasy'] Forrest Gump 142 mins. 8.8 [u'Drama', u'Romance'] The Matrix 136 mins. 8.7 [u'Action', u'Sci-Fi'] The Lord of the Rings: The Return of the King 201 mins. 8.9 [u'Adventure', u'Fantasy'] The Godfather 175 mins. 9.2 [u'Crime', u'Drama'] The Dark Knight Rises 165 mins. 8.5 [u'Action', u'Thriller'] The Lord of the Rings: The Two Towers 179 mins. 8.7 [u'Adventure', u'Fantasy'] Se7en 127 mins. 8.6 [u'Crime', u'Drama', u'Mystery', u'Thriller'] The Avengers 143 mins. 8.1 [u'Action', u'Adventure', u'Sci-Fi', u'Thriller'] Gladiator 155 mins. 8.5 [u'Action', u'Drama'] Batman Begins 140 mins. 8.3 [u'Action', u'Adventure'] Django Unchained 165 mins. 8.5 [u'Western'] Avatar 162 mins. 7.9 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] The Silence of the Lambs 118 mins. 8.6 [u'Crime', u'Drama', u'Thriller'] Saving Private Ryan 169 mins. 8.6 [u'Action', u'Drama', u'War'] Star Wars 121 mins. 8.7 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] The Departed 151 mins. 8.5 [u'Crime', u'Drama', u'Thriller'] Schindler's List 195 mins. 8.9 [u'Biography', u'Drama', u'History'] Inglourious Basterds 153 mins. 8.3 [u'Adventure', u'Drama', u'War'] Memento 113 mins. 8.5 [u'Mystery', u'Thriller'] The Prestige 130 mins. 8.5 [u'Drama', u'Mystery', u'Thriller'] Interstellar 169 mins. 8.7 [u'Adventure', u'Drama', u'Sci-Fi'] American Beauty 122 mins. 8.4 [u'Drama', u'Romance'] Pirates of the Caribbean: The Curse of the Black Pearl 143 mins. 8.1 [u'Action', u'Adventure', u'Fantasy'] Titanic 194 mins. 7.7 [u'Drama', u'Romance'] Star Wars: Episode V - The Empire Strikes Back 124 mins. 8.8 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] V for Vendetta 132 mins. 8.2 [u'Action', u'Drama', u'Thriller'] American History X 119 mins. 8.6 [u'Crime', u'Drama'] The Godfather: Part II 200 mins. 9.0 [u'Crime', u'Drama'] The Green Mile 189 mins. 8.5 [u'Crime', u'Drama', u'Fantasy', u'Mystery'] Shutter Island 138 mins. 8.1 [u'Mystery', u'Thriller'] Terminator 2: Judgment Day 137 mins. 8.5 [u'Action', u'Sci-Fi'] The Usual Suspects 106 mins. 8.6 [u'Crime', u'Drama', u'Thriller'] Braveheart 178 mins. 8.4 [u'Biography', u'Drama', u'History', u'War'] Kill Bill: Vol. 1 111 mins. 8.1 [u'Action'] Goodfellas 146 mins. 8.7 [u'Biography', u'Crime', u'Drama'] The Wolf of Wall Street 180 mins. 8.2 [u'Biography', u'Comedy', u'Crime', u'Drama'] Léon: The Professional 110 mins. 8.6 [u'Crime', u'Drama', u'Thriller'] Back to the Future 116 mins. 8.5 [u'Adventure', u'Comedy', u'Sci-Fi'] The Hunger Games 142 mins. 7.3 [u'Adventure', u'Drama', u'Sci-Fi', u'Thriller'] The Sixth Sense 107 mins. 8.2 [u'Drama', u'Mystery', u'Thriller'] WALL·E 98 mins. 8.4 [u'Animation', u'Adventure', u'Family', u'Sci-Fi'] Iron Man 126 mins. 7.9 [u'Action', u'Adventure', u'Sci-Fi'] One Flew Over the Cuckoo's Nest 133 mins. 8.7 [u'Drama'] Finding Nemo 100 mins. 8.2 [u'Animation', u'Adventure', u'Comedy', u'Family']
Beautiful Soup is a Python library for pulling data out of HTML and XML files.
Check documentation here.
http://www.crummy.com/software/BeautifulSoup/bs4/doc/
bs = BeautifulSoup(r.text) #gives you the source code
for movie in bs.findAll('td','title'):
title = movie.find('a').contents[0] #use only find when you know you want only one value of a tag
runtime = movie.find('span','runtime').contents[0]
rating = movie.find('span','value').contents[0]
genres = movie.find('span','genre').findAll('a')
genre = [g.contents[0] for g in genres]
print title, runtime, rating, genre
#http://www.crummy.com/software/BeautifulSoup/bs4/doc/#calling-a-tag-is-like-calling-find-all
The Shawshank Redemption 142 mins. 9.3 [u'Crime', u'Drama'] The Dark Knight 152 mins. 9.0 [u'Action', u'Crime', u'Drama'] Inception 148 mins. 8.8 [u'Action', u'Mystery', u'Sci-Fi', u'Thriller'] Fight Club 139 mins. 8.9 [u'Drama'] Pulp Fiction 154 mins. 8.9 [u'Crime', u'Drama'] The Lord of the Rings: The Fellowship of the Ring 178 mins. 8.8 [u'Adventure', u'Fantasy'] Forrest Gump 142 mins. 8.8 [u'Drama', u'Romance'] The Matrix 136 mins. 8.7 [u'Action', u'Sci-Fi'] The Lord of the Rings: The Return of the King 201 mins. 8.9 [u'Adventure', u'Fantasy'] The Godfather 175 mins. 9.2 [u'Crime', u'Drama'] The Dark Knight Rises 165 mins. 8.5 [u'Action', u'Thriller'] The Lord of the Rings: The Two Towers 179 mins. 8.7 [u'Adventure', u'Fantasy'] Se7en 127 mins. 8.6 [u'Crime', u'Drama', u'Mystery', u'Thriller'] The Avengers 143 mins. 8.1 [u'Action', u'Adventure', u'Sci-Fi', u'Thriller'] Gladiator 155 mins. 8.5 [u'Action', u'Drama'] Batman Begins 140 mins. 8.3 [u'Action', u'Adventure'] Django Unchained 165 mins. 8.5 [u'Western'] Avatar 162 mins. 7.9 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] The Silence of the Lambs 118 mins. 8.6 [u'Crime', u'Drama', u'Thriller'] Saving Private Ryan 169 mins. 8.6 [u'Action', u'Drama', u'War'] Star Wars 121 mins. 8.7 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] The Departed 151 mins. 8.5 [u'Crime', u'Drama', u'Thriller'] Schindler's List 195 mins. 8.9 [u'Biography', u'Drama', u'History'] Inglourious Basterds 153 mins. 8.3 [u'Adventure', u'Drama', u'War'] Memento 113 mins. 8.5 [u'Mystery', u'Thriller'] The Prestige 130 mins. 8.5 [u'Drama', u'Mystery', u'Thriller'] Interstellar 169 mins. 8.7 [u'Adventure', u'Drama', u'Sci-Fi'] American Beauty 122 mins. 8.4 [u'Drama', u'Romance'] Pirates of the Caribbean: The Curse of the Black Pearl 143 mins. 8.1 [u'Action', u'Adventure', u'Fantasy'] Titanic 194 mins. 7.7 [u'Drama', u'Romance'] Star Wars: Episode V - The Empire Strikes Back 124 mins. 8.8 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] V for Vendetta 132 mins. 8.2 [u'Action', u'Drama', u'Thriller'] American History X 119 mins. 8.6 [u'Crime', u'Drama'] The Godfather: Part II 200 mins. 9.0 [u'Crime', u'Drama'] The Green Mile 189 mins. 8.5 [u'Crime', u'Drama', u'Fantasy', u'Mystery'] Shutter Island 138 mins. 8.1 [u'Mystery', u'Thriller'] Terminator 2: Judgment Day 137 mins. 8.5 [u'Action', u'Sci-Fi'] The Usual Suspects 106 mins. 8.6 [u'Crime', u'Drama', u'Thriller'] Braveheart 178 mins. 8.4 [u'Biography', u'Drama', u'History', u'War'] Kill Bill: Vol. 1 111 mins. 8.1 [u'Action'] Goodfellas 146 mins. 8.7 [u'Biography', u'Crime', u'Drama'] The Wolf of Wall Street 180 mins. 8.2 [u'Biography', u'Comedy', u'Crime', u'Drama'] Léon: The Professional 110 mins. 8.6 [u'Crime', u'Drama', u'Thriller'] Back to the Future 116 mins. 8.5 [u'Adventure', u'Comedy', u'Sci-Fi'] The Hunger Games 142 mins. 7.3 [u'Adventure', u'Drama', u'Sci-Fi', u'Thriller'] The Sixth Sense 107 mins. 8.2 [u'Drama', u'Mystery', u'Thriller'] WALL·E 98 mins. 8.4 [u'Animation', u'Adventure', u'Family', u'Sci-Fi'] Iron Man 126 mins. 7.9 [u'Action', u'Adventure', u'Sci-Fi'] One Flew Over the Cuckoo's Nest 133 mins. 8.7 [u'Drama'] Finding Nemo 100 mins. 8.2 [u'Animation', u'Adventure', u'Comedy', u'Family']
url = 'http://www.imdb.com/search/title'
for i in xrange(1,200,50):
params = dict(sort='num_votes,desc', start=i, title_type='feature', year='1950,2015')
r = requests.get(url, params=params)
dom = web.Element(r.text)
for movie in dom.by_tag('td.title'):
title = movie.by_tag('a')[0].content
year = movie.by_tag('span.year_type')[0].content
runtime = movie.by_tag('span.runtime')[0].content
rating = movie.by_tag('span.value')[0].content
genres = movie.by_tag('span.genre')[0].by_tag('a')
genre = [g.content for g in genres]
print title, year, runtime, rating, genre
The Shawshank Redemption (1994) 142 mins. 9.3 [u'Crime', u'Drama'] The Dark Knight (2008) 152 mins. 9.0 [u'Action', u'Crime', u'Drama'] Inception (2010) 148 mins. 8.8 [u'Action', u'Mystery', u'Sci-Fi', u'Thriller'] Fight Club (1999) 139 mins. 8.9 [u'Drama'] Pulp Fiction (1994) 154 mins. 8.9 [u'Crime', u'Drama'] The Lord of the Rings: The Fellowship of the Ring (2001) 178 mins. 8.8 [u'Adventure', u'Fantasy'] Forrest Gump (1994) 142 mins. 8.8 [u'Drama', u'Romance'] The Matrix (1999) 136 mins. 8.7 [u'Action', u'Sci-Fi'] The Lord of the Rings: The Return of the King (2003) 201 mins. 8.9 [u'Adventure', u'Fantasy'] The Godfather (1972) 175 mins. 9.2 [u'Crime', u'Drama'] The Dark Knight Rises (2012) 165 mins. 8.5 [u'Action', u'Thriller'] The Lord of the Rings: The Two Towers (2002) 179 mins. 8.7 [u'Adventure', u'Fantasy'] Se7en (1995) 127 mins. 8.6 [u'Crime', u'Drama', u'Mystery', u'Thriller'] The Avengers (2012) 143 mins. 8.1 [u'Action', u'Adventure', u'Sci-Fi', u'Thriller'] Gladiator (2000) 155 mins. 8.5 [u'Action', u'Drama'] Batman Begins (2005) 140 mins. 8.3 [u'Action', u'Adventure'] Django Unchained (2012) 165 mins. 8.5 [u'Western'] Avatar (2009) 162 mins. 7.9 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] The Silence of the Lambs (1991) 118 mins. 8.6 [u'Crime', u'Drama', u'Thriller'] Saving Private Ryan (1998) 169 mins. 8.6 [u'Action', u'Drama', u'War'] Star Wars (1977) 121 mins. 8.7 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] The Departed (2006) 151 mins. 8.5 [u'Crime', u'Drama', u'Thriller'] Schindler's List (1993) 195 mins. 8.9 [u'Biography', u'Drama', u'History'] Inglourious Basterds (2009) 153 mins. 8.3 [u'Adventure', u'Drama', u'War'] Memento (2000) 113 mins. 8.5 [u'Mystery', u'Thriller'] The Prestige (2006) 130 mins. 8.5 [u'Drama', u'Mystery', u'Thriller'] Interstellar (2014) 169 mins. 8.7 [u'Adventure', u'Drama', u'Sci-Fi'] American Beauty (1999) 122 mins. 8.4 [u'Drama', u'Romance'] Pirates of the Caribbean: The Curse of the Black Pearl (2003) 143 mins. 8.1 [u'Action', u'Adventure', u'Fantasy'] Titanic (1997) 194 mins. 7.7 [u'Drama', u'Romance'] Star Wars: Episode V - The Empire Strikes Back (1980) 124 mins. 8.8 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] V for Vendetta (2005) 132 mins. 8.2 [u'Action', u'Drama', u'Thriller'] American History X (1998) 119 mins. 8.6 [u'Crime', u'Drama'] The Godfather: Part II (1974) 200 mins. 9.0 [u'Crime', u'Drama'] The Green Mile (1999) 189 mins. 8.5 [u'Crime', u'Drama', u'Fantasy', u'Mystery'] Shutter Island (2010) 138 mins. 8.1 [u'Mystery', u'Thriller'] Terminator 2: Judgment Day (1991) 137 mins. 8.5 [u'Action', u'Sci-Fi'] The Usual Suspects (1995) 106 mins. 8.6 [u'Crime', u'Drama', u'Thriller'] Braveheart (1995) 178 mins. 8.4 [u'Biography', u'Drama', u'History', u'War'] Kill Bill: Vol. 1 (2003) 111 mins. 8.1 [u'Action'] Goodfellas (1990) 146 mins. 8.7 [u'Biography', u'Crime', u'Drama'] The Wolf of Wall Street (2013) 180 mins. 8.2 [u'Biography', u'Comedy', u'Crime', u'Drama'] Léon: The Professional (1994) 110 mins. 8.6 [u'Crime', u'Drama', u'Thriller'] Back to the Future (1985) 116 mins. 8.5 [u'Adventure', u'Comedy', u'Sci-Fi'] The Hunger Games (2012) 142 mins. 7.3 [u'Adventure', u'Drama', u'Sci-Fi', u'Thriller'] The Sixth Sense (1999) 107 mins. 8.2 [u'Drama', u'Mystery', u'Thriller'] WALL·E (2008) 98 mins. 8.4 [u'Animation', u'Adventure', u'Family', u'Sci-Fi'] Iron Man (2008) 126 mins. 7.9 [u'Action', u'Adventure', u'Sci-Fi'] One Flew Over the Cuckoo's Nest (1975) 133 mins. 8.7 [u'Drama'] Finding Nemo (2003) 100 mins. 8.2 [u'Animation', u'Adventure', u'Comedy', u'Family'] Sin City (2005) 124 mins. 8.1 [u'Crime', u'Thriller'] Eternal Sunshine of the Spotless Mind (2004) 108 mins. 8.4 [u'Drama', u'Romance', u'Sci-Fi'] The Truman Show (1998) 103 mins. 8.1 [u'Drama'] Raiders of the Lost Ark (1981) 115 mins. 8.6 [u'Action', u'Adventure'] Reservoir Dogs (1992) 99 mins. 8.4 [u'Crime', u'Drama'] Slumdog Millionaire (2008) 120 mins. 8.0 [u'Drama', u'Romance'] Up (2009) 96 mins. 8.3 [u'Animation', u'Adventure', u'Comedy', u'Family'] The Hobbit: An Unexpected Journey (2012) 169 mins. 8.0 [u'Adventure', u'Fantasy'] Star Wars: Episode VI - Return of the Jedi (1983) 134 mins. 8.4 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] The Lion King (1994) 89 mins. 8.5 [u'Animation', u'Adventure', u'Drama', u'Family', u'Musical'] 300 (2006) 117 mins. 7.8 [u'Action', u'Fantasy', u'War'] Guardians of the Galaxy (2014) 121 mins. 8.1 [u'Action', u'Adventure', u'Sci-Fi'] No Country for Old Men (2007) 122 mins. 8.1 [u'Crime', u'Drama', u'Thriller'] Toy Story (1995) 81 mins. 8.3 [u'Animation', u'Adventure', u'Comedy', u'Family', u'Fantasy'] Snatch. (2000) 104 mins. 8.3 [u'Comedy', u'Crime'] The Shining (1980) 146 mins. 8.4 [u'Drama', u'Horror'] A Beautiful Mind (2001) 135 mins. 8.2 [u'Biography', u'Drama'] Good Will Hunting (1997) 126 mins. 8.3 [u'Drama'] The Terminator (1984) 107 mins. 8.1 [u'Action', u'Sci-Fi'] The Hangover (2009) 100 mins. 7.8 [u'Comedy'] Die Hard (1988) 131 mins. 8.3 [u'Action', u'Thriller'] Jurassic Park (1993) 127 mins. 8.1 [u'Adventure', u'Sci-Fi', u'Thriller'] Donnie Darko (2001) 113 mins. 8.1 [u'Drama', u'Sci-Fi'] Gravity (2013) 91 mins. 7.9 [u'Sci-Fi', u'Thriller'] Requiem for a Dream (2000) 102 mins. 8.4 [u'Drama'] Monsters, Inc. (2001) 92 mins. 8.1 [u'Animation', u'Adventure', u'Comedy', u'Family', u'Fantasy'] Gran Torino (2008) 116 mins. 8.2 [u'Drama'] Black Swan (2010) 108 mins. 8.0 [u'Drama', u'Thriller'] Alien (1979) 117 mins. 8.5 [u'Horror', u'Sci-Fi'] A Clockwork Orange (1971) 136 mins. 8.4 [u'Crime', u'Drama', u'Sci-Fi'] Iron Man 3 (2013) 130 mins. 7.3 [u'Action', u'Adventure', u'Sci-Fi'] Spider-Man (2002) 121 mins. 7.3 [u'Action', u'Adventure'] District 9 (2009) 112 mins. 8.0 [u'Action', u'Sci-Fi'] Silver Linings Playbook (2012) 122 mins. 7.8 [u'Comedy', u'Drama', u'Romance'] Scarface (1983) 170 mins. 8.3 [u'Crime', u'Drama'] City of God (2002) 130 mins. 8.7 [u'Crime', u'Drama'] The Big Lebowski (1998) 117 mins. 8.2 [u'Comedy', u'Crime'] Amélie (2001) 122 mins. 8.4 [u'Comedy', u'Romance'] Toy Story 3 (2010) 103 mins. 8.4 [u'Animation', u'Adventure', u'Comedy', u'Family', u'Fantasy'] Man of Steel (2013) 143 mins. 7.2 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] I Am Legend (2007) 101 mins. 7.2 [u'Drama', u'Sci-Fi', u'Thriller'] Harry Potter and the Deathly Hallows: Part 2 (2011) 130 mins. 8.1 [u'Adventure', u'Drama', u'Fantasy', u'Mystery'] Transformers (2007) 144 mins. 7.1 [u'Action', u'Adventure', u'Sci-Fi'] Thor (2011) 115 mins. 7.0 [u'Action', u'Adventure', u'Fantasy'] Pirates of the Caribbean: Dead Man's Chest (2006) 151 mins. 7.3 [u'Action', u'Adventure', u'Fantasy'] Gone Girl (2014) 149 mins. 8.2 [u'Drama', u'Mystery', u'Thriller'] Star Trek (2009) 127 mins. 8.0 [u'Action', u'Adventure', u'Sci-Fi'] Kill Bill: Vol. 2 (2004) 137 mins. 8.0 [u'Action', u'Crime', u'Thriller'] X-Men: First Class (2011) 132 mins. 7.8 [u'Action', u'Adventure', u'Sci-Fi'] Indiana Jones and the Last Crusade (1989) 127 mins. 8.3 [u'Action', u'Adventure'] Iron Man 2 (2010) 124 mins. 7.0 [u'Action', u'Adventure', u'Sci-Fi'] Skyfall (2012) 143 mins. 7.8 [u'Action', u'Adventure', u'Thriller'] Star Wars: Episode I - The Phantom Menace (1999) 136 mins. 6.5 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] The Good, the Bad and the Ugly (1966) 161 mins. 8.9 [u'Western'] Taxi Driver (1976) 113 mins. 8.4 [u'Crime', u'Drama'] Taken (2008) 93 mins. 7.9 [u'Action', u'Thriller'] The King's Speech (2010) 118 mins. 8.1 [u'Biography', u'Drama'] The Hunger Games: Catching Fire (2013) 146 mins. 7.6 [u'Adventure', u'Sci-Fi', u'Thriller'] X-Men: Days of Future Past (2014) 132 mins. 8.1 [u'Action', u'Adventure', u'Sci-Fi', u'Thriller'] Aliens (1986) 137 mins. 8.4 [u'Action', u'Horror', u'Sci-Fi'] Star Wars: Episode III - Revenge of the Sith (2005) 140 mins. 7.7 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] The Pianist (2002) 150 mins. 8.5 [u'Biography', u'Drama', u'War'] Catch Me If You Can (2002) 141 mins. 8.0 [u'Biography', u'Crime', u'Drama'] The Bourne Ultimatum (2007) 115 mins. 8.1 [u'Action', u'Thriller'] Captain America: The First Avenger (2011) 124 mins. 6.8 [u'Action', u'Adventure', u'Sci-Fi'] Sherlock Holmes (2009) 128 mins. 7.6 [u'Action', u'Adventure', u'Crime', u'Mystery', u'Thriller'] Full Metal Jacket (1987) 116 mins. 8.3 [u'Drama', u'War'] Million Dollar Baby (2004) 132 mins. 8.1 [u'Drama', u'Sport'] The Hobbit: The Desolation of Smaug (2013) 161 mins. 7.9 [u'Adventure', u'Fantasy'] The Intouchables (2011) 112 mins. 8.6 [u'Biography', u'Comedy', u'Drama'] The Social Network (2010) 120 mins. 7.8 [u'Biography', u'Drama'] Ted (2012) 106 mins. 7.0 [u'Comedy', u'Fantasy'] The Incredibles (2004) 115 mins. 8.0 [u'Animation', u'Action', u'Adventure', u'Family'] Pirates of the Caribbean: At World's End (2007) 169 mins. 7.1 [u'Action', u'Adventure', u'Fantasy'] How to Train Your Dragon (2010) 98 mins. 8.2 [u'Animation', u'Adventure', u'Family', u'Fantasy'] Trainspotting (1996) 94 mins. 8.2 [u'Drama'] Ratatouille (2007) 111 mins. 8.0 [u'Animation', u'Comedy', u'Family', u'Fantasy'] Pan's Labyrinth (2006) 118 mins. 8.2 [u'Drama', u'Fantasy', u'War'] Prometheus (2012) 124 mins. 7.0 [u'Adventure', u'Mystery', u'Sci-Fi'] Shrek (2001) 90 mins. 7.9 [u'Animation', u'Adventure', u'Comedy', u'Family', u'Fantasy'] The Curious Case of Benjamin Button (2008) 166 mins. 7.8 [u'Drama', u'Fantasy', u'Romance'] Twelve Monkeys (1995) 129 mins. 8.1 [u'Mystery', u'Sci-Fi', u'Thriller'] Blade Runner (1982) 117 mins. 8.2 [u'Sci-Fi', u'Thriller'] World War Z (2013) 116 mins. 7.0 [u'Action', u'Adventure', u'Horror', u'Sci-Fi', u'Thriller'] Casino Royale (2006) 144 mins. 8.0 [u'Action', u'Adventure', u'Thriller'] The Amazing Spider-Man (2012) 136 mins. 7.1 [u'Action', u'Adventure', u'Fantasy'] Captain America: The Winter Soldier (2014) 136 mins. 7.8 [u'Action', u'Adventure', u'Sci-Fi'] Apocalypse Now (1979) 153 mins. 8.5 [u'Drama', u'War'] Argo (2012) 120 mins. 7.8 [u'Drama', u'History', u'Thriller'] X-Men (2000) 104 mins. 7.4 [u'Action', u'Adventure', u'Sci-Fi'] Fargo (1996) 98 mins. 8.2 [u'Crime', u'Drama', u'Thriller'] Kick-Ass (2010) 117 mins. 7.7 [u'Action', u'Comedy'] Harry Potter and the Sorcerer's Stone (2001) 152 mins. 7.5 [u'Adventure', u'Family', u'Fantasy'] Drive (2011) 100 mins. 7.8 [u'Crime', u'Drama'] 12 Angry Men (1957) 96 mins. 8.9 [u'Crime', u'Drama'] Heat (1995) 170 mins. 8.3 [u'Action', u'Crime', u'Drama', u'Thriller'] The Matrix Reloaded (2003) 138 mins. 7.2 [u'Action', u'Sci-Fi'] Life of Pi (2012) 127 mins. 8.0 [u'Adventure', u'Drama', u'Fantasy'] Superbad (2007) 113 mins. 7.6 [u'Comedy'] The Grand Budapest Hotel (2014) 99 mins. 8.1 [u'Adventure', u'Comedy', u'Drama'] Looper (2012) 119 mins. 7.5 [u'Action', u'Crime', u'Sci-Fi', u'Thriller'] Star Wars: Episode II - Attack of the Clones (2002) 142 mins. 6.7 [u'Action', u'Adventure', u'Fantasy', u'Sci-Fi'] Groundhog Day (1993) 101 mins. 8.1 [u'Comedy', u'Fantasy', u'Romance'] Independence Day (1996) 145 mins. 6.9 [u'Action', u'Adventure', u'Sci-Fi'] Now You See Me (2013) 115 mins. 7.3 [u'Crime', u'Mystery', u'Thriller'] Juno (2007) 96 mins. 7.5 [u'Comedy', u'Drama', u'Romance'] Into the Wild (2007) 148 mins. 8.2 [u'Adventure', u'Biography', u'Drama'] 2001: A Space Odyssey (1968) 160 mins. 8.3 [u'Mystery', u'Sci-Fi'] L.A. Confidential (1997) 138 mins. 8.3 [u'Crime', u'Drama', u'Mystery', u'Thriller'] Limitless (2011) 105 mins. 7.4 [u'Mystery', u'Sci-Fi', u'Thriller'] Psycho (1960) 109 mins. 8.6 [u'Horror', u'Mystery', u'Thriller'] Lock, Stock and Two Smoking Barrels (1998) 107 mins. 8.2 [u'Comedy', u'Crime'] 12 Years a Slave (2013) 134 mins. 8.1 [u'Biography', u'Drama', u'History'] Edge of Tomorrow (2014) 113 mins. 7.9 [u'Action', u'Adventure', u'Sci-Fi'] Jaws (1975) 124 mins. 8.1 [u'Adventure', u'Drama', u'Thriller'] Rise of the Planet of the Apes (2011) 105 mins. 7.6 [u'Action', u'Drama', u'Sci-Fi', u'Thriller'] Frozen (2013) 102 mins. 7.6 [u'Animation', u'Adventure', u'Comedy', u'Family', u'Fantasy', u'Musical'] 21 Jump Street (2012) 109 mins. 7.2 [u'Action', u'Comedy', u'Crime'] Minority Report (2002) 145 mins. 7.7 [u'Action', u'Mystery', u'Sci-Fi', u'Thriller'] Life Is Beautiful (1997) 116 mins. 8.6 [u'Comedy', u'Drama', u'Romance'] Ocean's Eleven (2001) 116 mins. 7.8 [u'Crime', u'Thriller'] The Bourne Identity (2002) 119 mins. 7.9 [u'Action', u'Mystery', u'Thriller'] Spirited Away (2001) 125 mins. 8.6 [u'Animation', u'Adventure', u'Family', u'Fantasy'] X2 (2003) 134 mins. 7.5 [u'Action', u'Adventure', u'Sci-Fi', u'Thriller'] Men in Black (1997) 98 mins. 7.2 [u'Comedy', u'Sci-Fi'] Spider-Man 2 (2004) 127 mins. 7.3 [u'Action', u'Adventure', u'Fantasy'] Blood Diamond (2006) 143 mins. 8.0 [u'Adventure', u'Drama', u'Thriller'] Shaun of the Dead (2004) 99 mins. 8.0 [u'Comedy', u'Horror'] The Notebook (2004) 123 mins. 7.9 [u'Drama', u'Romance'] Star Trek Into Darkness (2013) 132 mins. 7.8 [u'Action', u'Adventure', u'Sci-Fi'] Thor: The Dark World (2013) 112 mins. 7.1 [u'Action', u'Adventure', u'Fantasy'] Rain Man (1988) 133 mins. 8.0 [u'Drama'] The Imitation Game (2014) 114 mins. 8.1 [u'Biography', u'Drama', u'Thriller', u'War'] Mad Max: Fury Road (2015) 120 mins. 8.2 [u'Action', u'Adventure', u'Sci-Fi', u'Thriller'] Cast Away (2000) 143 mins. 7.7 [u'Adventure', u'Drama'] Watchmen (2009) 162 mins. 7.6 [u'Action', u'Mystery', u'Sci-Fi'] Zombieland (2009) 88 mins. 7.7 [u'Adventure', u'Comedy', u'Horror'] I, Robot (2004) 115 mins. 7.1 [u'Action', u'Mystery', u'Sci-Fi', u'Thriller'] Oblivion (2013) 124 mins. 7.0 [u'Action', u'Adventure', u'Mystery', u'Sci-Fi'] Harry Potter and the Chamber of Secrets (2002) 161 mins. 7.4 [u'Adventure', u'Family', u'Fantasy', u'Mystery'] Crazy, Stupid, Love. (2011) 118 mins. 7.4 [u'Comedy', u'Drama', u'Romance'] Harry Potter and the Goblet of Fire (2005) 157 mins. 7.6 [u'Adventure', u'Family', u'Fantasy', u'Mystery'] Monty Python and the Holy Grail (1975) 91 mins. 8.3 [u'Adventure', u'Comedy', u'Fantasy'] Toy Story 2 (1999) 92 mins. 7.9 [u'Animation', u'Adventure', u'Comedy', u'Family', u'Fantasy'] Troy (2004) 163 mins. 7.2 [u'Adventure'] Pacific Rim (2013) 131 mins. 7.0 [u'Action', u'Adventure', u'Sci-Fi'] X-Men: The Last Stand (2006) 104 mins. 6.8 [u'Action', u'Adventure', u'Sci-Fi', u'Thriller'] The Hangover Part II (2011) 102 mins. 6.5 [u'Comedy'] Despicable Me (2010) 95 mins. 7.7 [u'Animation', u'Comedy', u'Family'] (500) Days of Summer (2009) 95 mins. 7.8 [u'Comedy', u'Drama', u'Romance']
Now, we'll write this data into a file for further analysis.
imdb = open('imdb_top_200.txt','a')
url = 'http://www.imdb.com/search/title'
for i in xrange(1,200,50):
params = dict(sort='num_votes,desc', start=i, title_type='feature', year='1950,2015')
r = requests.get(url, params=params)
dom = web.Element(r.text)
for movie in dom.by_tag('td.title'):
title = movie.by_tag('a')[0].content
year = movie.by_tag('span.year_type')[0].content
runtime = movie.by_tag('span.runtime')[0].content
rating = movie.by_tag('span.value')[0].content
genres = movie.by_tag('span.genre')[0].by_tag('a')
genre = [g.content for g in genres]
imdb.write(title+'\t'+year+'\t'+str(rating) +'\t'+str(runtime)+'\t'+str(genre)+'\n')
imdb.close()
This pretty much gives me the top 200 movies but there are some redundant movies being written at the top of the file.
Will have to look into that.