from IPython.display import YouTubeVideo
# a talk about IPython at Sage Days at U. Washington, Seattle.
# Video credit: William Stein.
YouTubeVideo('http://youtu.be/HuDIbSCnsqo')
One of the most common ways to analyze is to split it in to words and count how many times each word was used in a text. This may not sound very useful, but with large amounts of text you can build a fairly accurate classifier and this is what we will do in this tutorial.
import pandas as pd
import lxml
import requests
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.cross_validation import train_test_split
from sklearn.naive_bayes import MultinomialNB
import warnings
warnings.filterwarnings("ignore")
url_list = ["http://feeds.reuters.com/reuters/businessNews",
"http://feeds.reuters.com/reuters/technologyNews",
"http://feeds.reuters.com/reuters/sportsNews",
]
documents = []
for url in url_list:
response = requests.get(url)
xml_page = response.text
parser = lxml.etree.XMLParser(recover=True, encoding='utf-8')
documents.append(lxml.etree.fromstring(xml_page.encode("utf-8"), parser=parser))
def print_tag(node):
print "<%s %s>%s" % (node.tag, " ".join(["%s=%s" % (k,v)for k,v in node.attrib.iteritems()]), node.text)
for item in node[:25]:
print " <%s %s>%s</%s>" % (item.tag, " ".join(["%s=%s" % (k,v)for k,v in item.attrib.iteritems()]), item.text, item.tag)
print "</%s>" % node.tag
temp_node = documents[0]
print_tag(temp_node)
<rss version=2.0>None <channel >None</channel> </rss>
temp_node = temp_node[0]
print_tag(temp_node)
<channel >None <title >Reuters: Business News</title> <link >http://www.reuters.com</link> <description >Reuters.com is your source for breaking news, business, financial and investing news, including personal finance and stocks. Reuters is the leading global provider of news, financial information and technology solutions to the world's media, financial institutions, businesses and individuals.</description> <language >en-us</language> <copyright >All rights reserved. Users may download and print extracts of content from this website for their own personal and non-commercial use only. Republication or redistribution of Reuters content, including by framing or similar means, is expressly prohibited without the prior written consent of Reuters. Reuters and the Reuters sphere logo are registered trademarks or trademarks of the Reuters group of companies around the world. © Reuters 2015</copyright> <pubDate >Sat, 07 Feb 2015 14:34:41 GMT</pubDate> <lastBuildDate >Sat, 07 Feb 2015 14:34:41 GMT</lastBuildDate> <ttl >5</ttl> <image >None</image> <{http://www.w3.org/2005/Atom}link rel=self type=application/rss+xml href=http://feeds.reuters.com/reuters/businessNews>None</{http://www.w3.org/2005/Atom}link> <{http://rssnamespace.org/feedburner/ext/1.0}info uri=reuters/businessnews>None</{http://rssnamespace.org/feedburner/ext/1.0}info> <{http://www.w3.org/2005/Atom}link rel=hub href=http://pubsubhubbub.appspot.com/>None</{http://www.w3.org/2005/Atom}link> <{http://rssnamespace.org/feedburner/ext/1.0}feedFlare href=http://add.my.yahoo.com/rss?url=http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FbusinessNews src=http://us.i1.yimg.com/us.yimg.com/i/us/my/addtomyyahoo4.gif>Subscribe with My Yahoo!</{http://rssnamespace.org/feedburner/ext/1.0}feedFlare> <{http://rssnamespace.org/feedburner/ext/1.0}feedFlare href=http://www.newsgator.com/ngs/subscriber/subext.aspx?url=http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FbusinessNews src=http://www.newsgator.com/images/ngsub1.gif>Subscribe with NewsGator</{http://rssnamespace.org/feedburner/ext/1.0}feedFlare> <{http://rssnamespace.org/feedburner/ext/1.0}feedFlare href=http://feeds.my.aol.com/add.jsp?url=http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FbusinessNews src=http://o.aolcdn.com/favorites.my.aol.com/webmaster/ffclient/webroot/locale/en-US/images/myAOLButtonSmall.gif>Subscribe with My AOL</{http://rssnamespace.org/feedburner/ext/1.0}feedFlare> <{http://rssnamespace.org/feedburner/ext/1.0}feedFlare href=http://www.bloglines.com/sub/http://feeds.reuters.com/reuters/businessNews src=http://www.bloglines.com/images/sub_modern11.gif>Subscribe with Bloglines</{http://rssnamespace.org/feedburner/ext/1.0}feedFlare> <{http://rssnamespace.org/feedburner/ext/1.0}feedFlare href=http://www.netvibes.com/subscribe.php?url=http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FbusinessNews src=http://www.netvibes.com/img/add2netvibes.gif>Subscribe with Netvibes</{http://rssnamespace.org/feedburner/ext/1.0}feedFlare> <{http://rssnamespace.org/feedburner/ext/1.0}feedFlare href=http://fusion.google.com/add?feedurl=http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FbusinessNews src=http://buttons.googlesyndication.com/fusion/add.gif>Subscribe with Google</{http://rssnamespace.org/feedburner/ext/1.0}feedFlare> <{http://rssnamespace.org/feedburner/ext/1.0}feedFlare href=http://www.pageflakes.com/subscribe.aspx?url=http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FbusinessNews src=http://www.pageflakes.com/ImageFile.ashx?instanceId=Static_4&fileName=ATP_blu_91x17.gif>Subscribe with Pageflakes</{http://rssnamespace.org/feedburner/ext/1.0}feedFlare> <item >None</item> <item >None</item> <item >None</item> <item >None</item> <item >None</item> <item >None</item> </channel>
temp_node = temp_node.xpath("item")[0]
print_tag(temp_node)
<item >None <title >Exclusive: Top Fox investors seek to convert voting shares, Murdoch may benefit</title> <link >http://feeds.reuters.com/~r/reuters/businessNews/~3/Ti_4toCmS6w/story01.htm</link> <description >NEW YORK (Reuters) - Several top investors in Twenty-First Century Fox Inc are pressing for the right to swap their voting shares for ordinary shares, which are trading at an unusual premium, even though the move could hand even more control of the company to Rupert Murdoch, according to people familiar with the matter.<img width='1' height='1' src='http://reuters.us.feedsportal.com/c/35217/f/654199/s/4325e2cf/sc/2/mf.gif' border='0'/><br clear='all'/><div class="feedflare"> <a href="http://feeds.reuters.com/~ff/reuters/businessNews?a=Ti_4toCmS6w:UwKWzqPQZuQ:yIl2AUoC8zA"><img src="http://feeds.feedburner.com/~ff/reuters/businessNews?d=yIl2AUoC8zA" border="0"></img></a> <a href="http://feeds.reuters.com/~ff/reuters/businessNews?a=Ti_4toCmS6w:UwKWzqPQZuQ:F7zBnMyn0Lo"><img src="http://feeds.feedburner.com/~ff/reuters/businessNews?i=Ti_4toCmS6w:UwKWzqPQZuQ:F7zBnMyn0Lo" border="0"></img></a> <a href="http://feeds.reuters.com/~ff/reuters/businessNews?a=Ti_4toCmS6w:UwKWzqPQZuQ:V_sGLiPBpWU"><img src="http://feeds.feedburner.com/~ff/reuters/businessNews?i=Ti_4toCmS6w:UwKWzqPQZuQ:V_sGLiPBpWU" border="0"></img></a> </div><img src="//feeds.feedburner.com/~r/reuters/businessNews/~4/Ti_4toCmS6w" height="1" width="1" alt=""/></description> <category domain=>businessNews</category> <pubDate >Sat, 07 Feb 2015 14:30:10 GMT</pubDate> <guid isPermaLink=false>http://www.reuters.com/article/2015/02/07/us-fox-votingshares-exclusive-idUSKBN0LA2HP20150207?feedType=RSS&feedName=businessNews</guid> <{http://rssnamespace.org/feedburner/ext/1.0}origLink >http://reuters.us.feedsportal.com/c/35217/f/654199/s/4325e2cf/sc/2/l/0L0Sreuters0N0Carticle0C20A150C0A20C0A70Cus0Efox0Evotingshares0Eexclusive0EidUSKBN0ALA2HP20A150A20A70DfeedType0FRSS0GfeedName0FbusinessNews/story01.htm</{http://rssnamespace.org/feedburner/ext/1.0}origLink> </item>
title_list = []
description_list = []
category_list = []
for xml_doc in documents:
articles = xml_doc.xpath("//item")
for article in articles:
title_list.append(article[0].text)
description_list.append(article[2].text)
category_list.append(article[3].text)
news_data = pd.DataFrame(title_list, columns=["title"])
news_data["description"] = description_list
news_data["category"] = category_list
print len(news_data)
news_data
60
title | description | category | |
---|---|---|---|
0 | Exclusive: Top Fox investors seek to convert v... | NEW YORK (Reuters) - Several top investors in ... | businessNews |
1 | Different delivery, one message to Greece's ne... | ROME/PARIS (Reuters) - In Paris and Rome, it w... | businessNews |
2 | Isolated Greece wants no more bailout money wi... | ATHENS/BRUSSELS (Reuters) - Greece's new lefti... | businessNews |
3 | Valuations may hurt small caps, despite job gr... | NEW YORK (Reuters) - The good news from Friday... | businessNews |
4 | Shippers suspend weekend cargo loading at U.S.... | LOS ANGELES (Reuters) - The loading and unload... | businessNews |
5 | Building unions return to Kentucky refinery de... | NEW YORK (Reuters) - Workers represented by th... | businessNews |
6 | Motorola exploring possible sale: Bloomberg | (Reuters) - Walkie-talkie and radio systems ma... | businessNews |
7 | Wall St. ends down on interest rate, Greece ji... | NEW YORK (Reuters) - Wall Street stocks fell o... | businessNews |
8 | Strong U.S. job, wage gains open door to mid-y... | WASHINGTON (Reuters) - U.S. job growth rose so... | businessNews |
9 | Building unions return to Kentucky refinery de... | NEW YORK (Reuters) - Workers represented by th... | businessNews |
10 | Motorola exploring possible sale: Bloomberg | (Reuters) - Walkie-talkie and radio systems ma... | businessNews |
11 | JPMorgan under scrutiny over hiring of Chinese... | (Reuters) - JPMorgan Chase & Co is under feder... | businessNews |
12 | Exclusive: Top Fox investors seek to convert v... | NEW YORK (Reuters) - Several top investors in ... | businessNews |
13 | Isolated Greece wants no more bailout money wi... | ATHENS/BRUSSELS (Reuters) - Greece's new lefti... | businessNews |
14 | Wall St. ends down on interest rate, Greece ji... | NEW YORK (Reuters) - Wall Street stocks fell o... | businessNews |
15 | RadioShack would accept liquidation bids: lawyer | (Reuters) - A lawyer for RadioShack on Friday ... | businessNews |
16 | Strong U.S. job, wage gains open door to mid-y... | WASHINGTON (Reuters) - U.S. job growth rose so... | businessNews |
17 | Wall Street firms waver over June 2015 rate hi... | NEW YORK (Reuters) - Economists at Wall Street... | businessNews |
18 | Oil climbs, Brent posts best two weeks since 1998 | NEW YORK (Reuters) - Oil rallied again on Frid... | businessNews |
19 | Brazil's Petrobras taps state banker as CEO; s... | BRASILIA (Reuters) - Brazil's President Dilma ... | businessNews |
20 | Feds can't indefinitely gag Yahoo about subpoe... | SAN FRANCISCO (Reuters) - Law enforcement cann... | technologyNews |
21 | Motorola exploring possible sale: Bloomberg | (Reuters) - Walkie-talkie and radio systems ma... | technologyNews |
22 | Anthem warns U.S. customers of email scam afte... | (Reuters) - Health insurer Anthem Inc on Frida... | technologyNews |
23 | U.S. court orders Symantec to pay $17 mln for ... | (Reuters) - Symantec Corp, maker of the popula... | technologyNews |
24 | Motorola exploring possible sale: Bloomberg | (Reuters) - Walkie-talkie and radio systems ma... | technologyNews |
25 | RadioShack would accept liquidation bids: lawyer | (Reuters) - A lawyer for RadioShack on Friday ... | technologyNews |
26 | Anthem warns U.S. customers of email scam afte... | (Reuters) - Health insurer Anthem Inc on Frida... | technologyNews |
27 | Google can disrupt car industry but is no auto... | FRANKFURT (Reuters) - Technology companies suc... | technologyNews |
28 | Chipmaker Intel promotes senior executives, du... | SAN FRANCISCO (Reuters) - Chipmaker Intel Corp... | technologyNews |
29 | Feds can't indefinitely gag Yahoo about subpoe... | SAN FRANCISCO (Reuters) - Law enforcement cann... | technologyNews |
30 | Google panel backs firm on EU limit to 'right ... | BRUSSELS (Reuters) - A panel of experts appoin... | technologyNews |
31 | Family money puts its faith in European techno... | LONDON (Reuters) - Some of Europe's wealthiest... | technologyNews |
32 | U.S. states probe massive data breach at healt... | NEW YORK (Reuters) - Several U.S. states are i... | technologyNews |
33 | Health insurer Anthem hit by massive cybersecu... | (Reuters) - Health insurer Anthem Inc , which ... | technologyNews |
34 | China to have most robots in world by 2017 | FRANKFURT (Reuters) - China will have more rob... | technologyNews |
35 | U.S. businesses ask White House to help on Chi... | WASHINGTON (Reuters) - U.S. business lobbies c... | technologyNews |
36 | Sharp says not considering sale of any oversea... | TOKYO (Reuters) - Japanese electronics maker S... | technologyNews |
37 | RadioShack files for bankruptcy; Sprint to tak... | (Reuters) - Electronics retailer RadioShack Co... | technologyNews |
38 | SEC probes Blackberry options trading ahead of... | NEW YORK (Reuters) - The U.S. Securities and E... | technologyNews |
39 | Twitter tops Wall Street revenue target, user ... | SAN FRANCISCO (Reuters) - Twitter Inc said on ... | technologyNews |
40 | U.S. Golf Association to launch Senior Women's... | NEW YORK (Reuters) - The United States Golf As... | sportsNews |
41 | In-form Wiesberger goes two shots clear in Kua... | KUALA LUMPUR (Reuters) - Austrian Bernd Wiesbe... | sportsNews |
42 | Jansrud signals downhill ambition in final tra... | BEAVER CREEK, Colorado (Reuters) - Unheralded ... | sportsNews |
43 | NFL, players union spar in court over Peterson... | MINNEAPOLIS (Reuters) - The National Football ... | sportsNews |
44 | English takes charge as Mickelson exits Torrey... | LA JOLLA, California (Reuters) - Local favorit... | sportsNews |
45 | No pressure for double medallist Fenninger | BEAVER CREEK, Colorado (Reuters) - A Super-G v... | sportsNews |
46 | Referees association backs rookie in flare-up ... | (Reuters) - The National Basketball Referees A... | sportsNews |
47 | Rangers goalie Lundqvist out for three weeks w... | (Reuters) - New York Rangers goalie Henrik Lun... | sportsNews |
48 | Miracle now needed to win gold, says Vonn | BEAVER CREEK, Colorado (Reuters) - Denied gold... | sportsNews |
49 | NFL and players union spar in court over Peter... | MINNEAPOLIS (Reuters) - The National Football ... | sportsNews |
50 | No sharing world downhill gold for Maze | BEAVER CREEK, Colorado (Reuters) - Tina Maze h... | sportsNews |
51 | Doping should be criminal offense: Russia chief | MOSCOW (Reuters) - Russia's Athletics Federati... | sportsNews |
52 | Kansas police drop drug charges against Dallas... | DALLAS, Texas (Reuters) - Authorities have dro... | sportsNews |
53 | Thompson leads by one stroke at Torrey Pines | LA JOLLA, California (Reuters - American Nicho... | sportsNews |
54 | Borzakovskiy takes over as Russia's caretaker ... | MOSCOW (Reuters) - Former Olympic 800 meters c... | sportsNews |
55 | Clubs need long-term mental health plan: FIFPro | (Reuters) - Soccer clubs need to take a longer... | sportsNews |
56 | European Games will convince skeptics, attract... | (Reuters) - The inaugural European Games this ... | sportsNews |
57 | Departing VFLA President to continue fight aga... | MOSCOW (Reuters) - Russian Athletics Federatio... | sportsNews |
58 | Canizares joins champion Westwood at the top i... | KUALA LUMPUR (Reuters) - Spaniard Alejandro Ca... | sportsNews |
59 | Force India opposed Marussia's bid to use old car | LONDON (Reuters) - Marussia's hopes of rising ... | sportsNews |
news_data["description"].head()
0 NEW YORK (Reuters) - Several top investors in ... 1 ROME/PARIS (Reuters) - In Paris and Rome, it w... 2 ATHENS/BRUSSELS (Reuters) - Greece's new lefti... 3 NEW YORK (Reuters) - The good news from Friday... 4 LOS ANGELES (Reuters) - The loading and unload... Name: description, dtype: object
print news_data["description"][0]
NEW YORK (Reuters) - Several top investors in Twenty-First Century Fox Inc are pressing for the right to swap their voting shares for ordinary shares, which are trading at an unusual premium, even though the move could hand even more control of the company to Rupert Murdoch, according to people familiar with the matter.<img width='1' height='1' src='http://reuters.us.feedsportal.com/c/35217/f/654199/s/4325e2cf/sc/2/mf.gif' border='0'/><br clear='all'/><div class="feedflare"> <a href="http://feeds.reuters.com/~ff/reuters/businessNews?a=Ti_4toCmS6w:UwKWzqPQZuQ:yIl2AUoC8zA"><img src="http://feeds.feedburner.com/~ff/reuters/businessNews?d=yIl2AUoC8zA" border="0"></img></a> <a href="http://feeds.reuters.com/~ff/reuters/businessNews?a=Ti_4toCmS6w:UwKWzqPQZuQ:F7zBnMyn0Lo"><img src="http://feeds.feedburner.com/~ff/reuters/businessNews?i=Ti_4toCmS6w:UwKWzqPQZuQ:F7zBnMyn0Lo" border="0"></img></a> <a href="http://feeds.reuters.com/~ff/reuters/businessNews?a=Ti_4toCmS6w:UwKWzqPQZuQ:V_sGLiPBpWU"><img src="http://feeds.feedburner.com/~ff/reuters/businessNews?i=Ti_4toCmS6w:UwKWzqPQZuQ:V_sGLiPBpWU" border="0"></img></a> </div><img src="//feeds.feedburner.com/~r/reuters/businessNews/~4/Ti_4toCmS6w" height="1" width="1" alt=""/>
%%HTML
NEW YORK (Reuters) - Several top investors in Twenty-First Century Fox Inc are pressing for the right to swap their voting shares for ordinary shares, which are trading at an unusual premium, even though the move could hand even more control of the company to Rupert Murdoch, according to people familiar with the matter.<img width='1' height='1' src='http://reuters.us.feedsportal.com/c/35217/f/654199/s/4325e2cf/sc/2/mf.gif' border='0'/><br clear='all'/><div class="feedflare">
<a href="http://feeds.reuters.com/~ff/reuters/businessNews?a=Ti_4toCmS6w:UwKWzqPQZuQ:yIl2AUoC8zA"><img src="http://feeds.feedburner.com/~ff/reuters/businessNews?d=yIl2AUoC8zA" border="0"></img></a> <a href="http://feeds.reuters.com/~ff/reuters/businessNews?a=Ti_4toCmS6w:UwKWzqPQZuQ:F7zBnMyn0Lo"><img src="http://feeds.feedburner.com/~ff/reuters/businessNews?i=Ti_4toCmS6w:UwKWzqPQZuQ:F7zBnMyn0Lo" border="0"></img></a> <a href="http://feeds.reuters.com/~ff/reuters/businessNews?a=Ti_4toCmS6w:UwKWzqPQZuQ:V_sGLiPBpWU"><img src="http://feeds.feedburner.com/~ff/reuters/businessNews?i=Ti_4toCmS6w:UwKWzqPQZuQ:V_sGLiPBpWU" border="0"></img></a>
</div><img src="//feeds.feedburner.com/~r/reuters/businessNews/~4/Ti_4toCmS6w" height="1" width="1" alt=""/>
news_data["short_description"] = [item[item.find(" - ")+3:item.find("<")] for item in news_data["description"]]
news_data
title | description | category | short_description | |
---|---|---|---|---|
0 | Exclusive: Top Fox investors seek to convert v... | NEW YORK (Reuters) - Several top investors in ... | businessNews | Several top investors in Twenty-First Century ... |
1 | Different delivery, one message to Greece's ne... | ROME/PARIS (Reuters) - In Paris and Rome, it w... | businessNews | In Paris and Rome, it was sugar coated; in Ber... |
2 | Isolated Greece wants no more bailout money wi... | ATHENS/BRUSSELS (Reuters) - Greece's new lefti... | businessNews | Greece's new leftist-led government, isolated ... |
3 | Valuations may hurt small caps, despite job gr... | NEW YORK (Reuters) - The good news from Friday... | businessNews | The good news from Friday's jobs report may al... |
4 | Shippers suspend weekend cargo loading at U.S.... | LOS ANGELES (Reuters) - The loading and unload... | businessNews | The loading and unloading of freighters will b... |
5 | Building unions return to Kentucky refinery de... | NEW YORK (Reuters) - Workers represented by th... | businessNews | Workers represented by the Building Trades Uni... |
6 | Motorola exploring possible sale: Bloomberg | (Reuters) - Walkie-talkie and radio systems ma... | businessNews | Walkie-talkie and radio systems maker Motorola... |
7 | Wall St. ends down on interest rate, Greece ji... | NEW YORK (Reuters) - Wall Street stocks fell o... | businessNews | Wall Street stocks fell on Friday as a better-... |
8 | Strong U.S. job, wage gains open door to mid-y... | WASHINGTON (Reuters) - U.S. job growth rose so... | businessNews | U.S. job growth rose solidly in January and wa... |
9 | Building unions return to Kentucky refinery de... | NEW YORK (Reuters) - Workers represented by th... | businessNews | Workers represented by the Building Trades Uni... |
10 | Motorola exploring possible sale: Bloomberg | (Reuters) - Walkie-talkie and radio systems ma... | businessNews | Walkie-talkie and radio systems maker Motorola... |
11 | JPMorgan under scrutiny over hiring of Chinese... | (Reuters) - JPMorgan Chase & Co is under feder... | businessNews | JPMorgan Chase & Co is under federal scrutiny ... |
12 | Exclusive: Top Fox investors seek to convert v... | NEW YORK (Reuters) - Several top investors in ... | businessNews | Several top investors in Twenty-First Century ... |
13 | Isolated Greece wants no more bailout money wi... | ATHENS/BRUSSELS (Reuters) - Greece's new lefti... | businessNews | Greece's new leftist-led government, isolated ... |
14 | Wall St. ends down on interest rate, Greece ji... | NEW YORK (Reuters) - Wall Street stocks fell o... | businessNews | Wall Street stocks fell on Friday as a better-... |
15 | RadioShack would accept liquidation bids: lawyer | (Reuters) - A lawyer for RadioShack on Friday ... | businessNews | A lawyer for RadioShack on Friday said the ban... |
16 | Strong U.S. job, wage gains open door to mid-y... | WASHINGTON (Reuters) - U.S. job growth rose so... | businessNews | U.S. job growth rose solidly in January and wa... |
17 | Wall Street firms waver over June 2015 rate hi... | NEW YORK (Reuters) - Economists at Wall Street... | businessNews | Economists at Wall Street's biggest banks are ... |
18 | Oil climbs, Brent posts best two weeks since 1998 | NEW YORK (Reuters) - Oil rallied again on Frid... | businessNews | Oil rallied again on Friday, with benchmark Br... |
19 | Brazil's Petrobras taps state banker as CEO; s... | BRASILIA (Reuters) - Brazil's President Dilma ... | businessNews | Brazil's President Dilma Rousseff tapped a con... |
20 | Feds can't indefinitely gag Yahoo about subpoe... | SAN FRANCISCO (Reuters) - Law enforcement cann... | technologyNews | Law enforcement cannot indefinitely forbid Yah... |
21 | Motorola exploring possible sale: Bloomberg | (Reuters) - Walkie-talkie and radio systems ma... | technologyNews | Walkie-talkie and radio systems maker Motorola... |
22 | Anthem warns U.S. customers of email scam afte... | (Reuters) - Health insurer Anthem Inc on Frida... | technologyNews | Health insurer Anthem Inc on Friday warned U.S... |
23 | U.S. court orders Symantec to pay $17 mln for ... | (Reuters) - Symantec Corp, maker of the popula... | technologyNews | Symantec Corp, maker of the popular Norton ant... |
24 | Motorola exploring possible sale: Bloomberg | (Reuters) - Walkie-talkie and radio systems ma... | technologyNews | Walkie-talkie and radio systems maker Motorola... |
25 | RadioShack would accept liquidation bids: lawyer | (Reuters) - A lawyer for RadioShack on Friday ... | technologyNews | A lawyer for RadioShack on Friday said the ban... |
26 | Anthem warns U.S. customers of email scam afte... | (Reuters) - Health insurer Anthem Inc on Frida... | technologyNews | Health insurer Anthem Inc on Friday warned U.S... |
27 | Google can disrupt car industry but is no auto... | FRANKFURT (Reuters) - Technology companies suc... | technologyNews | Technology companies such as Google are unlike... |
28 | Chipmaker Intel promotes senior executives, du... | SAN FRANCISCO (Reuters) - Chipmaker Intel Corp... | technologyNews | Chipmaker Intel Corp said on Friday it promote... |
29 | Feds can't indefinitely gag Yahoo about subpoe... | SAN FRANCISCO (Reuters) - Law enforcement cann... | technologyNews | Law enforcement cannot indefinitely forbid Yah... |
30 | Google panel backs firm on EU limit to 'right ... | BRUSSELS (Reuters) - A panel of experts appoin... | technologyNews | A panel of experts appointed by Google to advi... |
31 | Family money puts its faith in European techno... | LONDON (Reuters) - Some of Europe's wealthiest... | technologyNews | Some of Europe's wealthiest families are inves... |
32 | U.S. states probe massive data breach at healt... | NEW YORK (Reuters) - Several U.S. states are i... | technologyNews | Several U.S. states are investigating a massiv... |
33 | Health insurer Anthem hit by massive cybersecu... | (Reuters) - Health insurer Anthem Inc , which ... | technologyNews | Health insurer Anthem Inc , which has nearly 4... |
34 | China to have most robots in world by 2017 | FRANKFURT (Reuters) - China will have more rob... | technologyNews | China will have more robots operating in its p... |
35 | U.S. businesses ask White House to help on Chi... | WASHINGTON (Reuters) - U.S. business lobbies c... | technologyNews | U.S. business lobbies called on the White Hous... |
36 | Sharp says not considering sale of any oversea... | TOKYO (Reuters) - Japanese electronics maker S... | technologyNews | Japanese electronics maker Sharp Corp said a s... |
37 | RadioShack files for bankruptcy; Sprint to tak... | (Reuters) - Electronics retailer RadioShack Co... | technologyNews | Electronics retailer RadioShack Corp filed for... |
38 | SEC probes Blackberry options trading ahead of... | NEW YORK (Reuters) - The U.S. Securities and E... | technologyNews | The U.S. Securities and Exchange Commission is... |
39 | Twitter tops Wall Street revenue target, user ... | SAN FRANCISCO (Reuters) - Twitter Inc said on ... | technologyNews | Twitter Inc said on Thursday the social media ... |
40 | U.S. Golf Association to launch Senior Women's... | NEW YORK (Reuters) - The United States Golf As... | sportsNews | The United States Golf Association will launch... |
41 | In-form Wiesberger goes two shots clear in Kua... | KUALA LUMPUR (Reuters) - Austrian Bernd Wiesbe... | sportsNews | Austrian Bernd Wiesberger will take a two-shot... |
42 | Jansrud signals downhill ambition in final tra... | BEAVER CREEK, Colorado (Reuters) - Unheralded ... | sportsNews | Unheralded Frenchman Brice Roger posted the fa... |
43 | NFL, players union spar in court over Peterson... | MINNEAPOLIS (Reuters) - The National Football ... | sportsNews | The National Football League and its players u... |
44 | English takes charge as Mickelson exits Torrey... | LA JOLLA, California (Reuters) - Local favorit... | sportsNews | Local favorite Phil Mickelson has joined Tiger... |
45 | No pressure for double medallist Fenninger | BEAVER CREEK, Colorado (Reuters) - A Super-G v... | sportsNews | A Super-G victory at last year's Winter Olympi... |
46 | Referees association backs rookie in flare-up ... | (Reuters) - The National Basketball Referees A... | sportsNews | The National Basketball Referees Association c... |
47 | Rangers goalie Lundqvist out for three weeks w... | (Reuters) - New York Rangers goalie Henrik Lun... | sportsNews | New York Rangers goalie Henrik Lundqvist will ... |
48 | Miracle now needed to win gold, says Vonn | BEAVER CREEK, Colorado (Reuters) - Denied gold... | sportsNews | Denied gold medals in her best events, America... |
49 | NFL and players union spar in court over Peter... | MINNEAPOLIS (Reuters) - The National Football ... | sportsNews | The National Football League and its players u... |
50 | No sharing world downhill gold for Maze | BEAVER CREEK, Colorado (Reuters) - Tina Maze h... | sportsNews | Tina Maze had to share Olympic downhill gold b... |
51 | Doping should be criminal offense: Russia chief | MOSCOW (Reuters) - Russia's Athletics Federati... | sportsNews | Russia's Athletics Federation (VFLA) president... |
52 | Kansas police drop drug charges against Dallas... | DALLAS, Texas (Reuters) - Authorities have dro... | sportsNews | Authorities have dropped drug charges against ... |
53 | Thompson leads by one stroke at Torrey Pines | LA JOLLA, California (Reuters - American Nicho... | sportsNews | American Nicholas Thompson has a one-shot lead... |
54 | Borzakovskiy takes over as Russia's caretaker ... | MOSCOW (Reuters) - Former Olympic 800 meters c... | sportsNews | Former Olympic 800 meters champion Yury Borzak... |
55 | Clubs need long-term mental health plan: FIFPro | (Reuters) - Soccer clubs need to take a longer... | sportsNews | Soccer clubs need to take a longer-term approa... |
56 | European Games will convince skeptics, attract... | (Reuters) - The inaugural European Games this ... | sportsNews | The inaugural European Games this year will co... |
57 | Departing VFLA President to continue fight aga... | MOSCOW (Reuters) - Russian Athletics Federatio... | sportsNews | Russian Athletics Federation (VFLA) President ... |
58 | Canizares joins champion Westwood at the top i... | KUALA LUMPUR (Reuters) - Spaniard Alejandro Ca... | sportsNews | Spaniard Alejandro Canizares picked up three s... |
59 | Force India opposed Marussia's bid to use old car | LONDON (Reuters) - Marussia's hopes of rising ... | sportsNews | Marussia's hopes of rising from the dead to ra... |
corpus = news_data["short_description"]
vectorizer = CountVectorizer(min_df=1)
X = vectorizer.fit_transform(corpus).toarray()
print X.shape
X
(60, 809)
array([[0, 0, 0, ..., 0, 0, 0], [0, 0, 0, ..., 1, 0, 0], [0, 0, 0, ..., 0, 0, 1], ..., [0, 0, 0, ..., 0, 0, 0], [0, 0, 0, ..., 0, 0, 0], [0, 0, 0, ..., 0, 0, 0]])
vectorizer.get_feature_names()[:25]
[u'11', u'14', u'17', u'18', u'2012', u'2017', u'2018', u'29', u'40', u'400', u'65', u'800', u'about', u'abroad', u'abuse', u'accept', u'according', u'account', u'added', u'admitted', u'adrian', u'advise', u'affiliate', u'after', u'again']
categories = news_data["category"].unique()
category_dict = {value:index for index, value in enumerate(categories)}
results = news_data["category"].map(category_dict)
category_dict
{'businessNews': 0, 'sportsNews': 2, 'technologyNews': 1}
print "corpus size: %s" % len(vectorizer.get_feature_names())
corpus size: 809
x_train,x_test, y_train,y_test = train_test_split(X, results, test_size=0.2, random_state=1, )
clf = MultinomialNB()
clf.fit(x_train, y_train)
clf.score(x_test, y_test)
0.83333333333333337
y_test
array([1, 2, 0, 2, 2, 2, 1, 1, 2, 1, 1, 2])
clf.predict(x_test)
array([1, 2, 0, 2, 2, 2, 1, 0, 2, 1, 0, 2])
category_dict
{'businessNews': 0, 'sportsNews': 2, 'technologyNews': 1}
text = ["Who won the Superbawl?"]
vec_text = vectorizer.transform(text).toarray()
category_dict.keys()[category_dict.values().index(clf.predict(vec_text)[0])]
'sportsNews'