# Import the required modules
import requests as re
import pandas as pd
# Set iPython's max column width to 50
pd.set_option('display.max_columns', 50)
# Create a variable with your CrisisNET API key
api_key = '532d8dc4ed3329652f114b73'
# Setup the request header
headers = {'Authorization': 'Bearer ' + api_key}
# Setup the request's URL
url = 'http://api.crisis.net/item?limit=200&sources=facebook'
# Make the API request
r = re.get(url, headers=headers)
# Define a function that,
def request_to_df(r):
# converts the json into a dataframe,
request_df = pd.DataFrame(r.json())
# expands the df.data cell,
df = request_df['data'].apply(pd.Series)
# converts df.updatedAt to a datetime object,
df["updatedAt"] = pd.to_datetime(df["updatedAt"])
# sets it as the dataframe's index,
df.index = df['updatedAt']
# expands the df.geo object,
geo_df = df['geo'].apply(pd.Series)
# expands the admin zones,
geo_admin_df = geo_df['addressComponents'].apply(pd.Series)
# merges the admin zones back into the dataframe,
df = pd.concat([df[:], geo_admin_df[:], geo_df[:]], axis=1)
# seperates the lat and long objects,
df['latitude'], df['longitude'] = df['coords'].str[1], df['coords'].str[0]
# expands df.tags,
tags_df = df['tags'].apply(pd.Series)
# defines a function called tag_extractor,
def tag_extractor(x):
# that, if x is a string,
if type(x) is float:
# just returns it untouched
return x
# but, if not, convert x to a dict() and return the value
elif x:
x = dict(x)
return x['name']
# and leave everything else
else:
return
# executes tag_extractor on the tags dataframe,
tags_df = tags_df.applymap(tag_extractor)
# renames all the tag columns,
tags_df = tags_df.rename(columns = lambda x : 'tag_' + str(x))
# merges everything back together,
df = pd.concat([df[:], tags_df[:]], axis=1)
# expands df.language,
lang_df = df['language'].apply(pd.Series)
# takes the language code,
df['lang'] = lang_df['code']
# and finally returns the primary dataframe
return df
# Create a dataframe, df, which is the output
# of running request_to_df() on the request, r
df = request_to_df(r)
# View the first three rows of the dataframe
df.head(3)
content | contentEnglish | createdAt | entities | fromURL | geo | id | image | language | license | lifespan | publishedAt | remoteID | source | summary | tags | updatedAt | adminArea1 | formattedAddress | addressComponents | coords | locationIdentifiers | latitude | longitude | tag_0 | tag_1 | tag_2 | tag_3 | tag_4 | lang | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
updatedAt | ||||||||||||||||||||||||||||||
2014-06-16 17:57:59.437975 | تيست تيست الله واحد بسم الله | Taste Test is one God the name of God | 2014-06-16T17:57:40.929115 | [Syria] | http://facebook.com/1374598669493054_142569920... | {'addressComponents': {'adminArea1': 'Syria', ... | WaG2-CL3TD6SPtKbO169Og | NaN | {'name': 'Arabic', 'code': 'ar', 'nativeName':... | temporary | 2014-06-16T17:57:14+00:00 | 1374598669493054_1425699207716333 | تيست تيست الله واحد بسم الله | NaN | 2014-06-16 17:57:59.437975 | Syria | Syria | {'adminArea1': 'Syria', 'formattedAddress': 'S... | [38.473472595214844, 35.03312683105469] | NaN | 35.033127 | 38.473473 | NaN | NaN | NaN | NaN | NaN | ar | ||
2014-06-16 17:56:08.025135 | #دمشق #الهامة :: 16 / 6 / 2014\n\nاطلاق نار يس... | # Damascus # important :: 06/16/2014\n\nNow he... | 2014-06-16T17:55:49.361887 | [Syria] | http://facebook.com/707441785943990_7502105516... | {'addressComponents': {'adminArea1': 'Syria', ... | bRLkCPaPSMumDB5n7fL1Vg | NaN | {'name': 'Arabic', 'code': 'ar', 'nativeName':... | temporary | 2014-06-16T17:54:47+00:00 | 707441785943990_750210551667113 | #دمشق #الهامة :: 16 / 6 / 2014\n\nاطلاق نار يس... | NaN | 2014-06-16 17:56:08.025135 | Syria | Syria | {'adminArea1': 'Syria', 'formattedAddress': 'S... | [38.473472595214844, 35.03312683105469] | NaN | 35.033127 | 38.473473 | NaN | NaN | NaN | NaN | NaN | ar | ||
2014-06-16 17:57:33.297103 | جمعية حنين الخيرية للرعاية الطبية والتامين ال... | Nostalgia Society charity for medical care and... | 2014-06-16T17:57:14.306554 | [Syria] | http://facebook.com/346796602092946_5254564375... | {'addressComponents': {'adminArea1': 'Syria', ... | ACXhEV-DTFudNpmVXoVS9Q | NaN | {'name': 'Arabic', 'code': 'ar', 'nativeName':... | temporary | 2014-06-16T17:54:29+00:00 | 346796602092946_525456437560294 | جمعية حنين الخيرية للرعاية الطبية والتامين ال... | NaN | 2014-06-16 17:57:33.297103 | Syria | Syria | {'adminArea1': 'Syria', 'formattedAddress': 'S... | [38.473472595214844, 35.03312683105469] | {'authorLocationName': 'سوريا إدلب-التمانعة'} | 35.033127 | 38.473473 | NaN | NaN | NaN | NaN | NaN | ar |
3 rows × 30 columns