import requests
import pandas as pd
api_key = '532d8dc4ed3329652f114b73'
api_url = 'http://api.crisis.net/item/?sources=twitter'
headers = {'Authorization': 'Bearer ' + api_key}
total = 10000
df = pd.DataFrame()
def get_data(offset=0, limit=100, df=None):
url = api_url + '&offset=' + str(offset) + '&limit=' + str(limit)
r = requests.get(url, headers=headers)
x = pd.DataFrame(r.json())
x = x['data'].apply(pd.Series)
df = df.append(x, ignore_index=True)
if total > offset + limit:
return get_data(offset + limit, limit, df)
return df
df = get_data(df=df)