import requests
import toolz
import dateutil.parser
import datetime
import pandas
import logging
/Users/baart_f/.virtualenvs/main/lib/python3.5/site-packages/matplotlib/__init__.py:913: UserWarning: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter. warnings.warn(self.msg_depr % (key, alt_key))
resp = requests.get('http://rolfhut.nl/DataFromParticleCloud/getGeoJSONFromMYSQLalt.php')
data = resp.json()
len(data['features'])
8434
def wtf2date(wtf):
"""convert a timestamp in a weird format to a date"""
timestamp = wtf
timepart = timestamp[6:].zfill(6)
datestr = "{year}-{month}-{day} {hour}:{minute}:{second}".format(
year='20' + timestamp[4:6],
month=timestamp[2:4],
day=timestamp[:2],
hour=timepart[:2],
minute=timepart[2:4],
second=timepart[4:6]
)
try:
date = dateutil.parser.parse(datestr)
except:
logging.exception("failed: %s", timestamp)
raise
return date
records = []
for feature in data['features']:
# skip old dates
if len(feature['properties']['timeStamp']) == 10:
logging.debug("skipping: %s", feature['properties']['timeStamp'])
continue
# skip missing location
if feature['geometry']['coordinates'][1] == -1:
logging.debug("skipping: %s", feature['geometry'])
continue
record = dict(
device=feature['properties']['deviceID'],
group=feature['properties']['group'],
time=wtf2date(feature['properties']['timeStamp']),
lat=feature['geometry']['coordinates'][1],
lon=feature['geometry']['coordinates'][0],
name=feature['properties']['name']
)
records.append(record)
df = pandas.DataFrame.from_records(records)
df.to_csv('alcatraz_drifters.csv')