#Use requests cache so we can keep an archive of the results HTML
import requests
import requests_cache
requests_cache.install_cache('dakar_cache', backend='sqlite')

import pandas as pd

STAGE = 8

URL_PATTERN='https://gaps.dakar.com/2019/dakar/?s={stage}&c=aso&l=ukie&vi={tab}&sv={timerank}&vh={vtype}&sws=99'
#sws - selected waypoint?

#Vehicle types
VTYPE_ ={ 'car':'a','moto':'m','quad':'q','sxs':'s','truck':'c'}

#Screen / tab selection
TAB_ = {'timing':0,'news':1,'ranking':2}

#Options for timing data
TIMING_ = {'gap':0,'time':1}

#Options for ranking data
RANKING_ = {'stage':0, 'general':1}

#pd.read_html(URL_PATTERN.format(stage=STAGE,tab=TAB_['timing'],vtype=VTYPE_['car'],timerank='time'))

#pd.read_html(URL_PATTERN.format(stage=STAGE,tab=TAB_['ranking'],vtype=VTYPE_['car'],timerank='stage'))

TIMING = RANKING = 0
CREWTEAM = 1
BRANDS = 2
COUNTRIES = 3

#Retrieve a page
def _data(stage,vtype='car',tab='timing', timerank='time'):
    ''' Retrieve timing or ranking HTML page and scrape HTML tables. '''
    timerank = RANKING_[timerank] if tab=='ranking' else TIMING_[timerank]
        
    url = URL_PATTERN.format(stage=stage,tab=TAB_[tab],vtype=VTYPE_[vtype],timerank=timerank)
    html = requests.get(url).content
    return pd.read_html(html, na_values=['-'])

def _fetch_timing_data(stage,vtype='car', timerank='time'):
    ''' Return data tables from timing page. '''
    _tmp = _data(stage,vtype=vtype, tab='timing', timerank=timerank)
    _tmp[TIMING].drop(columns=['View details','Select'], inplace=True)
    return _tmp

def _fetch_ranking_data(stage,vtype='car', timerank='stage'):
    ''' Return data tables from ranking page. '''
    rank_cols = ['Pos','Bib','Crew','Brand','Time','Gap','Penalty']
    _tmp = _data(stage,vtype=vtype, tab='ranking', timerank=timerank)
    _tmp[RANKING].drop(columns=['View details','Select'], inplace=True)
    return _tmp

rdata[RANKING]['Bib'] = rdata[RANKING]['Bib'].astype(int)

from pandas.api.types import is_string_dtype

#At first sight, this looks quite complicated, but a lot of it is backfilled 
# to take into account some of the cleaning we need to do for the full (messy) timing data
def _get_timing(df, typ=TIMING, kind='simple'):
    ''' Convert times to time deltas and
        prefix waypoint / timing columns with a two digit counter. '''
    
    #Some of the exclusion column names are backfilled into this function
    # from columns introduced later in the notebook
    # What we're trying to do is identify columns that aren't timing related
    timingcols = [c for c in df[typ].columns if c not in ['Pos','Bib','Crew','Brand', 'Refuel', 'Road Position'] ]
    
    #Clean up the data in a timing column, then cast to timedelta
    for col in timingcols:
        #In the simple approach, we just grab the timing data and dump the mess
        if kind=='simple':
            df[typ][col] = df[typ][col].str.extract(r'(\d{2}:\d{2}:\d{2})')
        else:
            #The full on extractor - try to parse out all the data
            #  that has been munged into a timing column
            if col==timingcols[-1]:
                #There's an end effect:
                #  the last column in the timing dataset doesn't have position embedded in it
                # In this case, just pull out the position gained/maintained/lost flag
                df[typ][[col,col+'_gain']] = df[typ][col].str.extract(r'(\d{2}:\d{2}:\d{2})(.*)', expand=True)
            else:
                #In the main body of the table, position gain as well as waypoint rank position are available
                df[typ][[col,col+'_gain',col+'_pos']] = df[typ][col].str.extract(r'(\d{2}:\d{2}:\d{2})(.*)\((\d*)\)', expand=True)
                #Ideally, the pos cols would be of int type, but int doesn't support NA
                df[typ][col+'_pos'] = df[typ][col+'_pos'].astype(float)
        
        #Cast the time string to a timedelta
        df[typ][col] = pd.to_timedelta( df[typ][col] )


    #In timing screen, rename cols with a leading two digit index
    #This allows us to report splits in order
    #We only want to do this for the timing data columns, not the rank timing columns...
    timingcols = [c for c in timingcols if c not in ['Time','Gap','Penalty']]
    timingcols_map = { x:'{}_{}'.format('{0:02d}'.format(i), x) for i, x in enumerate(timingcols, 0) }
    df[typ].rename(columns=timingcols_map, inplace=True)
                    
    return df

def get_ranking_data(stage,vtype='car', timerank='stage'):
    ''' Retrieve rank timing data and return it in a form we can work directly with. '''
    
    df = _fetch_ranking_data(stage,vtype=vtype, timerank=timerank)
    df[RANKING]['Bib'] = df[RANKING]['Bib'].astype(int)
    return _get_timing(df, typ=RANKING)

def get_refuel_status(df):
    ''' Parse the refuel status out of timing data Bib column.
        Return extended dataframe with a clean Bib column and a new Refuel column. '''
    
    #The .str.extract() function allows us to match separate groups using a regex
    #  and return the corresponding group data as distinct columns
    #Force the Bin type to a str if it isn't created as such so we can regex it...
    df[['Bib','_tmp']] = df['Bib'].astype(str).str.extract(r'(\d*)([^\d]*)', expand=True)
    
    #Set the Refuel status as a Boolean
    df.insert(2, 'Refuel', df['_tmp'])
    df.drop('_tmp', axis=1, inplace=True)
    df['Refuel'] = df['Refuel']!=''
    
    #Set the Bib value as an int
    df['Bib'] = df['Bib'].astype(int)
    
    return df

def get_timing_data(stage,vtype='car', timerank='time', kind='simple'):
    ''' Get timing data in a form ready to use. '''
    df = _fetch_timing_data(stage,vtype=vtype, timerank=timerank)

    df[TIMING] = get_refuel_status(df[TIMING])
    return _get_timing(df, typ=TIMING, kind=kind)
    

def get_annotated_timing_data(stage,vtype='car', timerank='time'):
    ''' Return a timing dataset that's ready to use. '''
    
    df = get_timing_data(stage, vtype, timerank)
    df[TIMING].insert(2,'Road Position', df[TIMING].sort_values('00_dss',ascending=True)['00_dss'].rank())
    return df

def _timing_long(df, nodss=True):
    ''' Cast timing data to long data frame. '''
    
    df = pd.melt(df,
                 id_vars=[c for c in df.columns if not any(_c in c for _c in ['dss','wp','ass'])],
                 var_name='Section', value_name='Time')

    if nodss:
        return df[df['Section']!='00_dss']
    return df

def get_long_annotated_timing_data(stage,vtype='car', timerank='time'):
    ''' Get annotated timing dataframe and convert it to long format. '''
    _tmp = get_annotated_timing_data(stage,vtype, timerank)
    _tmp[TIMING] = _timing_long(_tmp[TIMING])
    
    #Find the total seconds for each split / waypoint duration
    _tmp[TIMING]['TimeInS'] = _tmp[TIMING]['Time'].dt.total_seconds()
    
    return _tmp

def rebaseTimes(times, bib=None, col='splitS'):
    ''' Rebase times relative to a particular competitor. '''
    
    if bib is None: return deltas
    bib = int(bib)
    rebase = times[times['Bib']==bib][['Section',col]].set_index('Section').to_dict(orient='dict')[col]
    times['rebased']=times[col]-times['Section'].map(rebase)
    return times

REBASER = 304
#loeb 306, AL-ATTIYAH 301, pertehansel

timing_data_long_min = rebaseTimes( timing_data_long , REBASER, 'TimeInS')

def pivotRebasedSplits(rebasedSplits):
    ''' For each driver row, find the split. '''
    
    #If there are no splits...
    if rebasedSplits.empty:
        return pd.DataFrame(columns=['Bib']).set_index('Bib')
    
    rbp=-rebasedSplits.pivot('Bib','Section','rebased')
    rbp.columns=['D{}'.format(c) for c in rbp.columns]
    #The columns seem to be sorted? Need to sort in actual split order
    rbp.sort_values(rbp.columns[-1],ascending =True)
    return rbp

import seaborn as sns


#Changes from WRC
#applymap(color_negative, - change column identification
#final apply - add 'Stage Overall' to list
#.background_gradient Add Pos
def moreStyleDriverSplitReportBaseDataframe(rb2,ss, caption=None):
    ''' Style the driver split report dataframe. '''
    
    if rb2.empty: return ''
        
    def _subsetter(cols, items):
        ''' Generate a subset of valid columns from a list. '''
        return [c for c in cols if c in items]
    
    
    #https://community.modeanalytics.com/gallery/python_dataframe_styling/
    # Set CSS properties for th elements in dataframe
    th_props = [
      ('font-size', '11px'),
      ('text-align', 'center'),
      ('font-weight', 'bold'),
      ('color', '#6d6d6d'),
      ('background-color', '#f7f7f9')
      ]

    # Set CSS properties for td elements in dataframe
    td_props = [
      ('font-size', '11px'),
      ]

    # Set table styles
    styles = [
      dict(selector="th", props=th_props),
      dict(selector="td", props=td_props)
      ]
    
    #Define colour palettes
    #cmg = sns.light_palette("green", as_cmap=True)
    #The blue palette helps us scale the Road Position column
    # This may help us to help identify any obvious road position effect when sorting stage times by stage rank
    cm=sns.light_palette((210, 90, 60), input="husl",as_cmap=True)
    s2=(rb2.style
        .background_gradient(cmap=cm, subset=_subsetter(rb2.columns, ['Road Position', 'Pos']))
        .applymap(color_negative,
                  subset=[c for c in rb2.columns if rb2[c].dtype==float and (not c.startswith('D') and c not in ['Overall Position', 'Road Position', 'Pos'])])
        .highlight_min(subset=_subsetter(rb2.columns, ['Overall Position']), color='lightgrey')
        .highlight_max(subset=_subsetter(rb2.columns, ['Overall Time']), color='lightgrey')
        .highlight_max(subset=_subsetter(rb2.columns, ['Previous']), color='lightgrey')
        .apply(bg_color,subset=_subsetter(rb2.columns, ['{} Overall'.format(ss), 'Overall Time', 'Previous', 'Stage Overall']))
        .bar(subset=[c for c in rb2.columns if str(c).startswith('D')], align='zero', color=[ '#5fba7d','#d65f5f'])
        .set_table_styles(styles)
        #.format({'total_amt_usd_pct_diff': "{:.2%}"})
       )
    
    if caption is not None:
        s2.set_caption(caption)

    #nan issue: https://github.com/pandas-dev/pandas/issues/21527
    return s2.render().replace('nan','')

## Can we style the type in the red/green thing by using timedelta and formatter
# eg https://docs.python.org/3.4/library/datetime.html?highlight=weekday#datetime.date.__format__
#and https://stackoverflow.com/a/46370761/454773

#Maybe also add a sparkline? Need to set a common y axis on all charts?

#https://github.com/iiSeymour/sparkline-nb/blob/master/sparkline-nb.ipynb
import matplotlib.pyplot as plt

from io import BytesIO
import urllib
import base64

def fig2inlinehtml(fig):
    figfile = BytesIO()
    fig.savefig(figfile, format='png')
    figfile.seek(0) 
    figdata_png = base64.b64encode(figfile.getvalue())
    #imgstr = '<img src="data:image/png;base64,{}" />'.format(figdata_png)
    imgstr = '<img src="data:image/png;base64,{}" />'.format(urllib.parse.quote(figdata_png))
    return imgstr


def sparkline(data, figsize=(4, 0.5), **kwags):
    """
    Returns a HTML image tag containing a base64 encoded sparkline style plot
    """
    data = list(data)
    
    fig, ax = plt.subplots(1, 1, figsize=figsize, **kwags)
    ax.plot(data)
    for k,v in ax.spines.items():
        v.set_visible(False)
    ax.set_xticks([])
    ax.set_yticks([])    

    dot = 'r.' if sign(data[len(data) - 1] )<0 else 'g.'
       
    plt.plot(len(data) - 1, data[len(data) - 1], dot)

    ax.fill_between(range(len(data)), data, len(data)*[min(data)], alpha=0.1)

    return fig2inlinehtml(fig)

import scipy

def sparkline2(data, figsize=(4, 0.5), **kwags):
    """
    Returns a HTML image tag containing a base64 encoded sparkline style plot
    """
    data = [0 if pd.isnull(d) else d for d in data]
    
    fig, ax = plt.subplots(1, 1, figsize=figsize, **kwags)
    ax.plot(data, linewidth=0.0)
    for k,v in ax.spines.items():
        v.set_visible(False)
    ax.set_xticks([])
    ax.set_yticks([])

    d = scipy.zeros(len(data))
    
    #If we don't interpolate, we get a gap in the sections 
    #  where times change sign compared to the previous section
    ax.fill_between(range(len(data)), data, where=data>d, interpolate=True,  color='green')
    ax.fill_between(range(len(data)), data, where=data<d, interpolate=True, color='red')
    
    return fig2inlinehtml(fig)

%%capture
rb2cTop10['Stage Gap'] = rb2cTop10['test'].map(sparkline2)
rb2cTop10.drop('test', axis=1, inplace=True)

import os
import time
from selenium import webdriver

#Via https://stackoverflow.com/a/52572919/454773
def setup_screenshot(driver,path):
    # Ref: https://stackoverflow.com/a/52572919/
    original_size = driver.get_window_size()
    required_width = driver.execute_script('return document.body.parentNode.scrollWidth')
    required_height = driver.execute_script('return document.body.parentNode.scrollHeight')
    driver.set_window_size(required_width, required_height)
    # driver.save_screenshot(path)  # has scrollbar
    driver.find_element_by_tag_name('body').screenshot(path)  # avoids scrollbar
    driver.set_window_size(original_size['width'], original_size['height'])
    

def getTableImage(url, fn='dummy_table', basepath='.', path='.', delay=5, height=420, width=800):
    ''' Render HTML file in browser and grab a screenshot. '''
    browser = webdriver.Chrome()
    #browser.set_window_size(width, height)
    browser.get(url)
    #Give the map tiles some time to load
    time.sleep(delay)
    imgpath='{}/{}.png'.format(path,fn)
    imgfn = '{}/{}'.format(basepath, imgpath)
    imgfile = '{}/{}'.format(os.getcwd(),imgfn)
    
    setup_screenshot(browser,imgfile)
    browser.quit()
    os.remove(imgfile.replace('.png','.html'))
    #print(imgfn)
    return imgpath


def getTablePNG(tablehtml,basepath='.', path='testpng', fnstub='testhtml'):
    ''' Save HTML table as file. '''
    if not os.path.exists(path):
        os.makedirs('{}/{}'.format(basepath, path))
    fn='{cwd}/{basepath}/{path}/{fn}.html'.format(cwd=os.getcwd(), basepath=basepath, path=path,fn=fnstub)
    tmpurl='file://{fn}'.format(fn=fn)
    with open(fn, 'w') as out:
        out.write(tablehtml)
    return getTableImage(tmpurl, fnstub, basepath, path)