import sys,os
import collections, difflib
from IPython.display import HTML, display_pretty, display_html
from difflib import SequenceMatcher

import laf
from laf.fabric import LafFabric
from etcbc.preprocess import prepare
fabric = LafFabric()

API = fabric.load(source+version, 'lexicon', 'rare_lexemes', {
    "xmlids": {"node": False, "edge": False},
    "features": ('''
        otype
        language lex g_cons g_word_utf8 trailer_utf8 phono phono_sep
        sp
        book chapter verse label number
    ''',''),
    "prepare": prepare,
    "primary": False,
}, verbose='NORMAL')
exec(fabric.localnames.format(var='fabric'))

grams = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.Counter()))
ng = 4 # the max number of members in the n-gram
msg('Collecting all <= {}-grams'.format(ng))
for c in F.otype.s('clause'):
    bk = F.book.v(L.u('book', c))
    words = list(L.d('word', c))
    lenw = len(words)
    for n in range(1, ng + 1):
        for i in range(lenw - n):
            this_gram = '-'.join(F.lex.v(w) if F.sp.v(w) in ONLY_SP else '*' for w in (words[j] for j in range(i, n)))
            grams[n][bk][this_gram] += 1
msg('Done')

freqs_by_g = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.Counter()))
freqs_by_c = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.Counter()))

msg('Counting grams')
for n in grams:
    ngrams = grams[n]
    for bk in ngrams:
        cl = book_classes_index.get(bk, 'bible')
        these_grams = ngrams[bk]
        for g in these_grams:
            freqs_by_g[n][g][cl] += these_grams[g]
            freqs_by_c[n][cl][g] += these_grams[g]
msg('Done')

RARE = 1

class_a = 'legal'
class_b = 'prophets'

results = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.Counter()))

msg('Filtering')
for RARE in range(1, 10):
    for n in sorted(freqs_by_g):
        nfreqs_by_g = freqs_by_g[n]
        for g in nfreqs_by_g:
            g_info = nfreqs_by_g[g]
            nc = dict((cl, g_info.get(cl, 0)) for cl in book_classes)
            if nc[class_a] <= RARE and nc[class_b] <= RARE: results[RARE][n]['rr'] += 1 
            elif nc[class_a] <= RARE and nc[class_b] > RARE: results[RARE][n]['ra'] += 1
            elif nc[class_a] > RARE and nc[class_b] <= RARE: results[RARE][n]['ar'] += 1
            elif nc[class_a] > RARE and nc[class_b] > RARE: results[RARE][n]['aa'] += 1
            else: results[RARE][n]['xx'] += 1
msg('Done')

for RARE in sorted(results):
    print('RARE means <= {}'.format(RARE))
    for n in range(1, ng + 1):
        print('\t{}-grams: rr ra ar aa = {}'.format(
            n, 
            ' '.join(
                    '{:>5}'.format(results[RARE].get(n, {}).get(y, 0)) for y in ('rr', 'ra', 'ar', 'aa')
                ),
            ))

msg("Making a mapping between a passage specification and a verse node")
passage2vnode = {}
for vs in F.otype.s('verse'):
    lab = (F.book.v(vs), F.chapter.v(vs), F.verse.v(vs))
    passage2vnode[lab] = vs
msg("{} verses".format(len(passage2vnode)))

CROSSREF_TOOL = 'parallel'
CROSSREF_DB_FILE = 'crossrefdb.csv'
SHEBANQ_PATH = os.path.abspath('{}/../../../shebanq'.format(os.getcwd))
CROSSREF_DB_DIR = '{}/static/docs/tools/{}/files'.format(SHEBANQ_PATH, CROSSREF_TOOL)
CROSSREF_DB_PATH = '{}/{}'.format(CROSSREF_DB_DIR, CROSSREF_DB_FILE)
msg(CROSSREF_DB_PATH)
PRETTY_PAIRS = 'pairs.html'
DIFF_LEX = 'difflex.csv'

loc_tpl = 'https://rawgit.com/etcbc/laf-fabric-nbs/master/lingvar/{}'
HTML('''
<a target="_blank" href="{}">parallel_pairs</a>
<a target="_blank" href="{}">difference lexemes</a>
'''.format(loc_tpl.format(PRETTY_PAIRS), loc_tpl.format(DIFF_LEX)))

msg('Reading crossrefs database')
n = 0
parallel_pairs_proto = collections.defaultdict(lambda: set())
group_index = {}
group_number = 0
with open(CROSSREF_DB_PATH) as f:
    for line in f:
        n += 1
        if n == 1: continue
        (bkx, chx, vsx, bky, chy, vsy, rd) = line.rstrip('\n').split('\t')
        vx = passage2vnode[(bkx, chx, vsx)]
        vy = passage2vnode[(bky, chy, vsy)]
        gx = group_index.get(vx, None)
        gy = group_index.get(vy, None)
        if gx == None and gy == None:
            group_number += 1
            group_index[vx] = group_number
            group_index[vy] = group_number
        elif gx == None:
            group_index[vx] = gy
        elif gy == None:
            group_index[vy] = gx
        elif gx != gy:
            update = [x for x in group_index if group_index[x] == gx]
            for x in update: group_index[x] = gy
msg('{} entries read'.format(n))

for (x, n) in group_index.items(): parallel_pairs_proto[n].add(x)
parallel_pairs = [sorted(parallel_pairs_proto[n]) for n in parallel_pairs_proto]
parallel_pairs_proto = None
msg('Gathered {} sets of parallel verses'.format(len(parallel_pairs)))

msg('Building a difference lexeme table')
intersections = []
unions = []
for p in parallel_pairs:
    clex = None
    ulex = collections.defaultdict(lambda: set())
    for x in p:
        xlex = {F.lex.v(w) for w in L.d('word', x)}
        clex = xlex if clex == None else clex & xlex
        for lex in xlex:
            ulex[lex].add(x)
    intersections.append(clex)
    unions.append(ulex)
msg('{} groups in table'.format(len(intersections)))

msg('Pretty printing the table')
allgeni_html = []
allgenh_html = []
for (i, p) in enumerate(parallel_pairs):
    clex = intersections[i]
    allgeni_html.append(index_chunk(i, clex, p))
    allgenh_html.append(print_chunk(i, clex, p))
outf = open('pairs.html', 'w')
outf.write(html_file_tpl.format(
    'Pairs',
    css,
    ''.join(allgeni_html),
    ''.join(allgenh_html),
))
outf.close()
msg('Done')