word_list_tools repo¶

redo_coha_initial = False # change to True to redo munge and rewrite pickle

pickle_full_path = data_path + "coha_1.pickle"

if not os.path.isfile(pickle_full_path) or redo_coha_initial == True: # if pickle already exists, coha is not processed, only read
    print "Processing " + coha_filename
    coha = pd.read_table(init_path + coha_filename)
    print "\nTail of initial dataframe:"
    print coha.tail()
    coha.columns = ['freq', 'word', 'decade']
    coha = coha[['word', 'freq', 'decade']]
    coha = coha.fillna('nan')
    coha['nonalpha'] = False
    coha['nonalpha'][coha.word.str.contains('[^A-Za-z]')] = True
    coha['length'] = 0
    decades = [1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900,
               1910, 1920, 1930, 1940, 1950, 1960, 1970, 1980, 1990, 2000]
    for idx, row in coha.iterrows():  ## this step take 2-3 minutes on a medium-quality desktop computer ca 2013
        coha['decade'][idx] = decades[coha['decade'][idx] - 1]
        coha['length'][idx] = len(str(coha.word[idx]))
    def coha_pct(group):
        freq = group.freq.astype(float)
        group['pct'] = (freq / freq.sum() * 100)
        return group
    coha = coha.groupby(['decade']).apply(coha_pct)
    coha.to_pickle(pickle_full_path)
    
else:
    print "Reading " + pickle_full_path
    coha = pd.read_pickle(pickle_full_path)
    
print "\nTail of dataframe:"
print coha.tail()

# note: 'nonalpha' is True if word contains any characters but a-z or A-Z

redo_coha_words = False # change to True to redo munge and rewrite pickle

pickle_full_path = data_path + "coha_words.pickle"
mean_median_full_path = data_path + "coha_words_mean_median.pickle"

if not os.path.isfile(pickle_full_path) or redo_coha_words == True:
    print 'Creating dataframe.'
    total_freq = 0
    weighted_length = 0
    coha_words = coha[coha.nonalpha == 0]
    coha_words = pd.DataFrame(coha_words.groupby(['word']).sum()).reset_index(drop=False)
    coha_words.columns = ['word', 'freq', 'erase1', 'erase2', 'decadesxlen', 'erase3']
    coha_words['length'] = 0
    coha_words['decades'] = 0
    coha_words.sort('freq', ascending = False, inplace=True)
    total_freq = coha_words.freq.sum()
    median_counter = total_freq / 2
    median_found = False
    for idx, row in coha_words.iterrows():
        curr_len = len(coha_words.word[idx])
        coha_words.length[idx] = curr_len
        coha_words.decades[idx] = coha_words.decadesxlen[idx] / coha_words.length[idx]
        weighted_length += coha_words.freq[idx] * curr_len
        median_counter -= curr_len * coha_words.freq[idx]
        if median_counter < 0 and median_found == False:
            wt_median_len = curr_len
            median_found = True
    coha_words = coha_words[['word', 'freq', 'length', 'decades']]
    coha_words.to_pickle(pickle_full_path)
    wt_mean_len = weighted_length * 1.0 / total_freq
    pickle.dump((wt_mean_len, wt_median_len), open(mean_median_full_path, "w"))
    
else:
    print 'Reading pickle.'
    coha_words = pd.read_pickle(pickle_full_path)
    wt_mean_len, wt_median_len = pickle.load(open(mean_median_full_path, "r"))
    
print "Weighted mean word length: " + str(wt_mean_len)
print "Weighted median word length: " + str(wt_median_len)
print "Tail of coha_words dataframe:"
print coha_words.tail()

# create list of coha_words that also appears in moby crossword dictionary

xword = set()
with open(init_path + '/word_list_moby_crossword.flat.txt', 'r') as f:
    for line in f.readlines():
        xword.add(line.strip())

cohawords = set(coha_words.word)

coha_moby = xword.intersection(cohawords)

print "{0} words out of intersection of {1} and {2}".format(len(coha_moby), len(cohawords), len(xword))

open(data_path + '/coha_and_xword.json', 'w+').write(json.dumps(list(coha_moby)))

redo_brown_initial = False # change to True to redo munge and rewrite pickle

pickle_full_path = data_path + "/brown_df.pickle"

if not os.path.isfile(pickle_full_path) or redo_brown_initial == True: 
    
    print "Processing Brown corpus from NLTK"
    categories = []
    words = []
    frequencies = []
    
    for category in brown.categories():
        wordlist = brown.words(categories=category)
        freqs = nltk.FreqDist([w.lower() for w in wordlist])
        for key in freqs.keys():
            categories.append(category)
            words.append(key)
            frequencies.append(freqs[key])
    
    brown_df = pd.DataFrame({'word':words, 'freq':frequencies, 'category':categories})
    brown_df['nonalpha'] = False
    brown_df['nonalpha'][brown_df.word.str.contains('[^A-Za-z]')] = True
    brown_df.to_pickle(pickle_full_path)
    
else:
    print "Reading " + pickle_full_path
    brown_df = pd.read_pickle(pickle_full_path)
    
print "\nTail of dataframe:"
print brown_df.tail()

redo_brown_words = False # change to True to redo munge and rewrite pickle

pickle_full_path = data_path + "brown_words.pickle"

mean_median_full_path = data_path + "brown_words_mean_median.pickle"

if not os.path.isfile(pickle_full_path) or redo_brown_words == True:
    print 'Creating dataframe.'
    total_freq = 0
    weighted_length = 0
    brown_words = brown_df[brown_df.nonalpha == 0]
    brown_words = pd.DataFrame(brown_words.groupby(['word']).sum()).reset_index(drop=False)
    brown_words['length'] = 0
    brown_words.sort('freq', ascending = False, inplace=True)
    total_freq = brown_words.freq.sum()
    median_counter = total_freq / 2
    median_found = False
    for idx, row in brown_words.iterrows():
        curr_len = len(brown_words.word[idx])
        brown_words.length[idx] = curr_len
        weighted_length += brown_words.freq[idx] * curr_len
        median_counter -= curr_len * brown_words.freq[idx]
        if median_counter < 0 and median_found == False:
            wt_median_len = curr_len
            median_found = True
    brown_words = brown_words[['word', 'freq', 'length']]
    brown_words.to_pickle(pickle_full_path)
    wt_mean_len = weighted_length * 1.0 / total_freq
    pickle.dump((wt_mean_len, wt_median_len), open(mean_median_full_path, "w"))
    
else:
    print 'Reading pickle.'
    brown_words = pd.read_pickle(pickle_full_path)
    wt_mean_len, wt_median_len = pickle.load(open(mean_median_full_path, "r"))
    
print "Weighted mean word length: " + str(wt_mean_len)
print "Weighted median word length: " + str(wt_median_len)
print "Tail of brown_words dataframe:"
print brown_words.tail()

redo_europarl = True # change to True to redo munge and rewrite pickle

import time
start = time.time()

language = 'English' # This takes a long time -- Finnish was the longest, at 10.3 hours with a
                     # $500 Windows Acer desktop computer bought in 2013. Therefore I munged the files
                     # one by one instead of iterating over a list of them.

if language == 'English':

    file_path = init_path + europarl_filenames[language]
    pickle_path = data_path + 'europarl_' + language + ".pickle"
    
    if not os.path.isfile(pickle_path) or redo_europarl == True: 
        
        print "\nProcessing", language
        
        txt = open(file_path, 'rU').read()
        freqdist = nltk.FreqDist(nltk.word_tokenize(txt))
        
        europarl_df = pd.DataFrame()

        counter_max = len(freqdist.keys())
        counter_list = []
        for i in range(20):
            counter_list.append(int(i * counter_max / 20))
        counter = -1
        countdown = 100
        
        for token in freqdist.keys():
            counter += 1
            if counter in counter_list:
                print countdown,
                countdown -= 5
            token_mod = token.lower()
            for accent in accents:
                token_mod = re.sub(accent[0], accent[1], token_mod)
            if len(token_mod) > 1 and re.search('[aeiouy]', token_mod) and not re.search('[^a-z]', token_mod):
                df_to_append = pd.DataFrame({'language':[language], 'word':[token_mod], 'freq':[freqdist[token]]})
                europarl_df = europarl_df.append(df_to_append)
        
        europarl_df.to_pickle(pickle_path)
        
        print "\n%d minutes elapsed." % (int((time.time() - start) / 60))
        
print "Done."