diff --git a/.gitignore b/.gitignore index 0cd6b73..08730d5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ *.json *.json *.txt +.DS_Store +*.pyc diff --git a/__pycache__/settings_stats.cpython-37.pyc b/__pycache__/settings_stats.cpython-37.pyc new file mode 100644 index 0000000..ab0f9fd Binary files /dev/null and b/__pycache__/settings_stats.cpython-37.pyc differ diff --git a/assign_search.py b/assign_search.py new file mode 100644 index 0000000..e7e12a8 --- /dev/null +++ b/assign_search.py @@ -0,0 +1,1363 @@ +import sys +import csv +import tqdm +import psutil +import random +import time +from numpy import array, digitize, zeros, amin, amax, arange, percentile, histogram, argmin, transpose, delete, log, exp, nanmedian +from numpy import random as nrandom +from numpy import sum as nsum +from math import floor, ceil +from nltk import word_tokenize, pos_tag +import matplotlib.pyplot as plt +from copy import deepcopy +from re import match, sub, compile +import json +import os +import pandas as pd + +import gc +gc.enable() + +score_path = "E:/stimuli_coca_exp19/scores/" +mapping_path = "E:/stimuli_coca_exp19/mapping.json" + +def getSettings(): + + try: + num = int(input("How many bigrams do you want to find? ")) + except: + num = 100 + + try: + cuts = int(input("\nHow many bins should the scores be divided into? ")) + except: + cuts = 20 + + try: + disbalance_penalty = float(input("\nHow large a penalty should be given to imbalance (1-5)? ")) + except: + disbalance_penalty = 2 + + try: + max_time = int(input("\nWhat is the time limit for the search (in minutes)?")) + except: + max_time = 5 + print("\nDo you want to filter the bigrams by their POS-tags?\nWrite the RegEx to match (the tags are lower-cased): E.g. j.* to extract adjectives\nLeave empty to ignore") + pos_1 = input("\nWord 1: ") or ".*" + pos_2 = input("\nWord 2: ") or ".*" + + print("\nDo you want to filter the bigrams by the string?\nWrite the RegEx to match: E.g. [a-z].* to extract lower-case only\nLeave empty to ignore") + w_1 = input("\nWord 1: ") or ".*" + w_2 = input("\nWord 2: ") or ".*" + + try: + min_uni_freq = int(input("\nWhat should be the lowest unigram frequency included? ")) + except: + min_uni_freq = 0 + try: + min_bg_freq = int(input("\nWhat should be the lowest bigram frequency included? ")) + except: + min_bg_freq = 0 + try: + max_bg_freq = int(input("\nWhat should be the highest bigram frequency included? ")) + except: + max_bg_freq = 100000000 + + optimizer = "" + while optimizer not in ["quick", "pruning", "q", "p"]: + optimizer = input("\nWhich optimizer should be used? [(Q)uick/(P)runing] ") or "quick" + + if optimizer.startswith("q"): + optimizer = "quick" + else: + optimizer = "pruning" + + try: + seed = int(input("\nWhat should be the seed for random sampling? Leave empty if unsure, this will get you reproducible results. ")) + except: + seed = 1991 + + if optimizer == "quick": + try: + beam_width = int(input("\nHow many samples should be used in the sampling procedure? ")) + except: + beam_width = 1 + + else: + beam_width = 1000 + + return([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + +def saveSettings(settings): + + num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = settings + print("Do you want to save your settings? [y/N]") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + settings = "" + settings += "num = %i\n" % num + settings += "cuts = %i\n" % cuts + settings += "disbalance_penalty = %i\n" % disbalance_penalty + settings += 'w_1 ="' + w_1 + '"\n' + settings += 'w_2 ="' + w_2 + '"\n' + settings += 'pos_1 ="' + pos_1 + '"\n' + settings += 'pos_2 ="' + pos_2 + '"\n' + settings += "max_time = %i\n" % max_time + settings += "min_bg_freq = %i\n" % min_bg_freq + settings += "max_bg_freq = %i\n" % max_bg_freq + settings += "min_uni_freq = %i\n" % min_uni_freq + settings += 'optimizer = "' + optimizer + '"\n' + + with open("search_settings.py", "w+") as f: + f.write(settings) + +def getBins(it2score, cuts=20, type="dict", method = "exact", percent=95): + """Extract the range of scores. Take dict (or dict of dicts if specified), return lower/upper bound. Returns numpy array of edges.""" + if type == "dict": + try: + scores = [it2score[x] for x in it2score] # Get from dictionary to list + except: + print("Conversion not possible. Wrong type selected?") + elif type == "dictOfDicts": + scores = [it2score[x][y] for x in it2score for y in it2score[x]] # Get from dictionary to list + + else: + print("Dicts nested more than once not implemented") + raise ValueError() + + scores = array(scores) + + print(" Cropping extreme %i percent of data" % (100-percent)) + lower, upper = percentile(scores, [(100-percent)/2, ((100-percent)/2)+percent]) + scores = scores[(scores > lower) & (scores < upper)] + + if method == "log": + print(" Defining log-transform-based bin boundaries") + # bins = histogram_bin_edges(scores, cuts) + + translate = (0-amin(scores)) + 1 + #print(amin(scores)) + #print(amax(scores)) + scores = log(scores + translate) + + counts, bins = histogram(scores, min(cuts-2,1)) + # print(counts/sum(counts)) + bins = exp(bins) - translate + scores = exp(scores) - translate + + else: + print(" Defining bin boundaries") + # bins = histogram_bin_edges(scores, cuts) + + #translate = (0-amin(scores)) + 1 + #print(amin(scores)) + #print(amax(scores)) + #scores = log(scores + translate) + + counts, bins = histogram(scores, min(cuts-2,1)) + # print(counts/sum(counts)) + #bins = exp(bins) - translate + #scores = exp(scores) - translate + #plt.hist(scores, bins) + #plt.show() + + # x = input("Press RETURN to continue") + #print(bins) + return(bins) + +def getCorPerformance(scores, cutoff = 0.7): + scores = pd.DataFrame(scores) + cors = abs(scores.corr(method="spearman")) + + highs = sum(nsum(cors > cutoff)) + tiebreaker = nanmedian(cors[cors > cutoff]) + + return([highs, tiebreaker]) + +def getDisbalance(new, old, penalty=3, mod=1): + old = deepcopy(old) + for item in new: + old[arange(old.shape[0]), item] += 1*mod + m = sum(old[0,:])/old.shape[1] + old = old - m + old == old**penalty + return(nsum(old)) + +class Tagger: + def __init__(self, iterable_of_tagged): + self.tagging_map = {} + for tagged in iterable_of_tagged: + untagged = sub("_[^ ]+", "", tagged) + if untagged in self.tagging_map: + self.tagging_map[untagged].append(tagged) + else: + self.tagging_map[untagged] = [tagged] + + def tag(self, item_to_tag): + """Tag an item. If only one option is know, this is assumed to be the correct option. + If multiple options are known, the correct one can be picked or specified manually. + If no option is known, the tags should be written manually.""" + try: + tags = self.tagging_map[item_to_tag] + if len(tags) == 1: + return tags[0] + elif len(tags) > 1: + return self.pick_a_tag_from_selection(item_to_tag, tags) + except: + return self.tag_manually(item_to_tag) + + def pick_a_tag_from_selection(self, item_to_tag, tags): + """Allow the user to select one of multiple offered tags or specify their own tagging""" + print("_"*10) + print("There are multiple possible tags for '{}'".format(item_to_tag)) + for index, tag in enumerate(tags): + print("\t[{}] {}".format(index, tag)) + print("\n\t[m] a manual tag") + + choice = None; + while choice == None: + new_choice = input("\nPick the relevant tag\n> ") + if new_choice.lower() == "m": + return self.tag_manually(item_to_tag) + try: + new_choice = int(new_choice) + if (new_choice >= 0) and (new_choice < len(tags)): + choice = new_choice + except: + pass + + return tags[choice] + + def tag_manually(self, item_to_tag): + """Allow manual tagging of items not found in the simple tagging map. + Expects the user to input valid tags""" + print("_"*10) + print("The item '{}' cannot be tagged semi-automatically. Please tag it manually.".format(item_to_tag)) + w1, w2 = item_to_tag.split() + + # TODO: input validation/formatting (lowercase?) + w1_tag = input("{}_".format(w1)) + w2_tag = input("{}_".format(w2)) + + return "{}_{} {}_{}".format(w1,w1_tag,w2,w2_tag) + + + +class Scorer(object): + """Loads all the score lists, can then be used to assign the scores on a per-item base with the score method. + It is memory-heavy, but could be included in functions which allow interactive collocation input.""" + + def __init__(self): + """Load the required score files. If they are not present in the folder, throw an exception.""" + self.binning = "exact" + self.optimizer = "pruning" + self.beam_width = 5 + import os + #reduced options (to save RAM): + files = ["wfreqs.json", "bigrams.json","fwd.json","bckw.json","llscore.json", "tscore.json", "miscore.json","mi3score.json"] + #full options: + #files = ["wfreqs.json", "bigrams.json","fwd.json","bckw.json","llscore.json", "dicescore.json", "moddicescore.json","tscore.json","zscore.json", "delta_p21.json", "delta_p12.json", "miscore.json","mi3score.json","gscore.json"] + files = [score_path + x for x in files] + try: + filecheck = [os.path.isfile(f) for f in files] + if all(filecheck) != True: + raise IOError() + + except: + print("\nFollowing files could not be loaded. Check that they are in the /scores subfolder as this script.") + for i in range(len(files)): + if filecheck[i]==False: + print(files[i]) + print("Exiting") + sys.exit(1) + + #try: + print("\nLoading the saved scores") + print(" unigram frequency") + + file_folder = score_path + with open(file_folder + "wfreqs.json", "r") as i: + self.wfreq = json.loads(i.read()) + + print(" bigram frequency") + with open(file_folder + "bigrams.json", "r") as i: + self.bg_frq = json.loads(i.read()) + + # print(" TP-D") + # with open(file_folder + "fwd.json", "r") as i: + # self.tp_d = json.loads(i.read()) + # + # print(" TP-B") + # with open(file_folder + "bckw.json", "r") as i: + # self.tp_b = json.loads(i.read()) + # + # print(" Log likelihood") + # with open(file_folder + "llscore.json", "r") as i: + # self.log_lklhd = json.loads(i.read()) + + # print(" Dice") + # with open(file_folder + "dicescore.json", "r") as i: + # self.dice = json.loads(i.read()) + + # print(" Modified dice") + # with open(file_folder + "moddicescore.json", "r") as i: + # self.moddice = json.loads(i.read()) + + # print(" t-score") + # with open(file_folder + "tscore.json", "r") as i: + # self.t_score = json.loads(i.read()) + + # print(" z-score") + # with open(file_folder + "zscore.json", "r") as i: + # self.z_score = json.loads(i.read()) + + # print(" delta_p-12") + # with open(file_folder + "delta_p12.json", "r") as i: + # self.delta_p12 = json.loads(i.read()) + + # print(" delta_p-21") + # with open(file_folder + "delta_p21.json", "r") as i: + # self.delta_p21 = json.loads(i.read()) + + # print(" MI-score") + # with open(file_folder + "miscore.json", "r") as i: + # self.mi_score = json.loads(i.read()) + # + # print(" MI3-score") + # with open(file_folder + "mi3score.json", "r") as i: + # self.mi3_score = json.loads(i.read()) + + # print(" G-score") + # with open(file_folder + "gscore.json", "r") as i: + # self.g_score = json.loads(i.read()) + print("_________________________________") + + # except Exception as e: + # print("Not all score files could be loaded. Check that they are in the same folder as this script.") + # print("Exiting") + # print(e) + # sys.exit(1) + + def score(self, items): + """Score all the bigrams at once. Input is a list of lists with the sublist format + [item1, item2]. If a bigram is not in the score list, return NA.""" + + items_out = [] + for bigram in items: + + w1, w2 = bigram + bigram = " ".join(bigram) + + try: + w1_frq = self.wfreq[w1] + except: + w1_frq = "NA" + + try: + w2_frq = self.wfreq[w2] + except: + w2_frq = "NA" + + try: + bg_frq = self.bg_frq[bigram] + except: + bg_frq = "NA" + + # try: + # tp_d = self.tp_d[w1][w2] + # except: + # tp_d = "NA" + # + # try: + # tp_b = self.tp_b[w2][w1] + # except: + # tp_b = "NA" + # + # try: + # log_lklhd = self.log_lklhd[bigram] + # except: + # log_lklhd = "NA" + + # try: + # dice = self.dice[bigram] + # except: + # dice = "NA" + # + # try: + # moddice = self.moddice[bigram] + # except: + # moddice = "NA" + # + # try: + # t_score = self.t_score[bigram] + # except: + # t_score = "NA" + + # try: + # z_score = self.z_score[bigram] + # except: + # z_score = "NA" + # + # try: + # mi_score = self.mi_score[bigram] + # except: + # mi_score = "NA" + # + # try: + # mi3_score = self.mi3_score[bigram] + # except: + # mi3_score = "NA" + + # try: + # g_score = self.g_score[bigram] + # except: + # g_score = "NA" + # + # try: + # delta_p12 = self.delta_p12[bigram] + # except: + # delta_p12 = "NA" + # + # try: + # delta_p21 = self.delta_p21[bigram] + # except: + # delta_p21 = "NA" + + #items_out.append([bigram, w1_frq, w2_frq, bg_frq, tp_b, tp_d, log_lklhd, dice, moddice, t_score, z_score, mi_score, mi3_score, g_score, delta_p12, delta_p21]) + #items_out.append([bigram, w1_frq, w2_frq, bg_frq, tp_b, tp_d, log_lklhd, t_score, mi_score, mi3_score]) + #reduced total (save ram): + items_out.append([bigram, w1_frq, w2_frq, bg_frq]) + + return(items_out) + + + def score_one(self, bigram): + """Score the individual bigrams. Input is in the format + [item1, item2]. If a bigram is not in the score list, return NA.""" + + w1, w2 = bigram + bigram = " ".join(bigram) + + try: + w1_frq = self.wfreq[w1] + except: + w1_frq = "NA" + + try: + w2_frq = self.wfreq[w2] + except: + w2_frq = "NA" + + try: + bg_frq = self.bg_frq[bigram] + except: + bg_frq = "NA" + + try: + tp_d = self.tp_d[w1][w2] + except: + tp_d = "NA" + + try: + tp_b = self.tp_b[w2][w1] + except: + tp_b = "NA" + + try: + log_lklhd = self.log_lklhd[bigram] + except: + log_lklhd = "NA" + + try: + dice = self.dice[bigram] + except: + dice = "NA" + + try: + moddice = self.moddice[bigram] + except: + moddice = "NA" + + try: + t_score = self.t_score[bigram] + except: + t_score = "NA" + + try: + z_score = self.z_score[bigram] + except: + z_score = "NA" + + try: + mi_score = self.mi_score[bigram] + except: + mi_score = "NA" + + try: + mi3_score = self.mi3_score[bigram] + except: + mi3_score = "NA" + + try: + g_score = self.g_score[bigram] + except: + g_score = "NA" + + try: + delta_p12 = self.delta_p12[bigram] + except: + delta_p12 = "NA" + + try: + delta_p21 = self.delta_p21[bigram] + except: + delta_p21 = "NA" + + return([bigram, w1_frq, w2_frq, bg_frq, tp_b, tp_d, log_lklhd, dice, moddice, t_score, z_score, mi_score, mi3_score, g_score, delta_p12, delta_p21]) + + def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=20, words = ["", ""], pos=["",""], + max_bg_freq=100000000, min_bg_freq=0, min_uni_freq=5, percent=95): + num = max(num,1) + """Get num random bigrams, spread approximately evenly accross the ranges of the scores.""" + print("\nFinishing initialization") + ### FILTERING BY POS + + self.bgs_all = [x.split() for x in self.bg_frq if (" ".join(x.split()) in self.bg_frq)] + + if (min_bg_freq > 0) or (max_bg_freq < 100000000): + print(" Selecting bigrams by bigram frequency") + print(" Window: %.2f - %.2f" %(min_bg_freq, max_bg_freq)) + self.bgs = [[x,y] for x,y in self.bgs_all if (self.bg_frq[" ".join([x,y])] >= min_bg_freq) and (self.bg_frq[" ".join([x,y])] <= max_bg_freq)] + else: + self.bgs = self.bgs_all + + if min_uni_freq > 1: + print(" Selecting bigrams by unigram frequency") + self.bgs = [[x,y] for x,y in self.bgs if (self.wfreq[x]>=min_uni_freq and self.wfreq[y]>=min_uni_freq)] + else: + self.bgs = self.bgs + + if pos[0] != "" or pos[1] != "": + pos_1, pos_2 = pos + pos_1 = compile(pos_1) + pos_2 = compile(pos_2) + print(" Selecting bigrams by POS") + self.bgs = [[x,y] for x,y in self.bgs if (match(pos_1, x.split("_")[1])!=None and match(pos_2,y.split("_")[1])!=None)] + + if words[0] != "" or words[1] != "": + w_1, w_2 = words + w_1 = compile(w_1) + w_2 = compile(w_2) + print(" Selecting bigrams by words 1 & 2") + self.bgs = [[x,y] for x,y in self.bgs if (match(w_1, x.split("_")[0])!=None and match(w_2,y.split("_")[0])!=None)] + + print("_"*10+"\nRelevant bigrams: %i" % len(self.bgs)) + + # print(" Cleaning unfitting bigrams") + # print(" Initializing cleaner") + + # tempbgs = set([" ".join(x) for x in self.bgs]) + # wrong_keys = [x for x in tqdm.tqdm(self.bg_frq) if not x in tempbgs] + # del tempbgs + # print(" Cleaning") + # wrong_keys = [] + # for wrong_key in tqdm.tqdm(wrong_keys): + # try: + # c = wrong_key + # wrong_key = wrong_key.split() + # del self.bg_frq[c] + # del self.tp_d[wrong_key[0]][wrong_key[1]] + # del self.tp_b[wrong_key[1]][wrong_key[0]] + # del self.log_lklhd[c] + # del self.dice[c] + # del self.t_score[c] + # del self.z_score[c] + # del self.mi_score[c] + # del self.mi3_score[c] + # del self.g_score[c] + # del self.delta_p12[c] + # del self.delta_p21[c] + # except: + # pass + + # del wrong_keys + + if (len(self.bgs) <= num) and len(self.bgs) > 0: + print("Random sampling failed: There are fewer relevant cases than the sample size.") + scores = [self.score_one(x) for x in self.bgs] # remove bigram string to allow numpy operation + scores = array([x[1:] for x in scores]) + results = [[" ".join(self.bgs[x])]+list(scores[x,:]) for x in range(min(len(self.bgs), num))] + # results = [[sub("_[^ ]+",""," ".join(self.bgs[x]))]+list(scores[x,:]) for x in range(min(len(self.bgs), num))] + return(results) + + elif len(self.bgs) == 0: + raise ValueError("No case matches the criteria.") + + print("\nDefining data distributions") + + print(" unigram frequency") + self.wfreq_bins = getBins(self.wfreq, cuts=cuts, method = self.binning, percent=percent) + + print(" bigram frequency") + self.bg_frq_bins = getBins(self.bg_frq, cuts=cuts, method = self.binning, percent=percent) # Get from dictionary to list + + + if self.optimizer == "pruning": + print(" TP-D") + + self.tp_d_bins = getBins(self.tp_d, cuts=cuts, type="dictOfDicts", percent=percent) # Get from dictionary to list + + print(" TP-B") + self.tp_b_bins = getBins(self.tp_b, cuts=cuts, type="dictOfDicts", percent=percent) # Get from dictionary to list + + print(" Log likelihood") + self.log_lklhd_bins = getBins(self.log_lklhd, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" Dice") + self.dice_bins = getBins(self.dice, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" Modified dice") + self.moddice_bins = getBins(self.moddice, cuts=cuts, percent=percent) + + print(" t-score") + self.t_score_bins = getBins(self.t_score, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" z-score") + self.z_score_bins = getBins(self.z_score, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" MI-score") + self.mi_score_bins = getBins(self.mi_score, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" MI3-score") + self.mi3_score_bins = getBins(self.mi3_score, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" G-score") + self.g_score_bins = getBins(self.g_score, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" Delta_p-12") + self.delta_p12_bins = getBins(self.delta_p12, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" Delta_p-21") + self.delta_p21_bins = getBins(self.delta_p21, cuts=cuts, percent=percent) # Get from dictionary to list + + print("_________________________________") + + buf = 10 # The bufferring coefficient (how many extra elements should be collected) + self.dist = zeros([15, cuts]) # Array to save the distributions: columns=bins, rows=scores + self.results = zeros([num*buf, 15]) # Array to save the results + self.items = [] + self.populated = 0 + self.disbalance_penalty = floor(num*disbalance_penalty/100) + # self.indexes = arange(length(self.bg_pos)) + + random.seed(seed) + nrandom.seed(seed) + + print("\nStarting item selection") + + start = time.time() + lasttime = time.time() + + max_time = max_time*60 + # pbar = [tqdm.tqdm(total = num), tqdm.tqdm(total=num*(buf-1))] + # pbar[0].set_description("Stimuli collected") + # pbar[1].set_description("Additional buffer") + reached = False + + while (self.populated < num*buf) and (time.time() - start) < max_time: # Get a random item; check which bins would it increase for which score, if this disturbs balance, drop otherwise insert at the bottom + if self.populated >= num and reached == False: + reached = True + + samples = [nrandom.choice(range(len(self.bgs)),10, replace=False) for x in range(1000)] + samples = [[self.bgs[y] for y in x] for x in samples] + scores = [self.score(x) for x in samples] # remove bigram string to allow numpy operation + scores = [array([x[1:] for x in y]) for y in scores] + + binned = [array([digitize(x[:,0],self.bg_frq_bins), + digitize(x[:,1],self.wfreq_bins), + digitize(x[:,2],self.wfreq_bins), + digitize(x[:,3],self.tp_b_bins), + digitize(x[:,4],self.tp_d_bins), + digitize(x[:,5],self.log_lklhd_bins), + digitize(x[:,6],self.dice_bins), + digitize(x[:,7],self.moddice_bins), + digitize(x[:,8],self.t_score_bins), + digitize(x[:,9],self.z_score_bins), + digitize(x[:,10],self.mi_score_bins), + digitize(x[:,11],self.mi3_score_bins), + digitize(x[:,12],self.g_score_bins), + digitize(x[:,13],self.delta_p12_bins), + digitize(x[:,14],self.delta_p21_bins)]) for x in scores] + + # print(binned[0]) + binned = [transpose(x) for x in binned] + # binned = [] + performance = [getDisbalance(x, self.dist, self.disbalance_penalty) for x in binned] + best = argmin(performance) + + self.results[self.populated:self.populated+10,:] = scores[best] + for item in binned[best]: + self.dist[arange(15), item] += 1 + self.items += samples[best] + self.populated += 10 + # if reached: + # pbar[1].update(10) + # else: + # pbar[0].update(10) + + # pbar[0].close() + # pbar[1].close() + + if self.populated >= num and (time.time() - start) < max_time: + print("\nPruning") + + binned = array([digitize(self.results[:,0],self.bg_frq_bins), + digitize(self.results[:,1],self.wfreq_bins), + digitize(self.results[:,2],self.wfreq_bins), + digitize(self.results[:,3],self.tp_b_bins), + digitize(self.results[:,4],self.tp_d_bins), + digitize(self.results[:,5],self.log_lklhd_bins), + digitize(self.results[:,6],self.dice_bins), + digitize(self.results[:,7],self.moddice_bins), + digitize(self.results[:,8],self.t_score_bins), + digitize(self.results[:,9],self.z_score_bins), + digitize(self.results[:,10],self.mi_score_bins), + digitize(self.results[:,11],self.mi3_score_bins), + digitize(self.results[:,12],self.g_score_bins), + digitize(self.results[:,13],self.delta_p12_bins), + digitize(self.results[:,14],self.delta_p21_bins)]) + binned = transpose(binned) + + print("Removing duplicates") + firsts = set() + seconds = set() + dels = [] + for x in [y for y in range(self.populated)]: + w1,w2 = self.items[x] + if w1 in firsts or w2 in seconds: + dels.append(x) + else: + firsts.update(w1) + seconds.update(w2) + + dels.reverse() + for d_index in tqdm.tqdm(dels): + del self.items[d_index] + binned = delete(binned, dels,0) + self.results = delete(self.results, dels, 0) + self.populated -= len(dels) + + # pbar = tqdm.tqdm(total=self.populated - num) + while self.populated > num and (time.time() - start) < max_time: # If there is time left, prune the most problematic items away, one by one + rands = [random.randint(0,binned.shape[0]-1) for x in range(1000)] + performance = [getDisbalance(binned[i,:], self.dist, self.disbalance_penalty, mod=-1) for i in rands] + best = rands[argmin(performance)] + self.dist[arange(15), binned[best,:]] -= 1 + binned = delete(binned, best, 0) + del self.items[best] + self.results = delete(self.results, best, 0) + self.populated -= 1 + # pbar.update(1) + + print("\nSuccess! All %i items were found." % num) + + + else: + print("\nTimeout limit exceeded. Returning %i items" % self.populated) + results = self.results[0:min(self.populated, num)] # Crop which we don't have + + #plt.imshow(self.dist, cmap="hot", interpolation="bilinear") + #plt.suptitle("Distribution accross scores") + #plt.xlabel("Score bin") + #plt.ylabel("Score") + #plt.show() + + results = [[sub("_[^ ]+",""," ".join(self.items[x]))]+list(self.results[x, 0:15]) for x in range(min(self.populated, num))] + + return(results) + + if self.optimizer == "quick": + + beam = self.beam_width # The beam width to keep best samples + self.dist = zeros([15, cuts]) # Array to save the distributions: columns=bins, rows=scores + self.results = zeros([num, 15]) # Array to save the results + self.items = [] + + random.seed(seed) + nrandom.seed(seed) + + print("\nStarting item selection") + + start = time.time() + + max_time = max_time*60 + + + best = 15**2 + tiebreak = 1 + best_sample = [] + + # pbar = tqdm.tqdm(total = beam) + # pbar.set_description("Samples tried") + for sample in range(beam): + sample = list(nrandom.choice([x for x in range(len(self.bgs))], min(floor(num*1.1), len(self.bgs)), replace=False)) + sample = [self.bgs[x] for x in sample] + + scores = self.score(sample) # remove bigram string to allow numpy operation + scores = array([x[1:] for x in scores]) + performance, tiebreaker = getCorPerformance(scores) + + if performance < best: + best = performance + best_sample = sample + tiebreak = tiebreaker + elif (performance == best) and (tiebreaker < tiebreak): + best = performance + best_sample = sample + tiebreak = tiebreaker + + # pbar.update(1) + if (time.time() - start) >= max_time: + break + + # pbar.close() + + # print(best_sample) + if (time.time() - start) < max_time: + print("Removing duplicates") + firsts = set() + seconds = set() + dels = [] + for x in [y for y in range(len(best_sample))]: + w1,w2 = best_sample[x] + if w1 in firsts or w2 in seconds: + dels.append(x) + else: + firsts.update(w1) + seconds.update(w2) + + dels.reverse() + for d_index in tqdm.tqdm(dels): + del best_sample[d_index] + + results = best_sample[0:num] + + else: + print("\nTimeout limit exceeded, returning best sample at this moment.") + results = best_sample[0:min(num, len(best_sample))] # Crop which we don't have + + #plt.imshow(self.dist, cmap="hot", interpolation="bilinear") + #plt.suptitle("Distribution accross scores") + #plt.xlabel("Score bin") + #plt.ylabel("Score") + #plt.show() + + results = self.score(best_sample) + results = pd.DataFrame(results, columns = ["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "dice", "moddice", "t_score", "z_score", "mi_score", "mi3_score", "g_score", "delta_p12", "delta_p21"]) + # results["bigram"] = [sub("_[^ ]+","",x) for x in results["bigram"]] + results = results.values + # print(results) + return(results) + +if __name__ == "__main__": + #mode = None + mode = "score" + while mode not in ["score", "search", "strat_search", "match"]: + mode = input("Which mode should this program run in?\n Score/search/strat_search/match: ") + + if mode.lower() == "score": + + if len(sys.argv) > 1: + inpath = sys.argv[1] + else: + inpath = "stimuli_list.txt" + #inpath = input("Where is the file to load?\n ") + + try: + with open(inpath, "r") as infile: + items = infile.readlines() + items = [x.split() for x in items] + print("First read in items: " + str(items)) + if all([len(x)==2 for x in items]) == False: + raise IOError() + + except: + print("The input file does not seem to be formatted correctly (one bigram per line)") + sys.exit(2) + + ram_present = psutil.virtual_memory()[0] >> 30 + ram_available = psutil.virtual_memory()[1] >> 30 + + # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version + if ram_present > 7 and ram_available > 5: + pass + else: + print("This is a RAM-intensive operation. You need at least 6 GB of free RAM.") + print("Exiting...") + sys.exit(0) + + from assign_search_lemma import ScorerLemma + with open(mapping_path, "r") as f: + mapping = json.load(f) + tagger = Tagger(mapping) + + if match("[^_]+_", " ".join(items[0])) == None: + tagged_items = [] + for item in items: + item = " ".join(item) + try: + tagged_item = tagger.tag(item) + tagged_items.append(tagged_item.split()) + except Exception as e: + print(str(e) + str(item)) + + + #tagger = {sub("_[^ ]+", "", k):k for k in mapping} + #tagger = {} + #for k in mapping: + #untagged = sub("_[^ ]+", ", k") + #if untagged in tagger + #tagger[untagged].append(tagged) #tagged = k + #else: + #tagger[untagged] = [tagged] + + # tagging + #for item in items: + #tags = tagger[item] + # filter only for Adj - N combinations + # if len(`filtered`) == 1 --> tag found + # else decide manually + + #mapping = {sub("_[^ ]+", "", k):v for k,v in mapping.items()} + + #if match("[^_]+_", " ".join(items[0])) == None: + #new_items = [] + #for x in items: + #try: + #new_items.append(tagger[" ".join(x)].split()) + #except: + #print("Error: " + str(x)) + + # if match("[^_]+_", " ".join(items[0])) == None: + # try: + # items = [tagger[" ".join(x)].split() for x in items] + # except KeyError: + # print("test") + + print("Second step items: " + str(tagged_items)) + scorer = Scorer() + items = scorer.score(tagged_items) + print("Third step items: " + str(items)) + + #scores = pd.DataFrame(items, columns=["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "dice", "moddice", "t_score", "z_score", "mi_score", "mi3_score", "g_score", "delta_p12", "delta_p21"]) + #scores = pd.DataFrame(items, columns=["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "t_score", "mi_score", "mi3_score"]) + scores = pd.DataFrame(items, columns=["bigram", "w1_freq", "w2_freq", "bigram_freq"]) + + del scorer + del items + + tagged_lemmas = [] + for x in scores["bigram"]: + try: + tagged_lemmas.append(mapping[x]) + except KeyError: + tagged_lemmas.append("NA NA") + + scores["bigram_lemma"] = pd.Series(tagged_lemmas) + + scorer = ScorerLemma() + lemmascores = pd.DataFrame(scorer.score([x.split() for x in scores["bigram_lemma"]]), columns=["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + #lemmascores = pd.DataFrame(scorer.score([x.split() for x in scores["bigram_lemma"]]), columns=["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "t_score_lemma", "mi_score_lemma", "mi3_score_lemma"]) + lemmascores.drop_duplicates().reset_index(drop=True) + lemmascores["bigram_lemma"] = lemmascores["bigram_lemma"].astype(str) + + + scores = scores.merge(lemmascores, on="bigram_lemma", how="outer") + scores.drop_duplicates().reset_index(drop=True) + + print(scores) + + scores["bigram_tag"] = scores["bigram"] + scores["bigram_lemma_tag"] = scores["bigram_lemma"] + scores["bigram"] = [sub("_[^ ]+","",x) for x in scores["bigram"]] + scores["bigram_lemma"] = [sub("_[^ ]+","",x) for x in scores["bigram_lemma"]] + print("Saving") + + if len(sys.argv) > 2: + outpath = sys.argv[2] + else: + outpath = input("Where should the results be saved?\n ") + + print("Saving") + scores.to_csv(outpath, index=False) + print("Done. Press RETURN to exit") + wait = input() + sys.exit(0) + + elif mode.lower() == "search": + + ram_present = psutil.virtual_memory()[0] >> 30 + ram_available = psutil.virtual_memory()[1] >> 30 + + # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version + if ram_present > 7: + # if ram_present > 7 and ram_available > 5: + pass + else: + print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") + sys.exit(0) + + max_time = 0 + disbalance_penalty = 0 + cuts = 0 + num = 0 + pos = "" + + saved = False + if os.path.isfile("search_settings.py"): + print("Saved settings found. Do you want to use them? [Y/n]") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[Y/n]") or "Y" + + if dec.lower() == "y": + saved = True + + if saved == True: + from search_settings import * + print("Using saved settings") + + else: + num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = getSettings() + + scorer = Scorer() + scorer.optimizer = optimizer + items = scorer.get_random(num, cuts=cuts, seed=seed, disbalance_penalty = disbalance_penalty, words=[w_1, w_2], pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=min_bg_freq, max_bg_freq=max_bg_freq, min_uni_freq=min_uni_freq) + + scores = pd.DataFrame(items, columns=["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "dice", "moddice", "t_score", "z_score", "mi_score", "mi3_score", "g_score", "delta_p12", "delta_p21"]) + + del scorer + del items + + from assign_search_lemma import ScorerLemma + with open(mapping_path, "r") as f: + mapping = json.load(f) + + scores["bigram_lemma"] = [mapping[x] for x in scores["bigram"]] + scorer = ScorerLemma() + lemmascores = pd.DataFrame(scorer.score([x.split() for x in scores["bigram_lemma"]]), columns=["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + + scores = scores.merge(lemmascores, on=["bigram_lemma"]) + + print("Saving") + + if len(sys.argv) > 2: + outpath = sys.argv[2] + else: + outpath = input("Where should the results be saved?\n ") + + + print("Saving") + scores.to_csv(outpath, index = False) + + saveSettings([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + + elif mode.lower() == "strat_search": + + if len(sys.argv) > 2: + outpath = sys.argv[2] + else: + outpath = input("Where should the results be saved?\n ") + + with open("_temp.csv", "w+") as outfile: + out_csv = csv.writer(outfile) + out_csv.writerow(["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "dice", "moddice", "t_score", "z_score", "mi_score", "mi3_score", "g_score", "delta_p12", "delta_p21"]) + + ram_present = psutil.virtual_memory()[0] >> 30 + ram_available = psutil.virtual_memory()[1] >> 30 + + # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version + if ram_present > 7: + # if ram_present > 7 and ram_available > 5: + pass + else: + print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") + sys.exit(0) + + max_time = 0 + disbalance_penalty = 0 + cuts = 0 + num = 0 + pos = "" + + saved = False + if os.path.isfile("search_settings.py"): + print("Saved settings found. Do you want to use them? [Y/n]") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[Y/n]") or "Y" + + if dec.lower() == "y": + saved = True + + if saved == True: + from search_settings import * + print("Using saved settings") + + else: + num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = getSettings() + + scorer = Scorer() + + scorer.beam_width = beam_width + scorer.binning = "exact" + items = scorer.get_random(1, cuts=10, seed=seed, disbalance_penalty = disbalance_penalty, words=[w_1, w_2], + pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=min_bg_freq, max_bg_freq=max_bg_freq, min_uni_freq=min_uni_freq, percent=95) + + bins = list(scorer.bg_frq_bins) + [max_bg_freq] + bins[0] = [min_bg_freq] + + scorer.binning = "exact" + scorer.optimizer = optimizer + print(bins) + iter = 0 + for strat in tqdm.tqdm(range(len(bins)-1)): + items = scorer.get_random(floor((num*1.5)/(len(bins)-1)), cuts=cuts, seed=seed+iter, disbalance_penalty = disbalance_penalty, words=[w_1, w_2], + pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=floor(bins[strat]), max_bg_freq=ceil(bins[strat+1])+1, min_uni_freq=min_uni_freq, percent=100) + iter +=1 + + with open("_temp.csv", "a") as outfile: + out_csv = csv.writer(outfile) + for i in tqdm.tqdm(items): + out_csv.writerow(i) + + print("Saving") + scores = pd.read_csv("_temp.csv") + scores["w1"], scores["w2"] = scores["bigram"].str.split(' ', 1).str + scores.index = range(scores.shape[0]) + scores = scores.drop_duplicates(subset=["bigram"], keep="last") + scores = scores.drop_duplicates(subset=["w1"], keep="last") + scores = scores.drop_duplicates(subset=["w2"], keep="last") + scores.drop(["w1", "w2"], 1, inplace=True) + + if scores.shape[0] > num: + ints = sorted(nrandom.choice(range(scores.shape[0]), num, replace=False)) + scores = scores.iloc[ints,:] + + del scorer + from assign_search_lemma import ScorerLemma + with open(mapping_path, "r") as f: + mapping = json.load(f) + + scores["bigram_lemma"] = [mapping[x] for x in scores["bigram"]] + scorer = ScorerLemma() + lemmascores = pd.DataFrame(scorer.score([x.split() for x in scores["bigram_lemma"]]), columns=["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + + scores = scores.merge(lemmascores, on=["bigram_lemma"]) + + scores.to_csv(outpath, index=False) + try: + os.remove("_temp.csv") + except: + print("Couldn't remove the temp file.") + + saveSettings([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + + sys.exit(0) + + elif mode.lower() == "match": + infile = input("Where is the file with selected stimuli? ") or "not_a_file" + while os.path.isfile(infile) != True: + infile = input("Not a valid file, try to specify the full path. ") or "not_a_file" + + stimuli = pd.read_csv(infile) + stimuli.dropna(0, how="all", subset=["bigram"], inplace=True) + + try: + matchWord = int(input("Which word should be matched? [1-%i]" % len(stimuli["bigram"][0].split()))) - 1 + except: + matchWord = 0 + + # try: + # min_t_score = int(input("What is the lowest accepted t-score?")) + # except: + # min_t_score = -1000000 + # + # try: + # max_t_score = int(input("What is the highest accepted t-score?")) + # except: + # max_t_score = +1000000 + + try: + window = float(input("What is the bigram frequency window size?")) + except: + window = 0 + + + keys = [x.split()[matchWord] for x in list(stimuli["bigram"].values)] + locks = [x for x in list(stimuli["bigram"].values)] + locks2 = set([x.split()[1] for x in locks]) + bgfreqs = [x for x in list(stimuli["bigram_freq"].values)] + keys = [x for x in zip(keys, bgfreqs)] + + non_matched = 0 if matchWord==1 else 1 + forbidden = set([x.split()[non_matched] for x in list(stimuli["bigram"].values)]) + + if len(sys.argv) > 2: + outpath = sys.argv[2] + else: + outpath = input("Where should the results be saved?\n ") + + with open("_temp.csv", "w+") as outfile: + out_csv = csv.writer(outfile) + out_csv.writerow(["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "dice", "moddice", "t_score", "z_score", "mi_score", "mi3_score", "g_score", "delta_p12", "delta_p21"]) + + ram_present = psutil.virtual_memory()[0] >> 30 + ram_available = psutil.virtual_memory()[1] >> 30 + + # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version + if ram_present > 7: + # if ram_present > 7 and ram_available > 5: + pass + else: + print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") + sys.exit(0) + + max_time = 0 + disbalance_penalty = 0 + cuts = 0 + num = 0 + pos = "" + + saved = False + if os.path.isfile("search_settings.py"): + print("Saved settings found. Do you want to use them? [Y/n]") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[Y/n]") or "Y" + + if dec.lower() == "y": + saved = True + + if saved == True: + from search_settings import * + print("Using saved settings") + + else: + num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = getSettings() + + print("\n_____________________\nGrab a coffee...or ten") + + scorer = Scorer() + ##### + # Prefilter the data for only the adjectives? + # Prefilter the data by t-score? + ##### + + print("Pre-filtering the data for\n\t-matched words\n\t-fitting t-score") + scorer.bg_frq = {k:v for k,v in tqdm.tqdm(scorer.bg_frq.items()) if (k.split(" ")[matchWord].split("_")[0] in [x[0] for x in keys]) and ((sub("_[^ ]+","",k) not in locks) and (k.split()[1].split("_")[0] not in locks2))} + print(" Prefiltered: %i" % len(scorer.bg_frq)) + # scorer.bg_frq = {k:v for k,v in tqdm.tqdm(scorer.bg_frq.items()) if (k.split(" ")[matchWord].split("_")[0] in keys) and ((scorer.t_score[k] >= min_t_score) and (scorer.t_score[k] <= max_t_score))} + + scorer.beam_width = beam_width + scorer.binning = "exact" + scorer.optimizer = optimizer + + iter = 0 + fails = [] + for key, freq in tqdm.tqdm(keys): + print(key) + + try: + matched_pattern = [w_1, w_2] + matched_pattern[matchWord] = key + freq = log(freq) + print("Window: %.2f - %.2f" % (exp(freq-window*.5), exp(freq+window*.5))) + + items = scorer.get_random(num, cuts=cuts, seed=seed+iter, disbalance_penalty = disbalance_penalty, words=matched_pattern, + pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=exp(freq-window*.5), max_bg_freq=exp(freq+window*.5), min_uni_freq=min_uni_freq, percent=95) + + iter +=1 + with open("_temp.csv", "a") as outfile: + out_csv = csv.writer(outfile) + for i in tqdm.tqdm(items): + out_csv.writerow(i) + except Exception as e: + fails.append(key) + + print("Saving") + scores = pd.read_csv("_temp.csv") + if scores.shape[0] > 0: + scores["w1"], scores["w2"] = scores["bigram"].str.split(' ', 1).str + scores.index = range(scores.shape[0]) + scores = scores.drop_duplicates(subset=["bigram"], keep="last") + + if non_matched == 0: + scores = scores[~scores["w1"].isin(forbidden)] + else: + scores = scores[~scores["w2"].isin(forbidden)] + + keys_dict = {word[0]:bigram_freq for word, bigram_freq in keys} + print(keys_dict) + print(scores.head()) + scores["fordrop"] = scores["bigram_freq"].values + for item in keys_dict: + center = keys_dict[item] + scores.loc[scores["w1"]==item, "fordrop"] = abs(scores.loc[scores["w1"]==item, "fordrop"] - center) + scores.sort_values(by=["fordrop"], inplace=True, ascending=False) + #scores = scores.groupby(["w1"]).tail(2) + scores.drop_duplicates(subset=["w2"], keep="last", inplace=True) + scores.drop_duplicates(subset=["w1"], keep="last", inplace=True) + + # scores = scores[""] + + + scores.drop(["w1", "w2", "fordrop"], 1, inplace=True) + + # if scores.shape[0] > num: + # ints = sorted(nrandom.choice(range(scores.shape[0]), num, replace=False)) + # scores = scores.iloc[ints,:] + + del scorer + del items + # + from assign_search_lemma import ScorerLemma + with open(mapping_path, "r") as f: + mapping = json.load(f) + + scores["bigram_lemma"] = [mapping[x] for x in scores["bigram"]] + scorer = ScorerLemma() + lemmascores = pd.DataFrame(scorer.score([x.split() for x in scores["bigram_lemma"]]), columns=["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + + scores = scores.merge(lemmascores, on=["bigram_lemma"]) + scores["bigram_tag"] = scores["bigram"].values + scores["bigram"] = scores["bigram"].str.replace("_[^ ]+", "") + scores["bigram_lemma_tag"] = scores["bigram_lemma"].values + scores["bigram_lemma"] = scores["bigram_lemma"].str.replace("_[^ ]+", "") + + scores = scores[['bigram', 'bigram_tag', 'bigram_lemma', 'bigram_lemma_tag', 'w1_freq', 'w2_freq', 'bigram_freq', 'tp_b', 'tp_d', 'log_lklhd', 'dice', 'moddice', 't_score', 'z_score', 'mi_score', 'mi3_score', 'g_score', 'delta_p12', 'delta_p21', 'w1_freq_lemma', 'w2_freq_lemma', 'bigram_freq_lemma', 'tp_b_lemma', 'tp_d_lemma', 'log_lklhd_lemma', 'dice_lemma', 'moddice_lemma', 't_score_lemma', 'z_score_lemma', 'mi_score_lemma', 'mi3_score_lemma', 'g_score_lemma', 'delta_p12_lemma', 'delta_p21_lemma']] + scores.drop_duplicates(subset=["w2"], keep="last", inplace=True) + scores.drop_duplicates(subset=["w1"], keep="last", inplace=True) + scores.to_csv(outpath, index=False) + + print("________\nDone.\n\n\n\n") + if len(fails) > 0: + print("Could not find any matches for the following items:") + for fail in fails: + print("\t%s" % fail) + wait = input("Press RETURN to continue.") or "" + else: + print("_________\nFail: could not find anything.\n\n\n") + try: + #os.remove("_temp.csv") + pass + except: + print("Couldn't remove the temp file.") + + saveSettings([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + + sys.exit(0) diff --git a/asign_search.py b/assign_search_lemma.py similarity index 67% rename from asign_search.py rename to assign_search_lemma.py index 75f99a2..4aafbc8 100644 --- a/asign_search.py +++ b/assign_search_lemma.py @@ -19,13 +19,15 @@ import gc gc.enable() +file_folder = "E:/stimuli_coca_exp19/scores/" + def getSettings(): try: num = int(input("How many bigrams do you want to find? ")) except: num = 100 - + try: cuts = int(input("\nHow many bins should the scores be divided into? ")) except: @@ -33,7 +35,7 @@ def getSettings(): try: disbalance_penalty = float(input("\nHow large a penalty should be given to imbalance (1-5)? ")) - except: + except: disbalance_penalty = 2 try: @@ -41,13 +43,13 @@ def getSettings(): except: max_time = 5 print("\nDo you want to filter the bigrams by their POS-tags?\nWrite the RegEx to match (the tags are lower-cased): E.g. j.* to extract adjectives\nLeave empty to ignore") - pos_1 = input("\nWord 1: ") or "" - pos_2 = input("\nWord 2: ") or "" - + pos_1 = input("\nWord 1: ") or ".*" + pos_2 = input("\nWord 2: ") or ".*" + print("\nDo you want to filter the bigrams by the string?\nWrite the RegEx to match: E.g. [a-z].* to extract lower-case only\nLeave empty to ignore") - w_1 = input("\nWord 1: ") or "" - w_2 = input("\nWord 2: ") or "" - + w_1 = input("\nWord 1: ") or ".*" + w_2 = input("\nWord 2: ") or ".*" + try: min_uni_freq = int(input("\nWhat should be the lowest unigram frequency included? ")) except: @@ -59,39 +61,59 @@ def getSettings(): try: max_bg_freq = int(input("\nWhat should be the highest bigram frequency included? ")) except: - max_bg_freq = 0 + max_bg_freq = 100000000 - optimizer = "" + optimizer = "" while optimizer not in ["quick", "pruning", "q", "p"]: - optimizer = input("Which optimizer should be used? [(Q)uick/(P)runing] ") or "quick" - + optimizer = input("\nWhich optimizer should be used? [(Q)uick/(P)runing] ") or "quick" + if optimizer.startswith("q"): optimizer = "quick" else: optimizer = "pruning" - + try: seed = int(input("\nWhat should be the seed for random sampling? Leave empty if unsure, this will get you reproducible results. ")) except: seed = 1991 - + if optimizer == "quick": try: beam_width = int(input("\nHow many samples should be used in the sampling procedure? ")) except: - beam_width = 0 - + beam_width = 1 + else: beam_width = 1000 - + return([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) -def getPos(bigram): - bigram = word_tokenize(bigram) - bigram = pos_tag(bigram) - # bigram = "_".join([x[1][0] for x in bigram]) - return(bigram) - +def saveSettings(settings): + + num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = settings + print("Do you want to save your settings? [y/N]") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + settings = "" + settings += "num = %i\n" % num + settings += "cuts = %i\n" % cuts + settings += "disbalance_penalty = %i\n" % disbalance_penalty + settings += 'w_1 ="' + w_1 + '"\n' + settings += 'w_2 ="' + w_2 + '"\n' + settings += 'pos_1 ="' + pos_1 + '"\n' + settings += 'pos_2 ="' + pos_2 + '"\n' + settings += "max_time = %i\n" % max_time + settings += "min_bg_freq = %i\n" % min_bg_freq + settings += "max_bg_freq = %i\n" % max_bg_freq + settings += "min_uni_freq = %i\n" % min_uni_freq + settings += 'optimizer = "' + optimizer + '"\n' + + with open("search_settings.py", "w+") as f: + f.write(settings) + def getBins(it2score, cuts=20, type="dict", method = "exact", percent=95): """Extract the range of scores. Take dict (or dict of dicts if specified), return lower/upper bound. Returns numpy array of edges.""" if type == "dict": @@ -101,92 +123,82 @@ def getBins(it2score, cuts=20, type="dict", method = "exact", percent=95): print("Conversion not possible. Wrong type selected?") elif type == "dictOfDicts": scores = [it2score[x][y] for x in it2score for y in it2score[x]] # Get from dictionary to list - + else: print("Dicts nested more than once not implemented") raise ValueError() - + scores = array(scores) - + print(" Cropping extreme %i percent of data" % (100-percent)) lower, upper = percentile(scores, [(100-percent)/2, ((100-percent)/2)+percent]) scores = scores[(scores > lower) & (scores < upper)] - + if method == "log": print(" Defining log-transform-based bin boundaries") # bins = histogram_bin_edges(scores, cuts) - + translate = (0-amin(scores)) + 1 #print(amin(scores)) #print(amax(scores)) scores = log(scores + translate) - counts, bins = histogram(scores, cuts-2) + counts, bins = histogram(scores, min(cuts-2,1)) # print(counts/sum(counts)) bins = exp(bins) - translate scores = exp(scores) - translate - + else: print(" Defining bin boundaries") # bins = histogram_bin_edges(scores, cuts) - + #translate = (0-amin(scores)) + 1 #print(amin(scores)) #print(amax(scores)) #scores = log(scores + translate) - counts, bins = histogram(scores, cuts-2) + counts, bins = histogram(scores, min(cuts-2,1)) # print(counts/sum(counts)) #bins = exp(bins) - translate #scores = exp(scores) - translate - #plt.hist(scores, bins) - #plt.show() - + #plt.hist(scores, bins) + #plt.show() + # x = input("Press RETURN to continue") #print(bins) return(bins) - -def getDisbalance(new, old, penalty=3): - - old = deepcopy(old) - for item in new: - old[arange(old.shape[0]), item] += 1 - m = sum(old[0,:])/old.shape[1] - old = old - m - old == old**penalty - return(nsum(old)) def getCorPerformance(scores, cutoff = 0.7): scores = pd.DataFrame(scores) cors = abs(scores.corr(method="spearman")) - + highs = sum(nsum(cors > cutoff)) tiebreaker = nanmedian(cors[cors > cutoff]) - + return([highs, tiebreaker]) - -def getGain(new, old, penalty=3): +def getDisbalance(new, old, penalty=3, mod=1): old = deepcopy(old) for item in new: - old[arange(old.shape[0]), item] -= 1 + old[arange(old.shape[0]), item] += 1*mod m = sum(old[0,:])/old.shape[1] old = old - m old == old**penalty return(nsum(old)) - -class Scorer(object): + +class ScorerLemma(object): """Loads all the score lists, can then be used to asign the scores on a per-item base with the score method. It is memory-heavy, but could be included in functions which allow interactive collocation input.""" - + ##Required input format should be added to docstring.. + def __init__(self): """Load the required score files. If they are not present in the folder, throw an exception.""" self.binning = "exact" self.optimizer = "pruning" - self.beam_width = 1000 - import os - files = ["wfreqs.json", "bigrams.json","fwd.json","bckw.json","llscore.json", "dicescore.json","tscore.json","zscore.json", "delta_p21.json", "delta_p12.json", "miscore.json","mi3score.json","gscore.json"] - files = ["scores2/"+x for x in files] + self.beam_width = 5 + import os + files = ["wfreqs_lemma.json", "bigrams_lemma.json","fwd_lemma.json","bckw_lemma.json","llscore_lemma.json", "dicescore_lemma.json", "moddicescore_lemma.json","tscore_lemma.json","zscore_lemma.json", "delta_p21_lemma.json", "delta_p12_lemma.json", "miscore_lemma.json","mi3score_lemma.json","gscore_lemma.json"] + files = [file_folder +x for x in files] try: filecheck = [os.path.isfile(f) for f in files] if all(filecheck) != True: @@ -198,63 +210,67 @@ def __init__(self): if filecheck[i]==False: print(files[i]) print("Exiting") - sys.exit(1) - + sys.exit(1) + try: - print("\nLoading the saved scores") + print("\nLoading the saved LEMMA scores") print(" unigram frequency") - with open("scores2/wfreqs.json", "r") as i: - self.wfreq = json.loads(i.read()) - + with open(file_folder + "wfreqs_lemma.json", "r") as i: + self.wfreq = json.loads(i.read()) + print(" bigram frequency") - with open("scores2/bigrams.json", "r") as i: + with open(file_folder + "bigrams_lemma.json", "r") as i: self.bg_frq = json.loads(i.read()) - - print(" TP-D") - with open("scores2/fwd.json", "r") as i: - self.tp_d = json.loads(i.read()) - - print(" TP-B") - with open("scores2/bckw.json", "r") as i: - self.tp_b = json.loads(i.read()) - - print(" Log likelihood") - with open("scores2/llscore.json", "r") as i: - self.log_lklhd = json.loads(i.read()) - - print(" Modified dice") - with open("scores2/dicescore.json", "r") as i: - self.dice = json.loads(i.read()) - - print(" t-score") - with open("scores2/tscore.json", "r") as i: - self.t_score = json.loads(i.read()) - - print(" z-score") - with open("scores2/zscore.json", "r") as i: - self.z_score = json.loads(i.read()) - - print(" delta_p-12") - with open("scores2/delta_p12.json", "r") as i: - self.delta_p12 = json.loads(i.read()) - - print(" delta_p-21") - with open("scores2/delta_p21.json", "r") as i: - self.delta_p21 = json.loads(i.read()) - - print(" MI-score") - with open("scores2/miscore.json", "r") as i: - self.mi_score = json.loads(i.read()) - - print(" MI3-score") - with open("scores2/mi3score.json", "r") as i: - self.mi3_score = json.loads(i.read()) - - print(" G-score") - with open("scores2/gscore.json", "r") as i: - self.g_score = json.loads(i.read()) + + print(" TP-D") + with open(file_folder + "fwd_lemma.json", "r") as i: + self.tp_d = json.loads(i.read()) + + print(" TP-B") + with open(file_folder + "bckw_lemma.json", "r") as i: + self.tp_b = json.loads(i.read()) + + print(" Log likelihood") + with open(file_folder + "llscore_lemma.json", "r") as i: + self.log_lklhd = json.loads(i.read()) + + # print(" Dice") + # with open(file_folder + "dicescore_lemma.json", "r") as i: + # self.dice = json.loads(i.read()) + # + # print(" Modified dice") + # with open(file_folder + "moddicescore_lemma.json", "r") as i: + # self.moddice = json.loads(i.read()) + + print(" t-score") + with open(file_folder + "tscore_lemma.json", "r") as i: + self.t_score = json.loads(i.read()) + + # print(" z-score") + # with open(file_folder + "zscore_lemma.json", "r") as i: + # self.z_score = json.loads(i.read()) + # + # print(" delta_p-12") + # with open(file_folder + "delta_p12_lemma.json", "r") as i: + # self.delta_p12 = json.loads(i.read()) + # + # print(" delta_p-21") + # with open(file_folder + "delta_p21_lemma.json", "r") as i: + # self.delta_p21 = json.loads(i.read()) + + print(" MI-score") + with open(file_folder + "miscore_lemma.json", "r") as i: + self.mi_score = json.loads(i.read()) + + print(" MI3-score") + with open(file_folder + "mi3score_lemma.json", "r") as i: + self.mi3_score = json.loads(i.read()) + + # print(" G-score") + # with open(file_folder + "gscore_lemma.json", "r") as i: + # self.g_score = json.loads(i.read()) print("_________________________________") - + except: print("Not all score files could be loaded. Check that they are in the same folder as this script.") print("Exiting") @@ -263,10 +279,11 @@ def __init__(self): def score(self, items): """Score all the bigrams at once. Input is a list of lists with the sublist format [item1, item2]. If a bigram is not in the score list, return NA.""" - + items_out = [] - + for bigram in items: + #print("test " +str(bigram)) w1, w2 = bigram bigram = " ".join(bigram) @@ -274,192 +291,204 @@ def score(self, items): try: w1_frq = self.wfreq[w1] except: - w1_frq = "NA" - + w1_frq = "NA" + try: w2_frq = self.wfreq[w2] except: - w2_frq = "NA" - + w2_frq = "NA" + try: bg_frq = self.bg_frq[bigram] except: - bg_frq = "NA" + bg_frq = "NA" try: tp_d = self.tp_d[w1][w2] except: tp_d = "NA" - + try: tp_b = self.tp_b[w2][w1] except: tp_b = "NA" - + try: log_lklhd = self.log_lklhd[bigram] except: - log_lklhd = "NA" - + log_lklhd = "NA" + try: dice = self.dice[bigram] except: - dice = "NA" - + dice = "NA" + + try: + moddice = self.moddice[bigram] + except: + moddice = "NA" + try: t_score = self.t_score[bigram] except: - t_score = "NA" - + t_score = "NA" + try: z_score = self.z_score[bigram] except: - z_score = "NA" - + z_score = "NA" + try: mi_score = self.mi_score[bigram] except: - mi_score = "NA" + mi_score = "NA" try: mi3_score = self.mi3_score[bigram] except: - mi3_score = "NA" - + mi3_score = "NA" + try: g_score = self.g_score[bigram] except: - g_score = "NA" + g_score = "NA" try: delta_p12 = self.delta_p12[bigram] except: - delta_p12 = "NA" - + delta_p12 = "NA" + try: delta_p21 = self.delta_p21[bigram] except: - delta_p21 = "NA" - - items_out.append([bigram, w1_frq, w2_frq, bg_frq, tp_b, tp_d, log_lklhd, dice, t_score, z_score, mi_score, mi3_score, g_score, delta_p12, delta_p21]) - + delta_p21 = "NA" + + items_out.append([bigram, w1_frq, w2_frq, bg_frq, tp_b, tp_d, log_lklhd, dice, moddice, t_score, z_score, mi_score, mi3_score, g_score, delta_p12, delta_p21]) + return(items_out) - - + + def score_one(self, bigram): """Score the individual bigrams. Input is in the format [item1, item2]. If a bigram is not in the score list, return NA.""" - + w1, w2 = bigram bigram = " ".join(bigram) try: w1_frq = self.wfreq[w1] except: - w1_frq = "NA" - + w1_frq = "NA" + try: w2_frq = self.wfreq[w2] except: - w2_frq = "NA" + w2_frq = "NA" try: bg_frq = self.bg_frq[bigram] except: - bg_frq = "NA" + bg_frq = "NA" try: tp_d = self.tp_d[w1][w2] except: tp_d = "NA" - + try: tp_b = self.tp_b[w2][w1] except: tp_b = "NA" - + try: log_lklhd = self.log_lklhd[bigram] except: - log_lklhd = "NA" - + log_lklhd = "NA" + try: dice = self.dice[bigram] except: - dice = "NA" - + dice = "NA" + + try: + moddice = self.moddice[bigram] + except: + moddice = "NA" + try: t_score = self.t_score[bigram] except: - t_score = "NA" - + t_score = "NA" + try: z_score = self.z_score[bigram] except: - z_score = "NA" - + z_score = "NA" + try: mi_score = self.mi_score[bigram] except: - mi_score = "NA" + mi_score = "NA" try: mi3_score = self.mi3_score[bigram] except: - mi3_score = "NA" - + mi3_score = "NA" + try: g_score = self.g_score[bigram] except: - g_score = "NA" + g_score = "NA" try: delta_p12 = self.delta_p12[bigram] except: - delta_p12 = "NA" - + delta_p12 = "NA" + try: delta_p21 = self.delta_p21[bigram] except: - delta_p21 = "NA" - - return([bigram, w1_frq, w2_frq, bg_frq, tp_b, tp_d, log_lklhd, dice, t_score, z_score, mi_score, mi3_score, g_score, delta_p12, delta_p21]) - + delta_p21 = "NA" + + return([bigram, w1_frq, w2_frq, bg_frq, tp_b, tp_d, log_lklhd, dice, moddice, t_score, z_score, mi_score, mi3_score, g_score, delta_p12, delta_p21]) + def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=20, words = ["", ""], pos=["",""], max_bg_freq=100000000, min_bg_freq=0, min_uni_freq=5, percent=95): + num = max(num,1) """Get num random bigrams, spread approximately evenly accross the ranges of the scores.""" print("\nFinishing initialization") ### FILTERING BY POS - + self.bgs_all = [x.split() for x in tqdm.tqdm(self.bg_frq) if (" ".join(x.split()) in self.bg_frq)] - + if (min_bg_freq > 0) or (max_bg_freq < 100000000): - print(" Selecting bigrams by bigram frequency") - self.bgs = [[x,y] for x,y in tqdm.tqdm(self.bgs_all) if (self.bg_frq[" ".join([x,y])] > min_bg_freq) and (self.bg_frq[" ".join([x,y])] < max_bg_freq)] + print(" Selecting bigrams by bigram frequency") + self.bgs = [[x,y] for x,y in tqdm.tqdm(self.bgs_all) if (self.bg_frq[" ".join([x,y])] > min_bg_freq) and (self.bg_frq[" ".join([x,y])] < max_bg_freq)] else: self.bgs = self.bgs_all - + if min_uni_freq > 1: - print(" Selecting bigrams by unigram frequency") - self.bgs = [[x,y] for x,y in tqdm.tqdm(self.bgs) if (self.wfreq[x]>=min_uni_freq and self.wfreq[y]>=min_uni_freq)] + print(" Selecting bigrams by unigram frequency") + self.bgs = [[x,y] for x,y in tqdm.tqdm(self.bgs) if (self.wfreq[x]>=min_uni_freq and self.wfreq[y]>=min_uni_freq)] else: self.bgs = self.bgs - - if pos[0] != "" or pos[1] != "": + + if pos[0] != "" or pos[1] != "": pos_1, pos_2 = pos pos_1 = compile(pos_1) pos_2 = compile(pos_2) print(" Selecting bigrams by POS") self.bgs = [[x,y] for x,y in tqdm.tqdm(self.bgs) if (match(pos_1, x.split("_")[1])!=None and match(pos_2,y.split("_")[1])!=None)] - if words[0] != "" or words[1] != "": + if words[0] != "" or words[1] != "": w_1, w_2 = words w_1 = compile(w_1) w_2 = compile(w_2) print(" Selecting bigrams by words 1 & 2") self.bgs = [[x,y] for x,y in tqdm.tqdm(self.bgs) if (match(w_1, x.split("_")[0])!=None and match(w_2,y.split("_")[0])!=None)] - - + + print("_"*10+"\nRelevant bigrams: %i" % len(self.bgs)) + # print(" Cleaning unfitting bigrams") # print(" Initializing cleaner") @@ -486,101 +515,116 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 # del self.delta_p21[c] # except: # pass - + # del wrong_keys - + + if (len(self.bgs) <= num) and len(self.bgs) > 0: + print("Random sampling failed: There are fewer relevant cases than the sample size.") + scores = [self.score(x) for x in self.bgs] # remove bigram string to allow numpy operation + scores = [array([x[1:] for x in y]) for y in scores] + results = [[sub("_[^ ]+",""," ".join(self.bgs[x]))]+list(scores[x,:]) for x in range(min(len(self.bgs), num))] + + return(results) + + elif len(self.bgs) == 0: + raise ValueError("No case matches the criteria.") + print("\nDefining data distributions") - + print(" unigram frequency") self.wfreq_bins = getBins(self.wfreq, cuts=cuts, method = self.binning, percent=percent) - + print(" bigram frequency") self.bg_frq_bins = getBins(self.bg_frq, cuts=cuts, method = self.binning, percent=percent) # Get from dictionary to list - - + + if self.optimizer == "pruning": - print(" TP-D") - + print(" TP-D") + self.tp_d_bins = getBins(self.tp_d, cuts=cuts, type="dictOfDicts", percent=percent) # Get from dictionary to list - print(" TP-B") + print(" TP-B") self.tp_b_bins = getBins(self.tp_b, cuts=cuts, type="dictOfDicts", percent=percent) # Get from dictionary to list - print(" Log likelihood") + print(" Log likelihood") self.log_lklhd_bins = getBins(self.log_lklhd, cuts=cuts, percent=percent) # Get from dictionary to list - print(" Modified dice") + print(" Dice") self.dice_bins = getBins(self.dice, cuts=cuts, percent=percent) # Get from dictionary to list - print(" t-score") + print(" Modified dice") + self.moddice_bins = getBins(self.moddice, cuts=cuts, percent=percent) + + print(" t-score") self.t_score_bins = getBins(self.t_score, cuts=cuts, percent=percent) # Get from dictionary to list - print(" z-score") + print(" z-score") self.z_score_bins = getBins(self.z_score, cuts=cuts, percent=percent) # Get from dictionary to list - - print(" MI-score") + + print(" MI-score") self.mi_score_bins = getBins(self.mi_score, cuts=cuts, percent=percent) # Get from dictionary to list - - print(" MI3-score") + + print(" MI3-score") self.mi3_score_bins = getBins(self.mi3_score, cuts=cuts, percent=percent) # Get from dictionary to list - print(" G-score") + print(" G-score") self.g_score_bins = getBins(self.g_score, cuts=cuts, percent=percent) # Get from dictionary to list - - print(" Delta_p-12") + + print(" Delta_p-12") self.delta_p12_bins = getBins(self.delta_p12, cuts=cuts, percent=percent) # Get from dictionary to list - print(" Delta_p-21") - self.delta_p21_bins = getBins(self.delta_p21, cuts=cuts, percent=percent) # Get from dictionary to list - - print("_________________________________") - + print(" Delta_p-21") + self.delta_p21_bins = getBins(self.delta_p21, cuts=cuts, percent=percent) # Get from dictionary to list + + print("_________________________________") + buf = 10 # The bufferring coefficient (how many extra elements should be collected) - self.dist = zeros([14, cuts]) # Array to save the distributions: columns=bins, rows=scores - self.results = zeros([num*buf, 14]) # Array to save the results + self.dist = zeros([15, cuts]) # Array to save the distributions: columns=bins, rows=scores + self.results = zeros([num*buf, 15]) # Array to save the results self.items = [] self.populated = 0 self.disbalance_penalty = floor(num*disbalance_penalty/100) # self.indexes = arange(length(self.bg_pos)) - - random.seed(seed) + + random.seed(seed) nrandom.seed(seed) - + print("\nStarting item selection") start = time.time() - lasttime = time.time() - + lasttime = time.time() + max_time = max_time*60 pbar = [tqdm.tqdm(total = num), tqdm.tqdm(total=num*(buf-1))] pbar[0].set_description("Stimuli collected") pbar[1].set_description("Additional buffer") reached = False - + while (self.populated < num*buf) and (time.time() - start) < max_time: # Get a random item; check which bins would it increase for which score, if this disturbs balance, drop otherwise insert at the bottom if self.populated >= num and reached == False: reached = True - + samples = [nrandom.choice(range(len(self.bgs)),10, replace=False) for x in range(1000)] samples = [[self.bgs[y] for y in x] for x in samples] scores = [self.score(x) for x in samples] # remove bigram string to allow numpy operation scores = [array([x[1:] for x in y]) for y in scores] - + binned = [array([digitize(x[:,0],self.bg_frq_bins), digitize(x[:,1],self.wfreq_bins), digitize(x[:,2],self.wfreq_bins), - digitize(x[:,3],self.tp_b_bins), + digitize(x[:,3],self.tp_b_bins), digitize(x[:,4],self.tp_d_bins), digitize(x[:,5],self.log_lklhd_bins), digitize(x[:,6],self.dice_bins), - digitize(x[:,7],self.t_score_bins), - digitize(x[:,8],self.z_score_bins), - digitize(x[:,9],self.mi_score_bins), - digitize(x[:,10],self.mi3_score_bins), - digitize(x[:,11],self.g_score_bins), - digitize(x[:,12],self.delta_p12_bins), - digitize(x[:,13],self.delta_p21_bins)]) for x in scores] - + digitize(x[:,7],self.moddice_bins), + digitize(x[:,8],self.t_score_bins), + digitize(x[:,9],self.z_score_bins), + digitize(x[:,10],self.mi_score_bins), + digitize(x[:,11],self.mi3_score_bins), + digitize(x[:,12],self.g_score_bins), + digitize(x[:,13],self.delta_p12_bins), + digitize(x[:,14],self.delta_p21_bins)]) for x in scores] + # print(binned[0]) binned = [transpose(x) for x in binned] # binned = [] @@ -589,34 +633,35 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 self.results[self.populated:self.populated+10,:] = scores[best] for item in binned[best]: - self.dist[arange(14), item] += 1 + self.dist[arange(15), item] += 1 self.items += samples[best] self.populated += 10 if reached: pbar[1].update(10) else: pbar[0].update(10) - + pbar[0].close() pbar[1].close() - + if self.populated >= num and (time.time() - start) < max_time: print("\nPruning") - + binned = array([digitize(self.results[:,0],self.bg_frq_bins), digitize(self.results[:,1],self.wfreq_bins), digitize(self.results[:,2],self.wfreq_bins), - digitize(self.results[:,3],self.tp_b_bins), + digitize(self.results[:,3],self.tp_b_bins), digitize(self.results[:,4],self.tp_d_bins), digitize(self.results[:,5],self.log_lklhd_bins), digitize(self.results[:,6],self.dice_bins), - digitize(self.results[:,7],self.t_score_bins), - digitize(self.results[:,8],self.z_score_bins), - digitize(self.results[:,9],self.mi_score_bins), - digitize(self.results[:,10],self.mi3_score_bins), - digitize(self.results[:,11],self.g_score_bins), - digitize(self.results[:,12],self.delta_p12_bins), - digitize(self.results[:,13],self.delta_p21_bins)]) + digitize(self.results[:,7],self.moddice_bins), + digitize(self.results[:,8],self.t_score_bins), + digitize(self.results[:,9],self.z_score_bins), + digitize(self.results[:,10],self.mi_score_bins), + digitize(self.results[:,11],self.mi3_score_bins), + digitize(self.results[:,12],self.g_score_bins), + digitize(self.results[:,13],self.delta_p12_bins), + digitize(self.results[:,14],self.delta_p21_bins)]) binned = transpose(binned) print("Removing duplicates") @@ -630,66 +675,66 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 else: firsts.update(w1) seconds.update(w2) - + dels.reverse() for d_index in tqdm.tqdm(dels): del self.items[d_index] binned = delete(binned, dels,0) self.results = delete(self.results, dels, 0) self.populated -= len(dels) - + pbar = tqdm.tqdm(total=self.populated - num) while self.populated > num and (time.time() - start) < max_time: # If there is time left, prune the most problematic items away, one by one rands = [random.randint(0,binned.shape[0]-1) for x in range(1000)] - performance = [getGain(binned[i,:], self.dist, self.disbalance_penalty) for i in rands] + performance = [getDisbalance(binned[i,:], self.dist, self.disbalance_penalty, mod=-1) for i in rands] best = rands[argmin(performance)] - self.dist[arange(14), binned[best,:]] -= 1 + self.dist[arange(15), binned[best,:]] -= 1 binned = delete(binned, best, 0) del self.items[best] self.results = delete(self.results, best, 0) self.populated -= 1 pbar.update(1) - + print("\nSuccess! All %i items were found." % num) - - + + else: print("\nTimeout limit exceeded. Returning %i items" % self.populated) - results = self.results[0:min(self.populated, num)] # Crop which we don't have + results = self.results[0:min(self.populated, num)] # Crop which we don't have #plt.imshow(self.dist, cmap="hot", interpolation="bilinear") #plt.suptitle("Distribution accross scores") #plt.xlabel("Score bin") #plt.ylabel("Score") #plt.show() - - results = [[sub("_[^ ]+",""," ".join(self.items[x]))]+list(self.results[x, 0:14]) for x in range(min(self.populated, num))] - + + results = [[sub("_[^ ]+",""," ".join(self.items[x]))]+list(self.results[x, 0:15]) for x in range(min(self.populated, num))] + return(results) if self.optimizer == "quick": - + beam = self.beam_width # The beam width to keep best samples - self.dist = zeros([14, cuts]) # Array to save the distributions: columns=bins, rows=scores - self.results = zeros([num, 14]) # Array to save the results + self.dist = zeros([15, cuts]) # Array to save the distributions: columns=bins, rows=scores + self.results = zeros([num, 15]) # Array to save the results self.items = [] - - random.seed(seed) + + random.seed(seed) nrandom.seed(seed) - + print("\nStarting item selection") start = time.time() - + max_time = max_time*60 - - best = 14**2 + + best = 15**2 tiebreak = 1 - best_sample = [] - + best_sample = [] + pbar = tqdm.tqdm(total = beam) - pbar.set_description("Samples tried") + pbar.set_description("Samples tried") for sample in range(beam): sample = list(nrandom.choice([x for x in range(len(self.bgs))], min(floor(num*1.1), len(self.bgs)), replace=False)) sample = [self.bgs[x] for x in sample] @@ -697,7 +742,7 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 scores = self.score(sample) # remove bigram string to allow numpy operation scores = array([x[1:] for x in scores]) performance, tiebreaker = getCorPerformance(scores) - + if performance < best: best = performance best_sample = sample @@ -709,10 +754,10 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 pbar.update(1) if (time.time() - start) >= max_time: - break - + break + pbar.close() - + # print(best_sample) if (time.time() - start) < max_time: print("Removing duplicates") @@ -726,13 +771,13 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 else: firsts.update(w1) seconds.update(w2) - + dels.reverse() for d_index in tqdm.tqdm(dels): del best_sample[d_index] results = best_sample[0:num] - + else: print("\nTimeout limit exceeded, returning best sample at this moment.") results = best_sample[0:min(num, len(best_sample))] # Crop which we don't have @@ -742,26 +787,26 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 #plt.xlabel("Score bin") #plt.ylabel("Score") #plt.show() - + results = self.score(best_sample) - results = pd.DataFrame(results, columns = ["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "dice", "t_score", "z_score", "mi_score", "mi3_score", "g_score", "delta_p12", "delta_p21"]) - results["bigram"] = [sub("_[^ ]+","",x) for x in results["bigram"]] + results = pd.DataFrame(results, columns = ["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + results["bigram_lemma"] = [sub("_[^ ]+","",x) for x in results["bigram_lemma"]] results = results.values # print(results) return(results) - + if __name__ == "__main__": mode = None while mode not in ["score", "search", "strat_search", "match"]: - mode = input("Which mode should this program run in?\n Score/search: ") - + mode = input("Which mode should this program run in?\n Score/search/strat_search/match: ") + if mode.lower() == "score": - + if len(sys.argv) > 1: inpath = sys.argv[1] else: - inpath = raw_input("Where is the file to load?\n ") - + inpath = input("Where is the file to load?\n ") + try: with open(inpath, "r") as infile: items = infile.readlines() @@ -778,107 +823,85 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version if ram_present > 7 and ram_available > 5: - scorer = Scorer() + scorer = ScorerLemma() else: print("This is a RAM-intensive operation. You need at least 6 GB of free RAM.") print("Exiting...") sys.exit(0) - - items = scorer.score(items) - + + items = scorer.score(items) + if len(sys.argv) > 2: outpath = sys.argv[2] else: - outpath = raw_input("Where should the results be saved?\n ") - + outpath = input("Where should the results be saved?\n ") + print("Saving") with open(outpath, "w+") as outfile: out_csv = csv.writer(outfile) - out_csv.writerow(["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "dice", "t_score", "z_score", "mi_score", "mi3_score", "g_score", "delta_p12", "delta_p21"]) + out_csv.writerow(["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) for i in tqdm.tqdm(items): out_csv.writerow(i) print("Done. Press RETURN to exit") - wait = raw_input() + wait = input() sys.exit(0) elif mode.lower() == "search": - + ram_present = psutil.virtual_memory()[0] >> 30 ram_available = psutil.virtual_memory()[1] >> 30 # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version if ram_present > 7: # if ram_present > 7 and ram_available > 5: - pass + pass else: print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") sys.exit(0) - + max_time = 0 disbalance_penalty = 0 cuts = 0 num = 0 pos = "" - + saved = False if os.path.isfile("search_settings.py"): print("Saved settings found. Do you want to use them? [Y/n]") dec = "" while dec.lower() not in set(["y", "n"]): dec = input("[Y/n]") or "Y" - - if dec.lower() == "y": + + if dec.lower() == "y": saved = True - - if saved == True: + + if saved == True: from search_settings import * print("Using saved settings") - + else: - num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer = getSettings() - - scorer = Scorer() + num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = getSettings() + + scorer = ScorerLemma() scorer.optimizer = optimizer items = scorer.get_random(num, cuts=cuts, seed=seed, disbalance_penalty = disbalance_penalty, words=[w_1, w_2], pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=min_bg_freq, max_bg_freq=max_bg_freq, min_uni_freq=min_uni_freq) - + print("Saving") - + if len(sys.argv) > 2: outpath = sys.argv[2] else: outpath = input("Where should the results be saved?\n ") - + print("Saving") with open(outpath, "w+") as outfile: out_csv = csv.writer(outfile) - out_csv.writerow(["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "dice", "t_score", "z_score", "mi_score", "mi3_score", "g_score", "delta_p12", "delta_p21"]) + out_csv.writerow(["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) for i in tqdm.tqdm(items): out_csv.writerow(i) - - print("Do you want to save your settings? [y/N]") - dec = "" - while dec not in set(["y", "n"]): - dec = input("[y/N]").lower() - if dec == "": - dec = "n" - - if dec == "y": - settings = "" - settings += "num = %i\n" % num - settings += "cuts = %i\n" % cuts - settings += "disbalance_penalty = %i\n" % disbalance_penalty - settings += 'w_1 ="' + w_1 + '"\n' - settings += 'w_2 ="' + w_2 + '"\n' - settings += 'pos_1 ="' + pos_1 + '"\n' - settings += 'pos_2 ="' + pos_2 + '"\n' - settings += "max_time = %i\n" % max_time - settings += "min_bg_freq = %i\n" % min_bg_freq - settings += "max_bg_freq = %i\n" % max_bg_freq - settings += "min_uni_freq = %i\n" % min_uni_freq - settings += "optimizer = " + optimizer +"\n" - - with open("search_settings.py", "w+") as f: - f.write(settings) + + saveSettings([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + elif mode.lower() == "strat_search": @@ -888,64 +911,64 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 outpath = input("Where should the results be saved?\n ") with open("_temp.csv", "w+") as outfile: - out_csv = csv.writer(outfile) - out_csv.writerow(["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "dice", "t_score", "z_score", "mi_score", "mi3_score", "g_score", "delta_p12", "delta_p21"]) - + out_csv = csv.writer(outfile) + out_csv.writerow(["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + ram_present = psutil.virtual_memory()[0] >> 30 ram_available = psutil.virtual_memory()[1] >> 30 # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version if ram_present > 7: # if ram_present > 7 and ram_available > 5: - pass + pass else: print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") sys.exit(0) - + max_time = 0 disbalance_penalty = 0 cuts = 0 num = 0 pos = "" - + saved = False if os.path.isfile("search_settings.py"): print("Saved settings found. Do you want to use them? [Y/n]") dec = "" while dec.lower() not in set(["y", "n"]): dec = input("[Y/n]") or "Y" - - if dec.lower() == "y": + + if dec.lower() == "y": saved = True - - if saved == True: + + if saved == True: from search_settings import * print("Using saved settings") - + else: num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = getSettings() - - scorer = Scorer() - + + scorer = ScorerLemma() + scorer.beam_width = beam_width scorer.binning = "exact" items = scorer.get_random(1, cuts=10, seed=seed, disbalance_penalty = disbalance_penalty, words=[w_1, w_2], pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=min_bg_freq, max_bg_freq=max_bg_freq, min_uni_freq=min_uni_freq, percent=95) - + bins = list(scorer.bg_frq_bins) + [max_bg_freq] bins[0] = [min_bg_freq] - + scorer.binning = "exact" scorer.optimizer = optimizer - print(bins) + print(bins) iter = 0 for strat in tqdm.tqdm(range(len(bins)-1)): items = scorer.get_random(floor((num*1.5)/(len(bins)-1)), cuts=cuts, seed=seed+iter, disbalance_penalty = disbalance_penalty, words=[w_1, w_2], - pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=floor(bins[strat]), max_bg_freq=ceil(bins[strat+1])+1, min_uni_freq=min_uni_freq, percent=100) + pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=floor(bins[strat]), max_bg_freq=ceil(bins[strat+1])+1, min_uni_freq=min_uni_freq, percent=100) iter +=1 - + with open("_temp.csv", "a") as outfile: - out_csv = csv.writer(outfile) + out_csv = csv.writer(outfile) for i in tqdm.tqdm(items): out_csv.writerow(i) @@ -957,42 +980,19 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 scores = scores.drop_duplicates(subset=["w1"], keep="last") scores = scores.drop_duplicates(subset=["w2"], keep="last") scores.drop(["w1", "w2"], 1, inplace=True) - + if scores.shape[0] > num: - ints = sorted(nrandom.choice(range(scores.shape[0]), num, replace=False)) + ints = sorted(nrandom.choice(range(scores.shape[0]), num, replace=False)) scores = scores.iloc[ints,:] - + scores.to_csv(outpath, index=False) try: os.remove("_temp.csv") except: print("Couldn't remove the temp file.") - - print("Do you want to save your settings? [y/N]") - dec = "" - while dec.lower() not in set(["y", "n"]): - dec = input("[y/N]") or "N" - - if dec.lower() == "y": - settings = "" - settings += "num = %i\n" % num - settings += "cuts = %i\n" % cuts - settings += "disbalance_penalty = %i\n" % disbalance_penalty - settings += 'w_1 ="' + w_1 + '"\n' - settings += 'w_2 ="' + w_2 + '"\n' - settings += 'pos_1 ="' + pos_1 + '"\n' - settings += 'pos_2 ="' + pos_2 + '"\n' - settings += "max_time = %i\n" % max_time - settings += "min_bg_freq = %i\n" % min_bg_freq - settings += "max_bg_freq = %i\n" % max_bg_freq - settings += "min_uni_freq = %i\n" % min_uni_freq - settings += "optimizer = " + optimizer +"\n" - - with open("search_settings.py", "w+") as f: - f.write(settings) - - print("Done. Press RETURN to exit") - wait = input() + + saveSettings([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + sys.exit(0) elif mode.lower() == "match": @@ -1002,140 +1002,139 @@ def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=2 stimuli = pd.read_csv(infile) stimuli.dropna(0, how="all", subset=["bigram"], inplace=True) - + try: matchWord = int(input("Which word should be matched? [1-%i]" % len(stimuli["bigram"][0].split()))) - 1 - except: + except: matchWord = 0 try: min_t_score = int(input("What is the lowest accepted t-score?")) - except: - min_t_score = -1000000 - + except: + min_t_score = -1000000 + try: max_t_score = int(input("What is the highest accepted t-score?")) - except: + except: max_t_score = +1000000 - + keys = set([x.split()[matchWord] for x in list(stimuli["bigram"].values)]) - + non_matched = 0 if matchWord==1 else 1 + forbidden = set([x.split()[non_matched] for x in list(stimuli["bigram"].values)]) + if len(sys.argv) > 2: outpath = sys.argv[2] else: outpath = input("Where should the results be saved?\n ") with open("_temp.csv", "w+") as outfile: - out_csv = csv.writer(outfile) - out_csv.writerow(["bigram", "w1_freq", "w2_freq", "bigram_freq", "tp_b", "tp_d", "log_lklhd", "dice", "t_score", "z_score", "mi_score", "mi3_score", "g_score", "delta_p12", "delta_p21"]) - + out_csv = csv.writer(outfile) + out_csv.writerow(["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + ram_present = psutil.virtual_memory()[0] >> 30 ram_available = psutil.virtual_memory()[1] >> 30 # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version if ram_present > 7: # if ram_present > 7 and ram_available > 5: - pass + pass else: print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") sys.exit(0) - + max_time = 0 disbalance_penalty = 0 cuts = 0 num = 0 pos = "" - + saved = False if os.path.isfile("search_settings.py"): print("Saved settings found. Do you want to use them? [Y/n]") dec = "" while dec.lower() not in set(["y", "n"]): dec = input("[Y/n]") or "Y" - - if dec.lower() == "y": + + if dec.lower() == "y": saved = True - - if saved == True: + + if saved == True: from search_settings import * print("Using saved settings") - + else: num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = getSettings() - + print("\n_____________________\nGrab a coffee...or ten") - - scorer = Scorer() + + scorer = ScorerLemma() ##### # Prefilter the data for only the adjectives? # Prefilter the data by t-score? ##### - + print("Pre-filtering the data for\n\t-matched words\n\t-fitting t-score") - scorer.bg_frq = {k:v for k,v in tqdm.tqdm(scorer.bg_frq.items()) if (k.split(" ")[matchWord].split("_")[0] in keys) and ((scorer.t_score[k] >= min_t_score) and (scorer.t_score[k] <= max_t_score))} - - scorer.beam_width = beam_width + scorer.bg_frq = {k:v for k,v in tqdm.tqdm(scorer.bg_frq.items()) if (k.split(" ")[matchWord].split("_")[0] in keys) and ((scorer.t_score[k] >= min_t_score) and (scorer.t_score[k] <= max_t_score))} + + scorer.beam_width = beam_width scorer.binning = "exact" scorer.optimizer = optimizer - + iter = 0 + fails = [] for key in tqdm.tqdm(keys): + print(key) try: matched_pattern = [w_1, w_2] matched_pattern[matchWord] = key items = scorer.get_random(num, cuts=cuts, seed=seed+iter, disbalance_penalty = disbalance_penalty, words=matched_pattern, - pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=min_bg_freq, max_bg_freq=max_bg_freq, min_uni_freq=min_uni_freq, percent=95) + pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=min_bg_freq, max_bg_freq=max_bg_freq, min_uni_freq=min_uni_freq, percent=95) iter +=1 - + with open("_temp.csv", "a") as outfile: - out_csv = csv.writer(outfile) + out_csv = csv.writer(outfile) for i in tqdm.tqdm(items): out_csv.writerow(i) - except: - wait = input("No matches for %s. Hit RETURN to continue" % key) + except: + fails.append(key) print("Saving") scores = pd.read_csv("_temp.csv") - scores["w1"], scores["w2"] = scores["bigram"].str.split(' ', 1).str - scores.index = range(scores.shape[0]) - scores = scores.drop_duplicates(subset=["bigram"], keep="last") - # scores = scores.drop_duplicates(subset=["w1"], keep="last") - # scores = scores.drop_duplicates(subset=["w2"], keep="last") - scores.drop(["w1", "w2"], 1, inplace=True) - - if scores.shape[0] > num: - ints = sorted(nrandom.choice(range(scores.shape[0]), num, replace=False)) - scores = scores.iloc[ints,:] - - scores.to_csv(outpath, index=False) + if scores.shape[0] > 0: + scores["w1"], scores["w2"] = scores["bigram"].str.split(' ', 1).str + scores.index = range(scores.shape[0]) + scores = scores.drop_duplicates(subset=["bigram"], keep="last") + + if non_matched == 0: + scores = scores[~scores["w1"].isin(forbidden)] + else: + scores = scores[~scores["w2"].isin(forbidden)] + + scores.sort_values(by=["t_score_lemma"], inplace=True, ascending=False) + scores = scores.drop_duplicates(subset=["w2"], keep="last") + scores = scores.drop_duplicates(subset=["w1"], keep="last") + + scores.drop(["w1", "w2"], 1, inplace=True) + + # if scores.shape[0] > num: + # ints = sorted(nrandom.choice(range(scores.shape[0]), num, replace=False)) + # scores = scores.iloc[ints,:] + + scores.to_csv(outpath, index=False) + + print("________\nDone.\n\n\n\n") + if len(fails) > 0: + print("Could not find any matches for the following items:") + for fail in fails: + print("\t%s" % fail) + wait = input("Press RETURN to continue.") or "" + else: + print("_________\nFail: could not find anything.\n\n\n") try: os.remove("_temp.csv") except: print("Couldn't remove the temp file.") - - print("Do you want to save your settings? [y/N]") - dec = "" - while dec.lower() not in set(["y", "n"]): - dec = input("[y/N]") or "N" - - if dec.lower() == "y": - settings = "" - settings += "num = %i\n" % num - settings += "cuts = %i\n" % cuts - settings += "disbalance_penalty = %i\n" % disbalance_penalty - settings += 'w_1 ="' + w_1 + '"\n' - settings += 'w_2 ="' + w_2 + '"\n' - settings += 'pos_1 ="' + pos_1 + '"\n' - settings += 'pos_2 ="' + pos_2 + '"\n' - settings += "max_time = %i\n" % max_time - settings += "min_bg_freq = %i\n" % min_bg_freq - settings += "max_bg_freq = %i\n" % max_bg_freq - settings += "min_uni_freq = %i\n" % min_uni_freq - settings += "optimizer = " + optimizer +"\n" - - with open("search_settings.py", "w+") as f: - f.write(settings) - - print("Done. Press RETURN to exit") - wait = input() - sys.exit(0) \ No newline at end of file + + saveSettings([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + + sys.exit(0) diff --git a/assign_search_lemma_simple.py b/assign_search_lemma_simple.py new file mode 100644 index 0000000..3fb9064 --- /dev/null +++ b/assign_search_lemma_simple.py @@ -0,0 +1,1141 @@ +import sys +import csv +import tqdm +import psutil +import random +import time +from numpy import array, digitize, zeros, amin, amax, arange, percentile, histogram, argmin, transpose, delete, log, exp, nanmedian +from numpy import random as nrandom +from numpy import sum as nsum +from math import floor, ceil +from nltk import word_tokenize, pos_tag +import matplotlib.pyplot as plt +from copy import deepcopy +from re import match, sub, compile +import json +import os +import pandas as pd + +import gc +gc.enable() + +#file_folder = "E:/Stimuli_zameji/" +file_folder = "/Users/kylamcconnell/Documents/GitHub/Stimuli/" + +def getSettings(): + + try: + num = int(input("How many bigrams do you want to find? ")) + except: + num = 100 + + try: + cuts = int(input("\nHow many bins should the scores be divided into? ")) + except: + cuts = 20 + + try: + disbalance_penalty = float(input("\nHow large a penalty should be given to imbalance (1-5)? ")) + except: + disbalance_penalty = 2 + + try: + max_time = int(input("\nWhat is the time limit for the search (in minutes)?")) + except: + max_time = 5 + print("\nDo you want to filter the bigrams by their POS-tags?\nWrite the RegEx to match (the tags are lower-cased): E.g. j.* to extract adjectives\nLeave empty to ignore") + pos_1 = input("\nWord 1: ") or ".*" + pos_2 = input("\nWord 2: ") or ".*" + + print("\nDo you want to filter the bigrams by the string?\nWrite the RegEx to match: E.g. [a-z].* to extract lower-case only\nLeave empty to ignore") + w_1 = input("\nWord 1: ") or ".*" + w_2 = input("\nWord 2: ") or ".*" + + try: + min_uni_freq = int(input("\nWhat should be the lowest unigram frequency included? ")) + except: + min_uni_freq = 0 + try: + min_bg_freq = int(input("\nWhat should be the lowest bigram frequency included? ")) + except: + min_bg_freq = 0 + try: + max_bg_freq = int(input("\nWhat should be the highest bigram frequency included? ")) + except: + max_bg_freq = 100000000 + + optimizer = "" + while optimizer not in ["quick", "pruning", "q", "p"]: + optimizer = input("\nWhich optimizer should be used? [(Q)uick/(P)runing] ") or "quick" + + if optimizer.startswith("q"): + optimizer = "quick" + else: + optimizer = "pruning" + + try: + seed = int(input("\nWhat should be the seed for random sampling? Leave empty if unsure, this will get you reproducible results. ")) + except: + seed = 1991 + + if optimizer == "quick": + try: + beam_width = int(input("\nHow many samples should be used in the sampling procedure? ")) + except: + beam_width = 1 + + else: + beam_width = 1000 + + return([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + +def saveSettings(settings): + + num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = settings + print("Do you want to save your settings? [y/N]") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + settings = "" + settings += "num = %i\n" % num + settings += "cuts = %i\n" % cuts + settings += "disbalance_penalty = %i\n" % disbalance_penalty + settings += 'w_1 ="' + w_1 + '"\n' + settings += 'w_2 ="' + w_2 + '"\n' + settings += 'pos_1 ="' + pos_1 + '"\n' + settings += 'pos_2 ="' + pos_2 + '"\n' + settings += "max_time = %i\n" % max_time + settings += "min_bg_freq = %i\n" % min_bg_freq + settings += "max_bg_freq = %i\n" % max_bg_freq + settings += "min_uni_freq = %i\n" % min_uni_freq + settings += 'optimizer = "' + optimizer + '"\n' + + with open("search_settings.py", "w+") as f: + f.write(settings) + +def getBins(it2score, cuts=20, type="dict", method = "exact", percent=95): + """Extract the range of scores. Take dict (or dict of dicts if specified), return lower/upper bound. Returns numpy array of edges.""" + if type == "dict": + try: + scores = [it2score[x] for x in it2score] # Get from dictionary to list + except: + print("Conversion not possible. Wrong type selected?") + elif type == "dictOfDicts": + scores = [it2score[x][y] for x in it2score for y in it2score[x]] # Get from dictionary to list + + else: + print("Dicts nested more than once not implemented") + raise ValueError() + + scores = array(scores) + + print(" Cropping extreme %i percent of data" % (100-percent)) + lower, upper = percentile(scores, [(100-percent)/2, ((100-percent)/2)+percent]) + scores = scores[(scores > lower) & (scores < upper)] + + if method == "log": + print(" Defining log-transform-based bin boundaries") + # bins = histogram_bin_edges(scores, cuts) + + translate = (0-amin(scores)) + 1 + #print(amin(scores)) + #print(amax(scores)) + scores = log(scores + translate) + + counts, bins = histogram(scores, min(cuts-2,1)) + # print(counts/sum(counts)) + bins = exp(bins) - translate + scores = exp(scores) - translate + + else: + print(" Defining bin boundaries") + # bins = histogram_bin_edges(scores, cuts) + + #translate = (0-amin(scores)) + 1 + #print(amin(scores)) + #print(amax(scores)) + #scores = log(scores + translate) + + counts, bins = histogram(scores, min(cuts-2,1)) + # print(counts/sum(counts)) + #bins = exp(bins) - translate + #scores = exp(scores) - translate + #plt.hist(scores, bins) + #plt.show() + + # x = input("Press RETURN to continue") + #print(bins) + return(bins) + +def getCorPerformance(scores, cutoff = 0.7): + scores = pd.DataFrame(scores) + cors = abs(scores.corr(method="spearman")) + + highs = sum(nsum(cors > cutoff)) + tiebreaker = nanmedian(cors[cors > cutoff]) + + return([highs, tiebreaker]) + +def getDisbalance(new, old, penalty=3, mod=1): + old = deepcopy(old) + for item in new: + old[arange(old.shape[0]), item] += 1*mod + m = sum(old[0,:])/old.shape[1] + old = old - m + old == old**penalty + return(nsum(old)) + +class ScorerLemma(object): + """Loads all the score lists, can then be used to asign the scores on a per-item base with the score method. + It is memory-heavy, but could be included in functions which allow interactive collocation input.""" + ##Required input format should be added to docstring.. + + def __init__(self): + """Load the required score files. If they are not present in the folder, throw an exception.""" + self.binning = "exact" + self.optimizer = "pruning" + self.beam_width = 5 + import os + files = ["wfreqs_lemma.json", "bigrams_lemma.json","fwd_lemma.json","bckw_lemma.json"] + files = [file_folder +x for x in files] + try: + filecheck = [os.path.isfile(f) for f in files] + if all(filecheck) != True: + raise IOError() + + except: + print("\nFollowing files could not be loaded. Check that they are in the /scores subfolder as this script.") + for i in range(len(files)): + if filecheck[i]==False: + print(files[i]) + print("Exiting") + sys.exit(1) + + try: + print("\nLoading the saved LEMMA scores") + print(" unigram frequency") + with open(file_folder + "wfreqs_lemma.json", "r") as i: + self.wfreq = json.loads(i.read()) + + print(" bigram frequency") + with open(file_folder + "bigrams_lemma.json", "r") as i: + self.bg_frq = json.loads(i.read()) + + print(" TP-D") + with open(file_folder + "fwd_lemma.json", "r") as i: + self.tp_d = json.loads(i.read()) + + print(" TP-B") + with open(file_folder + "bckw_lemma.json", "r") as i: + self.tp_b = json.loads(i.read()) + + # print(" Log likelihood") + # with open(file_folder + "llscore_lemma.json", "r") as i: + # self.log_lklhd = json.loads(i.read()) + + # print(" Dice") + # with open(file_folder + "dicescore_lemma.json", "r") as i: + # self.dice = json.loads(i.read()) + # + # print(" Modified dice") + # with open(file_folder + "moddicescore_lemma.json", "r") as i: + # self.moddice = json.loads(i.read()) + + # print(" t-score") + # with open(file_folder + "tscore_lemma.json", "r") as i: + # self.t_score = json.loads(i.read()) + + # print(" z-score") + # with open(file_folder + "zscore_lemma.json", "r") as i: + # self.z_score = json.loads(i.read()) + # + # print(" delta_p-12") + # with open(file_folder + "delta_p12_lemma.json", "r") as i: + # self.delta_p12 = json.loads(i.read()) + # + # print(" delta_p-21") + # with open(file_folder + "delta_p21_lemma.json", "r") as i: + # self.delta_p21 = json.loads(i.read()) + + # print(" MI-score") + # with open(file_folder + "miscore_lemma.json", "r") as i: + # self.mi_score = json.loads(i.read()) + + # print(" MI3-score") + # with open(file_folder + "mi3score_lemma.json", "r") as i: + # self.mi3_score = json.loads(i.read()) + + # print(" G-score") + # with open(file_folder + "gscore_lemma.json", "r") as i: + # self.g_score = json.loads(i.read()) + print("_________________________________") + + except: + print("Not all score files could be loaded. Check that they are in the same folder as this script.") + print("Exiting") + sys.exit(1) + + def score(self, items): + """Score all the bigrams at once. Input is a list of lists with the sublist format + [item1, item2]. If a bigram is not in the score list, return NA.""" + + items_out = [] + + for bigram in items: + print(self.wfreq) + + w1, w2 = bigram + bigram = " ".join(bigram) + + try: + w1_frq = self.wfreq[w1] + except: + w1_frq = "NA" + + try: + w2_frq = self.wfreq[w2] + except: + w2_frq = "NA" + + try: + bg_frq = self.bg_frq[bigram] + except: + bg_frq = "NA" + + try: + tp_d = self.tp_d[w1][w2] + except: + tp_d = "NA" + + try: + tp_b = self.tp_b[w2][w1] + except: + tp_b = "NA" + + # try: + # log_lklhd = self.log_lklhd[bigram] + # except: + # log_lklhd = "NA" + + # try: + # dice = self.dice[bigram] + # except: + # dice = "NA" + + # try: + # moddice = self.moddice[bigram] + # except: + # moddice = "NA" + + # try: + # t_score = self.t_score[bigram] + # except: + # t_score = "NA" + + # try: + # z_score = self.z_score[bigram] + # except: + # z_score = "NA" + + # try: + # mi_score = self.mi_score[bigram] + # except: + # mi_score = "NA" + + # try: + # mi3_score = self.mi3_score[bigram] + # except: + # mi3_score = "NA" + + # try: + # g_score = self.g_score[bigram] + # except: + # g_score = "NA" + + # try: + # delta_p12 = self.delta_p12[bigram] + # except: + # delta_p12 = "NA" + + # try: + # delta_p21 = self.delta_p21[bigram] + # except: + # delta_p21 = "NA" + + items_out.append([bigram, w1_frq, w2_frq, bg_frq, tp_b, tp_d]) + + return(items_out) + + + def score_one(self, bigram): + """Score the individual bigrams. Input is in the format + [item1, item2]. If a bigram is not in the score list, return NA.""" + + w1, w2 = bigram + bigram = " ".join(bigram) + + try: + w1_frq = self.wfreq[w1] + except: + w1_frq = "NA" + + try: + w2_frq = self.wfreq[w2] + except: + w2_frq = "NA" + + try: + bg_frq = self.bg_frq[bigram] + except: + bg_frq = "NA" + + try: + tp_d = self.tp_d[w1][w2] + except: + tp_d = "NA" + + try: + tp_b = self.tp_b[w2][w1] + except: + tp_b = "NA" + + # try: + # log_lklhd = self.log_lklhd[bigram] + # except: + # log_lklhd = "NA" + + # try: + # dice = self.dice[bigram] + # except: + # dice = "NA" + + # try: + # moddice = self.moddice[bigram] + # except: + # moddice = "NA" + + # try: + # t_score = self.t_score[bigram] + # except: + # t_score = "NA" + + # try: + # z_score = self.z_score[bigram] + # except: + # z_score = "NA" + + # try: + # mi_score = self.mi_score[bigram] + # except: + # mi_score = "NA" + + # try: + # mi3_score = self.mi3_score[bigram] + # except: + # mi3_score = "NA" + + # try: + # g_score = self.g_score[bigram] + # except: + # g_score = "NA" + + # try: + # delta_p12 = self.delta_p12[bigram] + # except: + # delta_p12 = "NA" + + # try: + # delta_p21 = self.delta_p21[bigram] + # except: + # delta_p21 = "NA" + + return([bigram, w1_frq, w2_frq, bg_frq, tp_b, tp_d]) + + def get_random(self, num, cuts=20, seed=1991, disbalance_penalty = 3, max_time=20, words = ["", ""], pos=["",""], + max_bg_freq=100000000, min_bg_freq=0, min_uni_freq=5, percent=95): + num = max(num,1) + """Get num random bigrams, spread approximately evenly accross the ranges of the scores.""" + print("\nFinishing initialization") + ### FILTERING BY POS + + self.bgs_all = [x.split() for x in tqdm.tqdm(self.bg_frq) if (" ".join(x.split()) in self.bg_frq)] + + if (min_bg_freq > 0) or (max_bg_freq < 100000000): + print(" Selecting bigrams by bigram frequency") + self.bgs = [[x,y] for x,y in tqdm.tqdm(self.bgs_all) if (self.bg_frq[" ".join([x,y])] > min_bg_freq) and (self.bg_frq[" ".join([x,y])] < max_bg_freq)] + else: + self.bgs = self.bgs_all + + if min_uni_freq > 1: + print(" Selecting bigrams by unigram frequency") + self.bgs = [[x,y] for x,y in tqdm.tqdm(self.bgs) if (self.wfreq[x]>=min_uni_freq and self.wfreq[y]>=min_uni_freq)] + else: + self.bgs = self.bgs + + if pos[0] != "" or pos[1] != "": + pos_1, pos_2 = pos + pos_1 = compile(pos_1) + pos_2 = compile(pos_2) + print(" Selecting bigrams by POS") + self.bgs = [[x,y] for x,y in tqdm.tqdm(self.bgs) if (match(pos_1, x.split("_")[1])!=None and match(pos_2,y.split("_")[1])!=None)] + + if words[0] != "" or words[1] != "": + w_1, w_2 = words + w_1 = compile(w_1) + w_2 = compile(w_2) + print(" Selecting bigrams by words 1 & 2") + self.bgs = [[x,y] for x,y in tqdm.tqdm(self.bgs) if (match(w_1, x.split("_")[0])!=None and match(w_2,y.split("_")[0])!=None)] + + print("_"*10+"\nRelevant bigrams: %i" % len(self.bgs)) + + # print(" Cleaning unfitting bigrams") + # print(" Initializing cleaner") + + # tempbgs = set([" ".join(x) for x in self.bgs]) + # wrong_keys = [x for x in tqdm.tqdm(self.bg_frq) if not x in tempbgs] + # del tempbgs + # print(" Cleaning") + # wrong_keys = [] + # for wrong_key in tqdm.tqdm(wrong_keys): + # try: + # c = wrong_key + # wrong_key = wrong_key.split() + # del self.bg_frq[c] + # del self.tp_d[wrong_key[0]][wrong_key[1]] + # del self.tp_b[wrong_key[1]][wrong_key[0]] + # del self.log_lklhd[c] + # del self.dice[c] + # del self.t_score[c] + # del self.z_score[c] + # del self.mi_score[c] + # del self.mi3_score[c] + # del self.g_score[c] + # del self.delta_p12[c] + # del self.delta_p21[c] + # except: + # pass + + # del wrong_keys + + if (len(self.bgs) <= num) and len(self.bgs) > 0: + print("Random sampling failed: There are fewer relevant cases than the sample size.") + scores = [self.score(x) for x in self.bgs] # remove bigram string to allow numpy operation + scores = [array([x[1:] for x in y]) for y in scores] + results = [[sub("_[^ ]+",""," ".join(self.bgs[x]))]+list(scores[x,:]) for x in range(min(len(self.bgs), num))] + + return(results) + + elif len(self.bgs) == 0: + raise ValueError("No case matches the criteria.") + + print("\nDefining data distributions") + + print(" unigram frequency") + self.wfreq_bins = getBins(self.wfreq, cuts=cuts, method = self.binning, percent=percent) + + print(" bigram frequency") + self.bg_frq_bins = getBins(self.bg_frq, cuts=cuts, method = self.binning, percent=percent) # Get from dictionary to list + + + if self.optimizer == "pruning": + print(" TP-D") + + self.tp_d_bins = getBins(self.tp_d, cuts=cuts, type="dictOfDicts", percent=percent) # Get from dictionary to list + + print(" TP-B") + self.tp_b_bins = getBins(self.tp_b, cuts=cuts, type="dictOfDicts", percent=percent) # Get from dictionary to list + + print(" Log likelihood") + self.log_lklhd_bins = getBins(self.log_lklhd, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" Dice") + self.dice_bins = getBins(self.dice, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" Modified dice") + self.moddice_bins = getBins(self.moddice, cuts=cuts, percent=percent) + + print(" t-score") + self.t_score_bins = getBins(self.t_score, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" z-score") + self.z_score_bins = getBins(self.z_score, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" MI-score") + self.mi_score_bins = getBins(self.mi_score, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" MI3-score") + self.mi3_score_bins = getBins(self.mi3_score, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" G-score") + self.g_score_bins = getBins(self.g_score, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" Delta_p-12") + self.delta_p12_bins = getBins(self.delta_p12, cuts=cuts, percent=percent) # Get from dictionary to list + + print(" Delta_p-21") + self.delta_p21_bins = getBins(self.delta_p21, cuts=cuts, percent=percent) # Get from dictionary to list + + print("_________________________________") + + buf = 10 # The bufferring coefficient (how many extra elements should be collected) + self.dist = zeros([15, cuts]) # Array to save the distributions: columns=bins, rows=scores + self.results = zeros([num*buf, 15]) # Array to save the results + self.items = [] + self.populated = 0 + self.disbalance_penalty = floor(num*disbalance_penalty/100) + # self.indexes = arange(length(self.bg_pos)) + + random.seed(seed) + nrandom.seed(seed) + + print("\nStarting item selection") + + start = time.time() + lasttime = time.time() + + max_time = max_time*60 + pbar = [tqdm.tqdm(total = num), tqdm.tqdm(total=num*(buf-1))] + pbar[0].set_description("Stimuli collected") + pbar[1].set_description("Additional buffer") + reached = False + + while (self.populated < num*buf) and (time.time() - start) < max_time: # Get a random item; check which bins would it increase for which score, if this disturbs balance, drop otherwise insert at the bottom + if self.populated >= num and reached == False: + reached = True + + samples = [nrandom.choice(range(len(self.bgs)),10, replace=False) for x in range(1000)] + samples = [[self.bgs[y] for y in x] for x in samples] + scores = [self.score(x) for x in samples] # remove bigram string to allow numpy operation + scores = [array([x[1:] for x in y]) for y in scores] + + binned = [array([digitize(x[:,0],self.bg_frq_bins), + digitize(x[:,1],self.wfreq_bins), + digitize(x[:,2],self.wfreq_bins), + digitize(x[:,3],self.tp_b_bins), + digitize(x[:,4],self.tp_d_bins), + digitize(x[:,5],self.log_lklhd_bins), + digitize(x[:,6],self.dice_bins), + digitize(x[:,7],self.moddice_bins), + digitize(x[:,8],self.t_score_bins), + digitize(x[:,9],self.z_score_bins), + digitize(x[:,10],self.mi_score_bins), + digitize(x[:,11],self.mi3_score_bins), + digitize(x[:,12],self.g_score_bins), + digitize(x[:,13],self.delta_p12_bins), + digitize(x[:,14],self.delta_p21_bins)]) for x in scores] + + # print(binned[0]) + binned = [transpose(x) for x in binned] + # binned = [] + performance = [getDisbalance(x, self.dist, self.disbalance_penalty) for x in binned] + best = argmin(performance) + + self.results[self.populated:self.populated+10,:] = scores[best] + for item in binned[best]: + self.dist[arange(15), item] += 1 + self.items += samples[best] + self.populated += 10 + if reached: + pbar[1].update(10) + else: + pbar[0].update(10) + + pbar[0].close() + pbar[1].close() + + if self.populated >= num and (time.time() - start) < max_time: + print("\nPruning") + + binned = array([digitize(self.results[:,0],self.bg_frq_bins), + digitize(self.results[:,1],self.wfreq_bins), + digitize(self.results[:,2],self.wfreq_bins), + digitize(self.results[:,3],self.tp_b_bins), + digitize(self.results[:,4],self.tp_d_bins), + digitize(self.results[:,5],self.log_lklhd_bins), + digitize(self.results[:,6],self.dice_bins), + digitize(self.results[:,7],self.moddice_bins), + digitize(self.results[:,8],self.t_score_bins), + digitize(self.results[:,9],self.z_score_bins), + digitize(self.results[:,10],self.mi_score_bins), + digitize(self.results[:,11],self.mi3_score_bins), + digitize(self.results[:,12],self.g_score_bins), + digitize(self.results[:,13],self.delta_p12_bins), + digitize(self.results[:,14],self.delta_p21_bins)]) + binned = transpose(binned) + + print("Removing duplicates") + firsts = set() + seconds = set() + dels = [] + for x in tqdm.tqdm([y for y in range(self.populated)]): + w1,w2 = self.items[x] + if w1 in firsts or w2 in seconds: + dels.append(x) + else: + firsts.update(w1) + seconds.update(w2) + + dels.reverse() + for d_index in tqdm.tqdm(dels): + del self.items[d_index] + binned = delete(binned, dels,0) + self.results = delete(self.results, dels, 0) + self.populated -= len(dels) + + pbar = tqdm.tqdm(total=self.populated - num) + while self.populated > num and (time.time() - start) < max_time: # If there is time left, prune the most problematic items away, one by one + rands = [random.randint(0,binned.shape[0]-1) for x in range(1000)] + performance = [getDisbalance(binned[i,:], self.dist, self.disbalance_penalty, mod=-1) for i in rands] + best = rands[argmin(performance)] + self.dist[arange(15), binned[best,:]] -= 1 + binned = delete(binned, best, 0) + del self.items[best] + self.results = delete(self.results, best, 0) + self.populated -= 1 + pbar.update(1) + + print("\nSuccess! All %i items were found." % num) + + + else: + print("\nTimeout limit exceeded. Returning %i items" % self.populated) + results = self.results[0:min(self.populated, num)] # Crop which we don't have + + #plt.imshow(self.dist, cmap="hot", interpolation="bilinear") + #plt.suptitle("Distribution accross scores") + #plt.xlabel("Score bin") + #plt.ylabel("Score") + #plt.show() + + results = [[sub("_[^ ]+",""," ".join(self.items[x]))]+list(self.results[x, 0:15]) for x in range(min(self.populated, num))] + + return(results) + + if self.optimizer == "quick": + + beam = self.beam_width # The beam width to keep best samples + self.dist = zeros([15, cuts]) # Array to save the distributions: columns=bins, rows=scores + self.results = zeros([num, 15]) # Array to save the results + self.items = [] + + random.seed(seed) + nrandom.seed(seed) + + print("\nStarting item selection") + + start = time.time() + + max_time = max_time*60 + + + best = 15**2 + tiebreak = 1 + best_sample = [] + + pbar = tqdm.tqdm(total = beam) + pbar.set_description("Samples tried") + for sample in range(beam): + sample = list(nrandom.choice([x for x in range(len(self.bgs))], min(floor(num*1.1), len(self.bgs)), replace=False)) + sample = [self.bgs[x] for x in sample] + + scores = self.score(sample) # remove bigram string to allow numpy operation + scores = array([x[1:] for x in scores]) + performance, tiebreaker = getCorPerformance(scores) + + if performance < best: + best = performance + best_sample = sample + tiebreak = tiebreaker + elif (performance == best) and (tiebreaker < tiebreak): + best = performance + best_sample = sample + tiebreak = tiebreaker + + pbar.update(1) + if (time.time() - start) >= max_time: + break + + pbar.close() + + # print(best_sample) + if (time.time() - start) < max_time: + print("Removing duplicates") + firsts = set() + seconds = set() + dels = [] + for x in tqdm.tqdm([y for y in range(len(best_sample))]): + w1,w2 = best_sample[x] + if w1 in firsts or w2 in seconds: + dels.append(x) + else: + firsts.update(w1) + seconds.update(w2) + + dels.reverse() + for d_index in tqdm.tqdm(dels): + del best_sample[d_index] + + results = best_sample[0:num] + + else: + print("\nTimeout limit exceeded, returning best sample at this moment.") + results = best_sample[0:min(num, len(best_sample))] # Crop which we don't have + + #plt.imshow(self.dist, cmap="hot", interpolation="bilinear") + #plt.suptitle("Distribution accross scores") + #plt.xlabel("Score bin") + #plt.ylabel("Score") + #plt.show() + + results = self.score(best_sample) + results = pd.DataFrame(results, columns = ["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + results["bigram_lemma"] = [sub("_[^ ]+","",x) for x in results["bigram_lemma"]] + results = results.values + # print(results) + return(results) + +if __name__ == "__main__": + mode = None + while mode not in ["score", "search", "strat_search", "match"]: + mode = input("Which mode should this program run in?\n Score/search/strat_search/match: ") + + if mode.lower() == "score": + + if len(sys.argv) > 1: + inpath = sys.argv[1] + else: + inpath = input("Where is the file to load?\n ") + + try: + with open(inpath, "r") as infile: + items = infile.readlines() + items = [x.split() for x in items] + if all([len(x)==2 for x in items]) == False: + raise IOError() + + except: + print("The input file does not seem to be formatted correctly (one bigram per line)") + sys.exit(2) + + ram_present = psutil.virtual_memory()[0] >> 30 + ram_available = psutil.virtual_memory()[1] >> 30 + + # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version + if ram_present > 7 and ram_available > 5: + scorer = ScorerLemma() + else: + print("This is a RAM-intensive operation. You need at least 6 GB of free RAM.") + print("Exiting...") + sys.exit(0) + + items = scorer.score(items) + + if len(sys.argv) > 2: + outpath = sys.argv[2] + else: + outpath = input("Where should the results be saved?\n ") + + print("Saving") + with open(outpath, "w+") as outfile: + out_csv = csv.writer(outfile) + out_csv.writerow(["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma"]) + for i in tqdm.tqdm(items): + out_csv.writerow(i) + print("Done. Press RETURN to exit") + wait = input() + sys.exit(0) + + elif mode.lower() == "search": + + ram_present = psutil.virtual_memory()[0] >> 30 + ram_available = psutil.virtual_memory()[1] >> 30 + + # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version + if ram_present > 7: + # if ram_present > 7 and ram_available > 5: + pass + else: + print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") + sys.exit(0) + + max_time = 0 + disbalance_penalty = 0 + cuts = 0 + num = 0 + pos = "" + + saved = False + if os.path.isfile("search_settings.py"): + print("Saved settings found. Do you want to use them? [Y/n]") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[Y/n]") or "Y" + + if dec.lower() == "y": + saved = True + + if saved == True: + from search_settings import * + print("Using saved settings") + + else: + num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = getSettings() + + scorer = ScorerLemma() + scorer.optimizer = optimizer + items = scorer.get_random(num, cuts=cuts, seed=seed, disbalance_penalty = disbalance_penalty, words=[w_1, w_2], pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=min_bg_freq, max_bg_freq=max_bg_freq, min_uni_freq=min_uni_freq) + + print("Saving") + + if len(sys.argv) > 2: + outpath = sys.argv[2] + else: + outpath = input("Where should the results be saved?\n ") + + print("Saving") + with open(outpath, "w+") as outfile: + out_csv = csv.writer(outfile) + out_csv.writerow(["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + for i in tqdm.tqdm(items): + out_csv.writerow(i) + + saveSettings([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + + + elif mode.lower() == "strat_search": + + if len(sys.argv) > 2: + outpath = sys.argv[2] + else: + outpath = input("Where should the results be saved?\n ") + + with open("_temp.csv", "w+") as outfile: + out_csv = csv.writer(outfile) + out_csv.writerow(["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + + ram_present = psutil.virtual_memory()[0] >> 30 + ram_available = psutil.virtual_memory()[1] >> 30 + + # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version + if ram_present > 7: + # if ram_present > 7 and ram_available > 5: + pass + else: + print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") + sys.exit(0) + + max_time = 0 + disbalance_penalty = 0 + cuts = 0 + num = 0 + pos = "" + + saved = False + if os.path.isfile("search_settings.py"): + print("Saved settings found. Do you want to use them? [Y/n]") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[Y/n]") or "Y" + + if dec.lower() == "y": + saved = True + + if saved == True: + from search_settings import * + print("Using saved settings") + + else: + num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = getSettings() + + scorer = ScorerLemma() + + scorer.beam_width = beam_width + scorer.binning = "exact" + items = scorer.get_random(1, cuts=10, seed=seed, disbalance_penalty = disbalance_penalty, words=[w_1, w_2], + pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=min_bg_freq, max_bg_freq=max_bg_freq, min_uni_freq=min_uni_freq, percent=95) + + bins = list(scorer.bg_frq_bins) + [max_bg_freq] + bins[0] = [min_bg_freq] + + scorer.binning = "exact" + scorer.optimizer = optimizer + print(bins) + iter = 0 + for strat in tqdm.tqdm(range(len(bins)-1)): + items = scorer.get_random(floor((num*1.5)/(len(bins)-1)), cuts=cuts, seed=seed+iter, disbalance_penalty = disbalance_penalty, words=[w_1, w_2], + pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=floor(bins[strat]), max_bg_freq=ceil(bins[strat+1])+1, min_uni_freq=min_uni_freq, percent=100) + iter +=1 + + with open("_temp.csv", "a") as outfile: + out_csv = csv.writer(outfile) + for i in tqdm.tqdm(items): + out_csv.writerow(i) + + print("Saving") + scores = pd.read_csv("_temp.csv") + scores["w1"], scores["w2"] = scores["bigram"].str.split(' ', 1).str + scores.index = range(scores.shape[0]) + scores = scores.drop_duplicates(subset=["bigram"], keep="last") + scores = scores.drop_duplicates(subset=["w1"], keep="last") + scores = scores.drop_duplicates(subset=["w2"], keep="last") + scores.drop(["w1", "w2"], 1, inplace=True) + + if scores.shape[0] > num: + ints = sorted(nrandom.choice(range(scores.shape[0]), num, replace=False)) + scores = scores.iloc[ints,:] + + scores.to_csv(outpath, index=False) + try: + os.remove("_temp.csv") + except: + print("Couldn't remove the temp file.") + + saveSettings([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + + sys.exit(0) + + elif mode.lower() == "match": + infile = input("Where is the file with selected stimuli? ") or "not_a_file" + while os.path.isfile(infile) != True: + infile = input("Not a valid file, try to specify the full path. ") or "not_a_file" + + stimuli = pd.read_csv(infile) + stimuli.dropna(0, how="all", subset=["bigram"], inplace=True) + + try: + matchWord = int(input("Which word should be matched? [1-%i]" % len(stimuli["bigram"][0].split()))) - 1 + except: + matchWord = 0 + + try: + min_t_score = int(input("What is the lowest accepted t-score?")) + except: + min_t_score = -1000000 + + try: + max_t_score = int(input("What is the highest accepted t-score?")) + except: + max_t_score = +1000000 + + keys = set([x.split()[matchWord] for x in list(stimuli["bigram"].values)]) + non_matched = 0 if matchWord==1 else 1 + forbidden = set([x.split()[non_matched] for x in list(stimuli["bigram"].values)]) + + if len(sys.argv) > 2: + outpath = sys.argv[2] + else: + outpath = input("Where should the results be saved?\n ") + + with open("_temp.csv", "w+") as outfile: + out_csv = csv.writer(outfile) + out_csv.writerow(["bigram_lemma", "w1_freq_lemma", "w2_freq_lemma", "bigram_freq_lemma", "tp_b_lemma", "tp_d_lemma", "log_lklhd_lemma", "dice_lemma", "moddice_lemma", "t_score_lemma", "z_score_lemma", "mi_score_lemma", "mi3_score_lemma", "g_score_lemma", "delta_p12_lemma", "delta_p21_lemma"]) + + ram_present = psutil.virtual_memory()[0] >> 30 + ram_available = psutil.virtual_memory()[1] >> 30 + + # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version + if ram_present > 7: + # if ram_present > 7 and ram_available > 5: + pass + else: + print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") + sys.exit(0) + + max_time = 0 + disbalance_penalty = 0 + cuts = 0 + num = 0 + pos = "" + + saved = False + if os.path.isfile("search_settings.py"): + print("Saved settings found. Do you want to use them? [Y/n]") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[Y/n]") or "Y" + + if dec.lower() == "y": + saved = True + + if saved == True: + from search_settings import * + print("Using saved settings") + + else: + num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width = getSettings() + + print("\n_____________________\nGrab a coffee...or ten") + + scorer = ScorerLemma() + ##### + # Prefilter the data for only the adjectives? + # Prefilter the data by t-score? + ##### + + print("Pre-filtering the data for\n\t-matched words\n\t-fitting t-score") + scorer.bg_frq = {k:v for k,v in tqdm.tqdm(scorer.bg_frq.items()) if (k.split(" ")[matchWord].split("_")[0] in keys) and ((scorer.t_score[k] >= min_t_score) and (scorer.t_score[k] <= max_t_score))} + + scorer.beam_width = beam_width + scorer.binning = "exact" + scorer.optimizer = optimizer + + iter = 0 + fails = [] + for key in tqdm.tqdm(keys): + print(key) + try: + matched_pattern = [w_1, w_2] + matched_pattern[matchWord] = key + items = scorer.get_random(num, cuts=cuts, seed=seed+iter, disbalance_penalty = disbalance_penalty, words=matched_pattern, + pos=[pos_1, pos_2], max_time=max_time, min_bg_freq=min_bg_freq, max_bg_freq=max_bg_freq, min_uni_freq=min_uni_freq, percent=95) + iter +=1 + + with open("_temp.csv", "a") as outfile: + out_csv = csv.writer(outfile) + for i in tqdm.tqdm(items): + out_csv.writerow(i) + except: + fails.append(key) + + print("Saving") + scores = pd.read_csv("_temp.csv") + if scores.shape[0] > 0: + scores["w1"], scores["w2"] = scores["bigram"].str.split(' ', 1).str + scores.index = range(scores.shape[0]) + scores = scores.drop_duplicates(subset=["bigram"], keep="last") + + if non_matched == 0: + scores = scores[~scores["w1"].isin(forbidden)] + else: + scores = scores[~scores["w2"].isin(forbidden)] + + scores.sort_values(by=["t_score_lemma"], inplace=True, ascending=False) + scores = scores.drop_duplicates(subset=["w2"], keep="last") + scores = scores.drop_duplicates(subset=["w1"], keep="last") + + scores.drop(["w1", "w2"], 1, inplace=True) + + # if scores.shape[0] > num: + # ints = sorted(nrandom.choice(range(scores.shape[0]), num, replace=False)) + # scores = scores.iloc[ints,:] + + scores.to_csv(outpath, index=False) + + print("________\nDone.\n\n\n\n") + if len(fails) > 0: + print("Could not find any matches for the following items:") + for fail in fails: + print("\t%s" % fail) + wait = input("Press RETURN to continue.") or "" + else: + print("_________\nFail: could not find anything.\n\n\n") + try: + os.remove("_temp.csv") + except: + print("Couldn't remove the temp file.") + + saveSettings([num, cuts, disbalance_penalty, max_time, pos_1, pos_2, w_1, w_2, min_uni_freq, min_bg_freq, max_bg_freq, seed, optimizer, beam_width]) + + sys.exit(0) diff --git a/coca_stats.py b/coca_stats.py index 841df5e..d0362b2 100644 --- a/coca_stats.py +++ b/coca_stats.py @@ -5,7 +5,7 @@ import shutil import pickle from decimal import Decimal -import settings +import settings_stats import re from multiprocessing import Pool, Manager import collections @@ -16,20 +16,21 @@ import os import sys -path_to_coca = "C:/projects/COCA" +path_to_coca = "/Users/kylamcconnell/Documents/coca_2019_test" +#path_to_coca = "E:/coca_2019_wlp" # Helper functions to allow multiprocessing def gscorer(items): """Take a string line (JSON serialized list) as input, return bigram + G-score""" global gscores bigram, bf, item1_2, item2_2, item1_3, item2_3 = json.loads(items) - + score1 = (Decimal(bf) * Decimal(item1_2))/Decimal(item1_3) score2 = (Decimal(bf) * Decimal(item2_2))/Decimal(item2_3) - + return([bigram, float(score1.ln() + score2.ln())]) -# The operations in calculation of Log-likelihood. +# The operations in calculation of Log-likelihood. # This allows list comprehension to be used instead of searching through the list one item at a time order = [1, 1, 1, 1, -1, -1, -1, -1, 1] def llscorer(items): @@ -46,40 +47,84 @@ def llscorer(items): - (a+b)*log(a+b) - (a+c)*log(a+c) - (b+d)*log(b+d) - (c+d)*log(c+d) + (a+b+c+d)*log(a+b+c+d))""" - + global ll_score bigram, a,b,c,d = json.loads(items) - + a = Decimal(a) b = Decimal(b) c = Decimal(c) d = Decimal(d) - + base = [a, b, c, d, a+b, a+c, b+d, c+d, a+b+c+d] - - logs = [Decimal(log(x, 10)) for x in base] + + logs = [Decimal(log(x, 10)) if x > 0 else Decimal(0) for x in base] parts = [x*y for x,y in zip(base, logs)] parts = [x*float(y) for x,y in zip(order, parts)] - + return([bigram, 2*sum(parts)]) def preprocess(filename, queue): - # print(filename) - with open(filename, "r") as i: + with open(filename, "r", errors="replace") as i: doc = i.read() - doc = re.sub("_", "", doc) - doc = doc.split("\n") - doc = [word.split("\t") for word in doc] - doc = [word for word in doc if len(word) == 3] - doc = [word for word in doc if "@" != word[0]] - doc = ["_".join([word[0], word[2]]) for word in doc] - doc = [str(doc[x]).strip() + " " + str(doc[x+1]).strip() for x in range(len(doc)-1) if not (doc[x].endswith("_y") or doc[x+1].endswith("_y"))] # Drop bigrams that have the full stop in them - - queue.put(json.dumps(doc)) + #doc = re.sub("_.*", "", doc) + #doc = re.sub("\t+", "\t", doc) + doc = doc.split("\n") + doc = [word.split("\t") for word in doc] + + docs = [] + d = [] + declined = re.compile("@_") + for word in doc: + if len(word) == 4: #rows of length 3 were those that had been removed for copyright reasons + pos = re.sub("_.*", "", word[-1]) + d.append([word[1].lower(), pos]) #changed from 0 2 to 1 3 + + d = ["_".join(w).strip() for w in d] + d = [x + " " + y for x,y in zip(d[0:-1], d[1:]) if re.match("\W", x) == None and re.match("\W", y) == None] + d = [x for x in d if re.match(declined, x) == None] + + docs.append(d) + + docs = [item for sublist in docs for item in sublist] + #print(docs) + + queue.put(json.dumps(docs)) + + +def preprocess_wfreq(filename, queue): + # print(filename) + with open(filename, "r", errors="replace") as i: + doc = i.read() + + #doc = re.sub("_.*", "", doc) + #doc = re.sub("_", "", doc) + #doc = re.sub("\t+", "\t", doc) + doc = doc.split("\n") + doc = [word.split("\t") for word in doc] + + docs = [] + d = [] + declined = re.compile("@_") + for word in doc: + if len(word) == 4: + pos = re.sub("_.*", "", word[-1]) + d.append([word[1].lower(), pos]) #changed from 0 2 to 1 3 + + d = ["_".join(w).strip() for w in d] + d = [x for x in d if re.match("\W", x) == None] + d = [x for x in d if re.match(declined, x) == None] + + docs.append(d) + + docs = [item for sublist in docs for item in sublist] + + queue.put(json.dumps(docs)) + -def listener(queue): - f = open("_COCA2.txt", 'w') +def listener(queue, filename): + f = open(filename, 'w') while 1: m = queue.get() if m == 'kill': @@ -87,26 +132,26 @@ def listener(queue): f.write(str(m) + '\n') f.flush() f.close() - -# Multiprocessing needs this if-statement, otherwise it won't work properly + +# Multiprocessing needs this if-statement, otherwise it won't work properly if __name__ == "__main__": ram_present = psutil.virtual_memory()[0] >> 30 if ram_present < 7: print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") - sys.exit(0) - - total, used, free = shutil.disk_usage("\\") - print("Free drive space: %d/%d GB" % ((free // (2**30)), (total // (2**30)))) - if (free // (2**30)) < 15: - print("WARNING: This is space-intensive operation. It cannot continue if you don't have at least 15 GB of free space on the same drive as the script.\nExiting...") - sys.exit(0) - + sys.exit(0) + + # total, used, free = shutil.disk_usage("\\") + # print("Free drive space: %d/%d GB" % ((free // (2**30)), (total // (2**30)))) + # if (free // (2**30)) < 7: + # print("WARNING: This is space-intensive operation. It cannot continue if you don't have at least 15 GB of free space on the same drive as the script.\nExiting...") + # sys.exit(0) + print("Initializing...") files = [] for dirpath, dirnames, filenames in os.walk(path_to_coca): files.extend([os.path.join(dirpath, file) for file in filenames]) - + if os.path.isfile("_COCA2.txt"): print("A file _COCA2.txt was found in the script's directory.") print("This could be the preprocessed corpus. In that case, you can skip the preprocessing.") @@ -115,17 +160,17 @@ def listener(queue): dec = "" while dec.lower() not in set(["y", "n"]): dec = input("[y/N]") or "N" - + if dec.lower() == "y": f = open("_COCA2.txt", 'w+') f.close() - + manager = Manager() - queue = manager.Queue() + queue = manager.Queue() pool = Pool(4) #put listener to work first - watcher = pool.apply_async(listener, (queue,)) + watcher = pool.apply_async(listener, (queue, "_COCA2.txt")) #fire off workers jobs = [] @@ -134,25 +179,25 @@ def listener(queue): jobs.append(job) # collect results from the workers through the pool result queue - for job in tqdm.tqdm(jobs): + for job in tqdm.tqdm(jobs): job.get() #now we are done, kill the listener queue.put('kill') pool.close() - + else: pass - + else: f = open("_COCA2.txt", 'w+') - f.close() + f.close() manager = Manager() - queue = manager.Queue() + queue = manager.Queue() pool = Pool(4) #put listener to work first - watcher = pool.apply_async(listener, (queue,)) + watcher = pool.apply_async(listener, (queue, "_COCA2.txt")) #fire off workers jobs = [] @@ -161,56 +206,145 @@ def listener(queue): jobs.append(job) # collect results from the workers through the pool result queue - for job in tqdm.tqdm(jobs): + for job in tqdm.tqdm(jobs): job.get() #now we are done, kill the listener queue.put('kill') pool.close() - print(" Bigrams collected") - + print(" Bigrams collected") + + files = [] + for dirpath, dirnames, filenames in os.walk(path_to_coca): + files.extend([os.path.join(dirpath, file) for file in filenames]) + + if os.path.isfile("_COCA2wfreq.txt"): + print("A file _COCA2wfreq.txt was found in the script's directory.") + print("This could be the preprocessed corpus. In that case, you can skip the preprocessing.") + print("Otherwise, this script can delete it and preprocess the corpus.") + print("Clean the file _COCA2.txt?") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + f = open("_COCA2wfreq.txt", 'w+') + f.close() + + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2wfreq.txt" )) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess_wfreq, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + else: + pass + + else: + f = open("_COCA2wfreq.txt", 'w+') + f.close() + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2wfreq.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess_wfreq, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + print(" Wfreq preprocessing finished") + print("Preprocessing finished") print("Counting scores") - - print(" Bigram frequency") - #- Bigram frequency (bi.freq.NXT) - coca = open("_COCA2.txt", "r") - counter = collections.Counter() + #skip = input("Skip?").lower() - for line in tqdm.tqdm(coca): - bgs = json.loads(line) - bgs = [x.strip() for x in bgs if not x.startswith("##")] - counter.update(bgs) - - coca.close() - print(" Cropping the bigram dict to items with freq > 4") - counter = {k:v for k,v in counter.items() if v > 4} - counter = dict(counter) - - # backup bigram stats file - print(" Saving") - backup_out = open("bigrams.json", "w+") - backup_out.write(json.dumps(counter)) - backup_out.close() - - print(" Making a wordcount from %i bigrams" % (len(counter))) - w_freq = collections.Counter() - for item in tqdm.tqdm(counter): # Get the word frequency for each word - buffer = counter[item]*item.split() - w_freq.update(buffer) - - w_freq = dict(w_freq) + skip = "n" + + if skip == "n": + print(" Bigram frequency") + #- Bigram frequency (bi.freq.NXT) + + + coca = open("_COCA2.txt", "r") + counter = collections.Counter() + + for line in tqdm.tqdm(coca): + bgs = json.loads(line) + bgs = [x.strip() for x in bgs if not x.startswith("##")] + counter.update(bgs) + # print("\n\n\n\nEARLY STOPPING") + # break + + coca.close() + print(" Cropping the bigram dict to items with freq > 0") #changed 4 to 0 + counter = {k:v for k,v in counter.items() if v > 0} #changed 4 to 0 + counter = dict(counter) + + # backup bigram stats file + print(" Saving") + + backup_out = open("bigrams.json", "w+") + backup_out.write(json.dumps(counter)) + backup_out.close() + + print(" Making a wordcount from %i bigrams" % (len(counter))) + w_freq = collections.Counter() + coca = open("_COCA2wfreq.txt", "r") + for line in tqdm.tqdm(coca): + bgs = json.loads(line) + bgs = [x.strip() for x in bgs if not x.startswith("##")] + w_freq.update(bgs) + + w_freq = dict(w_freq) + coca.close() + + backup_out = open("wfreqs.json", "w+") + backup_out.write(json.dumps(w_freq)) + backup_out.close() + + with open("bigrams.json", "r") as i: + counter = json.loads(i.read()) + with open("wfreqs.json", "r") as i: + w_freq = json.loads(i.read()) - backup_out = open("wfreqs.json", "w+") - backup_out.write(json.dumps(w_freq)) - backup_out.close() - w_count = 0 - for item in w_freq: # Sum the word frequency to get the total - w_count += w_freq[item] - + for item, frq in w_freq.items(): # Sum the word frequency to get the total + w_count += frq + + print("Keeping only nice bigrams") + bad = len(counter) + counter = {k:v for k,v in counter.items() if (k.split()[0] in w_freq) and (k.split()[1] in w_freq)} + print("Kept %i/%i bigrams" % (len(counter), bad)) + print(" TP-D/TP-B") #- Direct transitional probability (TPD.bi.NXT) @@ -220,283 +354,328 @@ def listener(queue): forward_pairs = {} backward_pairs = {} - + print(" Dictionarizing word-pair counts") #create dict of dicts: how many times is a given word followed by another word and vice versa - - for item in tqdm.tqdm(counter): + + for item, freq in tqdm.tqdm(counter.items()): x_word = item.split()[0] y_word = item.split()[1] - # Check the forward freq dictionary, if word x+1 there, add 1 to freq. - # If word x+1 not there add it with freq = 1. + # Check the forward freq dictionary, if word x+1 there, add 1 to freq. + # If word x+1 not there add it with freq = 1. # If word x not there, create it and add word x+1 as first item with freq = 1. if x_word in forward_pairs: if y_word in forward_pairs[x_word]: - forward_pairs[x_word][y_word] += 1 + forward_pairs[x_word][y_word] += freq else: - forward_pairs[x_word][y_word] = 1 + forward_pairs[x_word][y_word] = freq else: - forward_pairs[x_word] = {y_word : 1} - - # Check the backward freq dictionary, if word y-1 there, add 1 to freq. - # If word y-1 not there add it with freq = 1. - # If word y not there, create it and add word y-1 as first item with freq = 1. + forward_pairs[x_word] = {y_word : freq} + + # Check the backward freq dictionary, if word y-1 there, add 1 to freq. + # If word y-1 not there add it with freq = 1. + # If word y not there, create it and add word y-1 as first item with freq = 1. if y_word in backward_pairs: if x_word in backward_pairs[y_word]: - backward_pairs[y_word][x_word] += 1 + backward_pairs[y_word][x_word] += freq else: - backward_pairs[y_word][x_word] = 1 + backward_pairs[y_word][x_word] = freq else: - backward_pairs[y_word] = {x_word : 1} + backward_pairs[y_word] = {x_word : freq} print(" Pairs collected") - print(" Counting TPD") - # Calculates forward probabilities and saves them as a Decimal(probability) - # The output variable forward_probs is a dict of dicts, form: {"word_x": {"word_x+1(1)": Decimal(0.73), "word_x+1(2)": Decimal(0.27)}} - forward_probs = {} - for x_word in tqdm.tqdm(forward_pairs): - total = Decimal(0) - for item in forward_pairs[x_word]: - total += Decimal(forward_pairs[x_word][item]) - forward_probs[x_word] = {} - for item in forward_pairs[x_word]: - forward_probs[x_word][item] = float(Decimal(forward_pairs[x_word][item])/total) - - del forward_pairs - - print(" Counting TPB") - # Calculates forward probabilities and saves them as a Decimal(probability) - # The output variabe forward_probs is a dict of dicts, form: {"word_x": {"word_x+1(1)": Decimal(0.73), "word_x+1(2)": Decimal(0.27)}} - backward_probs = {} - for x_word in tqdm.tqdm(backward_pairs): - total = Decimal(0) - for item in backward_pairs[x_word]: - total += Decimal(backward_pairs[x_word][item]) - backward_probs[x_word] = {} - for item in backward_pairs[x_word]: - backward_probs[x_word][item] = float(Decimal(backward_pairs[x_word][item])/total) - - del backward_pairs - - backup_out = open("fwd.json", "w+") - backup_out.write(json.dumps(forward_probs)) - backup_out.close() - - backup_out = open("bckw.json", "w+") - backup_out.write(json.dumps(backward_probs)) - backup_out.close() + if skip == "n": + + print(" Counting TPD") + # Calculates forward probabilities and saves them as a Decimal(probability) + # The output variable forward_probs is a dict of dicts, form: {"word_x": {"word_x+1(1)": Decimal(0.73), "word_x+1(2)": Decimal(0.27)}} + forward_probs = {} + for x_word in tqdm.tqdm(forward_pairs): + # total = Decimal(0) + # for item in forward_pairs[x_word]: + # total += Decimal(forward_pairs[x_word][item]) + forward_probs[x_word] = {} + for item in forward_pairs[x_word]: + forward_probs[x_word][item] = float(Decimal(forward_pairs[x_word][item])/w_freq[x_word]) + + del forward_pairs + + print(" Counting TPB") + # Calculates forward probabilities and saves them as a Decimal(probability) + # The output variabe forward_probs is a dict of dicts, form: {"word_x": {"word_x-1(1)": Decimal(0.73), "word_x-1(2)": Decimal(0.27)}} + backward_probs = {} + for y_word in tqdm.tqdm(backward_pairs): + # total = Decimal(0) + # for item in backward_pairs[y_word]: + # total += Decimal(backward_pairs[y_word][item]) + backward_probs[y_word] = {} + for item in backward_pairs[y_word]: + backward_probs[y_word][item] = float(Decimal(backward_pairs[y_word][item])/w_freq[y_word]) + + del backward_pairs + + backup_out = open("fwd.json", "w+") + backup_out.write(json.dumps(forward_probs)) + backup_out.close() + + backup_out = open("bckw.json", "w+") + backup_out.write(json.dumps(backward_probs)) + backup_out.close() + + print(" MI/MI3") + + # - Mutual information score (MI.NXT for word i given word i-1; doesn't look beyond!) + #log(Bigram_freq/((item1_freq*item2_freq)/WORDCOUNT)) + + mi_score = {} + mi3_score = {} + + log10_2 = Decimal(log10(2)) # Do not calculate log10(2) every time around + if settings_stats.mi == "BNC" or settings_stats.mi == "BYU": + for bigram in tqdm.tqdm(counter): + + try: + item1, item2 = bigram.split() + item1_freq = Decimal(w_freq[item1]) + item2_freq = Decimal(w_freq[item2]) + + ## Used by BNCweb/BYU + denom = item1_freq*item2_freq + score = Decimal(counter[bigram]*Decimal(w_count))/denom + mi_score[bigram] = float(Decimal(log(score,10))/log10_2) + + score3 = Decimal((counter[bigram]**3)*Decimal(w_count))/denom + mi3_score[bigram] = float(Decimal(log(score3,10))/log10_2) + except ValueError: + print("Line 465 ValueError" + bigram) + + else: + try: + ## Based on Wiechmann 2008 + item1, item2 = bigram.split() + item1_freq = Decimal(w_freq[item1]) + item2_freq = Decimal(w_freq[item2]) + + score = Decimal(counter[bigram])/((item1_freq*item2_freq)/Decimal(w_count)) + mi_score[bigram] = float(score.ln()) + except ValueError: + print("Line 477 ValueError: " + bigram) + + backup_out = open("miscore.json", "w+") + backup_out.write(json.dumps(mi_score)) + backup_out.close() + del mi_score + backup_out = open("mi3score.json", "w+") + backup_out.write(json.dumps(mi3_score)) + backup_out.close() + del mi3_score + + print(" z-score") + + # - Z score + + # prob = Wi-1/(w_count-Wi) + # expected = prob * Wi + # z-score = bigram-expected/sqrt(expected*(1-prob)) + + z_score = {} + for bigram in tqdm.tqdm(counter): - print(" MI/MI3") - - # - Mutual information score (MI.NXT for word i given word i-1; doesn't look beyond!) - #log(Bigram_freq/((item1_freq*item2_freq)/WORDCOUNT)) - - mi_score = {} - mi3_score = {} - - log10_2 = Decimal(log10(2)) # Do not calculate log10(2) every time around - if settings.mi == "BNC" or settings.mi == "BYU": + try: + item1, item2 = bigram.split() + item1_freq = Decimal(w_freq[item1]) + item2_freq = Decimal(w_freq[item2]) + + ## Used by BNCweb/BYU + prob = item1_freq/Decimal(w_count-item2_freq) # probability of item1 + expe = prob*item2_freq # expected number of bigrams + numer = Decimal(counter[bigram])-expe # deviation from the expected number + denom = Decimal(sqrt(expe*(Decimal(1)-prob))) # std.deviation (kind of) + z_score[bigram] = float(numer/denom) + except ValueError: + print("Line 508 ValueError: " + bigram) + + backup_out = open("zscore.json", "w+") + backup_out.write(json.dumps(z_score)) + backup_out.close() + del z_score + + print(" t-score") + t_score = {} + + dec_w_count = Decimal(w_count) # Do not express the word count as a Decimal every time for bigram in tqdm.tqdm(counter): - + item1, item2 = bigram.split() - item1_freq = Decimal(w_freq[item1]) - item2_freq = Decimal(w_freq[item2]) - - ## Used by BNCweb/BYU - denom = item1_freq*item2_freq - score = Decimal(counter[bigram]*Decimal(w_count))/denom - mi_score[bigram] = float(Decimal(log(score,10))/log10_2) - - score3 = Decimal((counter[bigram]**3)*Decimal(w_count))/denom - mi3_score[bigram] = float(Decimal(log(score3,10))/log10_2) - - else: - ## Based on Wiechmann 2008 - item1, item2 = bigram.split() - item1_freq = Decimal(w_freq[item1]) - item2_freq = Decimal(w_freq[item2]) - - score = Decimal(counter[bigram])/((item1_freq*item2_freq)/Decimal(w_count)) - mi_score[bigram] = float(score.ln()) - - backup_out = open("miscore.json", "w+") - backup_out.write(json.dumps(mi_score)) - backup_out.close() - del mi_score - backup_out = open("mi3score.json", "w+") - backup_out.write(json.dumps(mi3_score)) - backup_out.close() - del mi3_score - - print(" z-score") - - # - Z score - - # prob = Wi-1/(w_count-Wi) - # expected = prob * Wi - # z-score = bigram-expected/sqrt(expected*(1-prob)) - - z_score = {} - for bigram in tqdm.tqdm(counter): - - item1, item2 = bigram.split() - item1_freq = Decimal(w_freq[item1]) - item2_freq = Decimal(w_freq[item2]) - - ## Used by BNCweb/BYU - prob = item1_freq/Decimal(w_count-item2_freq) # probability of item1 - expe = prob*item2_freq # expected number of bigrams - numer = Decimal(counter[bigram])-expe # deviation from the expected number - denom = Decimal(sqrt(expe*(Decimal(1)-prob))) # std.deviation (kind of) - z_score[bigram] = float(numer/denom) - - backup_out = open("zscore.json", "w+") - backup_out.write(json.dumps(z_score)) - backup_out.close() - del z_score - print(" t-score") - t_score = {} + a = Decimal(counter[bigram]) + b = Decimal(w_freq[item1]) - a + c = Decimal(w_freq[item2]) - a - dec_w_count = Decimal(w_count) # Do not express the word count as a Decimal every time - for bigram in tqdm.tqdm(counter): - - item1, item2 = bigram.split() + expe = ((a+b)*(a+c))/dec_w_count - a = Decimal(counter[bigram]) - b = Decimal(w_freq[item1]) - c = Decimal(w_freq[item2]) - - expe = ((a+b)*(a+c))/dec_w_count - - # Based on Gries - t_score[bigram]= float((a-expe)/Decimal(sqrt(expe))) - - backup_out = open("tscore.json", "w+") - backup_out.write(json.dumps(t_score)) - backup_out.close() - del t_score - - print(" delta-p-score") - delta_p21 = {} - delta_p12 = {} - - dec_w_count = Decimal(w_count) # Do not express the word count as a Decimal every time - for bigram in tqdm.tqdm(counter): - - item1, item2 = bigram.split() + # Based on Gries + t_score[bigram]= float((a-expe)/Decimal(sqrt(expe))) - a = Decimal(counter[bigram]) - b = Decimal(w_freq[item1]) - c = Decimal(w_freq[item2]) - d = dec_w_count - - p1 = Decimal(a)/Decimal(a+b) - p2 = Decimal(c)/Decimal(c+d) - - # Based on Gries - delta_p21[bigram]= float(p1-p2) + backup_out = open("tscore.json", "w+") + backup_out.write(json.dumps(t_score)) + backup_out.close() + del t_score - p1 = Decimal(a)/Decimal(a+c) - p2 = Decimal(b)/Decimal(b+d) - - # Based on Gries - delta_p12[bigram]= float(p1-p2) - - backup_out = open("delta_p21.json", "w+") - backup_out.write(json.dumps(delta_p21)) - backup_out.close() - - backup_out = open("delta_p12.json", "w+") - backup_out.write(json.dumps(delta_p12)) - backup_out.close() - - del delta_p21 - del delta_p12 - - print(" Modified Dice-score") - # modified Dice coefficient; using a,b,c,d just like LL - # mod. dice = log2() - - dice_score = {} - for bigram in tqdm.tqdm(counter): - - item1, item2 = bigram.split() + print(" delta-p-score") + delta_p21 = {} + delta_p12 = {} - a = Decimal(counter[bigram]) - b = Decimal(w_freq[item1]) - c = Decimal(w_freq[item2]) - - score = Decimal(2)*(a*a)/(a+b+a+c) - - dice_score[bigram]= log(float(score),2) - - backup_out = open("dicescore.json", "w+") - backup_out.write(json.dumps(dice_score)) - backup_out.close() - del dice_score - - print(" Log-likelihood") - print(" Preparing LL-score calculation") - - lltemp = open("lltemp.bck", "w+") # We'll use a temp file to save memory - - for bigram in tqdm.tqdm(counter): # Prepare the inputs and save them to a temp file - allow multiprocessing without straining RAM - item1, item2 = bigram.split() - - a = counter[bigram] - b = w_freq[item1] - c = w_freq[item2] - d = w_count-b-c + dec_w_count = Decimal(w_count) # Do not express the word count as a Decimal every time + for bigram in tqdm.tqdm(counter): - lltemp.write(json.dumps([bigram, a, b, c, d]) + "\n") - - lltemp.close() # Write access no longer needed - lltemp = open("lltemp.bck", "r") # Open with read access only - lt = lltemp.readlines() # Could be moved to the imap() call, but then the length would be uncertain - - print(" Counting") - worker = Pool(4) - ll_score = [] - for i in tqdm.tqdm(worker.imap_unordered(llscorer, lt), total=len(lt)): # Counting is done on 4 cores, output saved in ll_score - ll_score.append(i) - - del lt - worker.close() - worker.join() - - print(" Dictionarizing and saving") - ll_score = dict(ll_score) - backup_out = open("llscore.json", "w+") - backup_out.write(json.dumps(ll_score)) - backup_out.close() - del ll_score - - lltemp.close() # We don't need the connection anymore - try: - os.remove("lltemp.bck") - except: - print("Couldn't remove the file lltemp.bck, please do it manually") - print(" G-score") + item1, item2 = bigram.split() + + a = Decimal(counter[bigram]) + b = Decimal(w_freq[item1]) - a + c = Decimal(w_freq[item2]) - a + d = dec_w_count-a-b-c + + p1 = Decimal(a)/Decimal(a+b) + p2 = Decimal(c)/Decimal(c+d) + + # Based on Gries + delta_p21[bigram]= float(p1-p2) + + p1 = Decimal(a)/Decimal(a+c) + p2 = Decimal(b)/Decimal(b+d) + + # Based on Gries + delta_p12[bigram]= float(p1-p2) + + backup_out = open("delta_p21.json", "w+") + backup_out.write(json.dumps(delta_p21)) + backup_out.close() + + backup_out = open("delta_p12.json", "w+") + backup_out.write(json.dumps(delta_p12)) + backup_out.close() + + del delta_p21 + del delta_p12 + + print(" Dice-score") + # Dice coefficient; using a,b,c,d just like LL + # As used in Sketch Engine + # dice = (2*bigram)/(w1_freq + w2_freq) + + dice_score = {} + for bigram in tqdm.tqdm(counter): + + item1, item2 = bigram.split() + + a = Decimal(counter[bigram]) + b = Decimal(w_freq[item1]) - a + c = Decimal(w_freq[item2]) - a + + score = Decimal(2*a)/(a+b+a+c) + + dice_score[bigram]= float(score) + + backup_out = open("dicescore.json", "w+") + backup_out.write(json.dumps(dice_score)) + backup_out.close() + del dice_score + + print(" Modified Dice-score") + # modified Dice coefficient; using a,b,c,d just like LL + # Kitamura, M., and Y. Matsumoto. 1996. Automatic Ex-traction of Word Sequence Correspondences in Par-allel Corpora. In Proc. 4'" Workshop on Very Large Cmpora, 79-87. 4 August, Copenhagen, Denmark. + # mod. dice = log2() + + dice_score = {} + for bigram in tqdm.tqdm(counter): + + item1, item2 = bigram.split() + + a = Decimal(counter[bigram]) + b = Decimal(w_freq[item1]) - a + c = Decimal(w_freq[item2]) - a + + score = Decimal(2)*(a*a)/(a+b+a+c) + + dice_score[bigram]= log(float(score),2) + + backup_out = open("moddicescore.json", "w+") + backup_out.write(json.dumps(dice_score)) + backup_out.close() + del dice_score + + print(" Log-likelihood") + print(" Preparing LL-score calculation") + + lltemp = open("lltemp.bck", "w+") # We'll use a temp file to save memory + + for bigram in tqdm.tqdm(counter): # Prepare the inputs and save them to a temp file - allow multiprocessing without straining RAM + item1, item2 = bigram.split() + + a = counter[bigram] + b = w_freq[item1] - a + c = w_freq[item2] - a + d = w_count-a-b-c + + lltemp.write(json.dumps([bigram, a, b, c, d]) + "\n") + + lltemp.close() # Write access no longer needed + lltemp = open("lltemp.bck", "r") # Open with read access only + lt = lltemp.readlines() # Could be moved to the imap() call, but then the length would be uncertain + + print(" Counting") + worker = Pool(4) + ll_score = [] + for i in tqdm.tqdm(worker.imap_unordered(llscorer, lt), total=len(lt)): # Counting is done on 4 cores, output saved in ll_score + ll_score.append(i) + + del lt + worker.close() + worker.join() + + print(" Dictionarizing and saving") + ll_score = dict(ll_score) + backup_out = open("llscore.json", "w+") + backup_out.write(json.dumps(ll_score)) + backup_out.close() + del ll_score + + lltemp.close() # We don't need the connection anymore + try: + os.remove("lltemp.bck") + except: + print("Couldn't remove the file lltemp.bck, please do it manually") + print(" G-score") # - Lexical gravity G (G.NXT) # log((Fbigr * TypeFreqAfterW1)/Fw1) + log((Fbigr * TypeFreqBeforeW2)/Fw2) # G-score needs the number of types following/preceding an item + backup_out = open("fwd.json", "r") + forward_probs = json.loads(backup_out.read()) + backup_out.close() + + backup_out = open("bckw.json", "r") + backward_probs = json.loads(backup_out.read()) + backup_out.close() + fwd_types = {} for item in forward_probs: fwd_types[item] = len(forward_probs[item]) - + bckw_types = {} for item in backward_probs: bckw_types[item] = len(backward_probs[item]) - + gtemp = open("gscoretemp.bck", "w+") # We'll use a temp file to save memory - + print(" Preparing G-score calculation") + + for bigram in tqdm.tqdm(counter): # The calculation is prepared as a file with JSON-serialized inputs to the llcounter() function item1, item2 = bigram.split() item1_3 = w_freq[item1] @@ -507,16 +686,16 @@ def listener(queue): gtemp.write(json.dumps([bigram, bf, item1_2, item2_2, item1_3, item2_3]) + "\n") gtemp.close() # We don't need the write access anymore - + gtemp = open("gscoretemp.bck", "r") # Open with read-only gt = gtemp.readlines() # Could be moved to the imap() call, but then the length would be uncertain print(" Ready") worker = Pool(4) g_score = [] - + for i in tqdm.tqdm(worker.imap_unordered(gscorer, gt), total=len(gt)): # Counting is done on 4 cores, output saved in g_score g_score.append(i) - + del gt worker.close() worker.join() @@ -524,32 +703,32 @@ def listener(queue): g_score = dict(g_score) backup_out = open("gscore.json", "w+") backup_out.write(json.dumps(g_score)) - backup_out.close() + backup_out.close() del g_score - + gtemp.close() # We don't need the connection anymore try: - os.remove("gscoretemp.bck") + os.remove("gscoretemp.bck") except: print("Couldn't remove the file gscoretemp.bck, please do it manually") - + ########### This is a clumsy way of converting the calculated scores into a pandas DataFrame; future versions should get rid of it # from convert_to_pd import Converter - # worker = Converter() - # worker.convert() - - - ########## This is a clumsy way of calculating dispersion scores; efficient implementation would do that during preprocessing + # worker = Converter() + # worker.convert() + + + ########## This is a clumsy way of calculating dispersion scores; efficient implementation would do that during preprocessing ####(though memory may be limiting there) - + # from dispersion_counter import DispersionCounter # worker = DispersionCounter(path=path_to_coca) # print("Collecting dispersion scores") # for ext in tqdm.tqdm(["acad", "fic", "news", "mag", "spok"]): # worker.collect(ext) # gc.collect() - + # print("Collecting done - preprocessing final data") - # worker.save() - - exit() \ No newline at end of file + # worker.save() + + exit() diff --git a/coca_stats_lemma.py b/coca_stats_lemma.py new file mode 100644 index 0000000..27f3d70 --- /dev/null +++ b/coca_stats_lemma.py @@ -0,0 +1,740 @@ +from os import walk +from bs4 import BeautifulSoup +import json +import psutil +import shutil +import pickle +from decimal import Decimal +import settings_stats +import re +from multiprocessing import Pool, Manager +import collections +import tqdm +from math import log10 +from math import sqrt +from math import log +import os +import sys + +path_to_coca = "/Users/kylamcconnell/Documents/coca_2019_test" +#path_to_coca = "E:/coca_2019_wlp" + +# Helper functions to allow multiprocessing +def gscorer(items): + """Take a string line (JSON serialized list) as input, return bigram + G-score""" + global gscores + bigram, bf, item1_2, item2_2, item1_3, item2_3 = json.loads(items) + + score1 = (Decimal(bf) * Decimal(item1_2))/Decimal(item1_3) + score2 = (Decimal(bf) * Decimal(item2_2))/Decimal(item2_3) + + return([bigram, float(score1.ln() + score2.ln())]) + +# The operations in calculation of Log-likelihood. +# This allows list comprehension to be used instead of searching through the list one item at a time +order = [1, 1, 1, 1, -1, -1, -1, -1, 1] +def llscorer(items): + """Take a string line (JSON serialized list) as input, return bigram + LL-score + + a the frequency of node - collocate pairs + b number of instances where the node does not co-occur with the collocate + c number of instances where the collocate does not co-occur with the node + b the number of words in the corpus minus the number of occurrences of the node and the collocate + + The collocation value is calculated as follows: + + 2*( a*log(a) + b*log(b) + c*log(c) + d*log(d) + - (a+b)*log(a+b) - (a+c)*log(a+c) + - (b+d)*log(b+d) - (c+d)*log(c+d) + + (a+b+c+d)*log(a+b+c+d))""" + + global ll_score + bigram, a,b,c,d = json.loads(items) + + a = Decimal(a) + b = Decimal(b) + c = Decimal(c) + d = Decimal(d) + + base = [a, b, c, d, a+b, a+c, b+d, c+d, a+b+c+d] + + logs = [Decimal(log(x, 10)) if x >0 else Decimal(0) for x in base] + parts = [x*y for x,y in zip(base, logs)] + parts = [x*float(y) for x,y in zip(order, parts)] + + return([bigram, 2*sum(parts)]) + +def preprocess(filename, queue): + with open(filename, "r", errors="replace") as i: + doc = i.read() + doc = re.sub("_.*", "", doc) + doc = re.sub("_", "", doc) + doc = re.sub("\t+", "\t", doc) + doc = doc.split("\n") + doc = [word.split("\t") for word in doc] + + docs = [] + d = [] + declined = re.compile("@_") + for word in doc: + + if len(word) == 4: #changed from 3 to 4 + pos = word[3][0] #take simplified tag (first letter only) + d.append([word[2].lower(), pos]) #changed from 1 20 to 2 31\ + + # elif word[0].startswith("##"): + # d = ["_".join(w).strip() for w in d] + # punctuation = [",", ".", "!", "?", ";", ":"] + # d = [x + " " + y for x,y in zip(d[0:-1], d[1:]) if not (x in punctuation) or (y in punctuation)] + # d = [x for x in d if re.match(declined, x) == None] + # docs.append(d) + # d = [] + + d = ["_".join(w).strip() for w in d if not w[1].endswith("y")] + d = [x + " " + y for x,y in zip(d[0:-1], d[1:]) if re.match("\W", x) == None and re.match("\W", y) == None] + d = [x for x in d if re.match(declined, x) == None] + + docs.append(d) + + docs = [item for sublist in docs for item in sublist] + + queue.put(json.dumps(docs)) + + # except: + # print("Error: " + filename) + +def preprocess_wfreq(filename, queue): + # print(filename) + with open(filename, "r", errors="replace") as i: + try: + doc = i.read() + + #doc = re.sub("_", "", doc) + doc = re.sub("\t+", "\t", doc) + doc = doc.split("\n") + doc = [word.split("\t") for word in doc] + + docs = [] + d = [] + declined = re.compile("@_") + for word in doc: + if len(word) == 4: #changed from 3 to 4 + pos = word[3][0] + d.append([word[2].lower(), pos]) + + # elif word[0].startswith("##"): + # d = ["_".join(w).strip() for w in d] + # d = [x for x in d if not x.endswith("_y")] + # d = [x for x in d if re.match(declined, x) == None] + # docs.append(d) + # d = [] + + else: + pass + + d = ["_".join(w).strip() for w in d] + d = [x for x in d if re.match("\W", x) == None] + d = [x for x in d if re.match(declined, x) == None] + + docs.append(d) + + docs = [item for sublist in docs for item in sublist] + + queue.put(json.dumps(docs)) + + except: + print("Error: " + str(filename)) + + # except UnicodeError as e: + # offending = e.object[e.start:e.end] + # print("This file isn't encoded with", e.encoding) + # print("Illegal bytes:", repr(offending)) + # seen_text = e.object[:e.start] + # line_no = seent_text.count(b'\n') + 1 + # print("Line number: " + line_no) + # raise + +def listener(queue, filename): + f = open(filename, 'w') + while 1: + m = queue.get() + if m == 'kill': + break + f.write(str(m) + '\n') + f.flush() + f.close() + +# Multiprocessing needs this if-statement, otherwise it won't work properly +if __name__ == "__main__": + + ram_present = psutil.virtual_memory()[0] >> 30 + if ram_present < 7: + print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") + sys.exit(0) + + #total, used, free = shutil.disk_usage("\\") + total, used, free = shutil.disk_usage("/") + print("Free drive space: %d/%d GB" % ((free // (2**30)), (total // (2**30)))) + if (free // (2**30)) < 7: + print("WARNING: This is space-intensive operation. It cannot continue if you don't have at least 15 GB of free space on the same drive as the script.\nExiting...") + sys.exit(0) + + print("Initializing...") + files = [] + for dirpath, dirnames, filenames in os.walk(path_to_coca): + files.extend([os.path.join(dirpath, file) for file in filenames]) + + if os.path.isfile("_COCA2_lemma.txt"): + print("A file _COCA2_lemma.txt was found in the script's directory.") + print("This could be the preprocessed corpus. In that case, you can skip the preprocessing.") + print("Otherwise, this script can delete it and preprocess the corpus.") + print("Clean the file _COCA2_lemma.txt?") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + f = open("_COCA2_lemma.txt", 'w+') + f.close() + + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2_lemma.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + else: + pass + + else: + f = open("_COCA2_lemma.txt", 'w+') + f.close() + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2_lemma.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + print(" Bigrams collected") + + files = [] + for dirpath, dirnames, filenames in os.walk(path_to_coca): + files.extend([os.path.join(dirpath, file) for file in filenames]) + + if os.path.isfile("_COCA2wfreq_lemma.txt"): + print("A file _COCA2wfreq_lemma.txt was found in the script's directory.") + print("This could be the preprocessed corpus. In that case, you can skip the preprocessing.") + print("Otherwise, this script can delete it and preprocess the corpus.") + print("Clean the file _COCA2wfreq_lemma.txt?") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + f = open("_COCA2wfreq_lemma.txt", 'w+') + f.close() + + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2wfreq_lemma.txt" )) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess_wfreq, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + else: + pass + + else: + f = open("_COCA2wfreq_lemma.txt", 'w+') + f.close() + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2wfreq_lemma.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess_wfreq, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + print(" Wfreq preprocessing finished") + + print("Preprocessing finished") + print("Counting scores") + + print(" Bigram frequency") + #- Bigram frequency (bi.freq.NXT) + + coca = open("_COCA2_lemma.txt", "r") + counter = collections.Counter() + + for line in tqdm.tqdm(coca): + bgs = json.loads(line) + bgs = [x.strip() for x in bgs if not x.startswith("##")] + counter.update(bgs) + # print("\n\n\n\nEARLY STOPPING") + # break + + coca.close() + print(" Cropping the bigram dict to items with freq > 0") #changed 4 to 0 + counter = {k:v for k,v in counter.items() if v > 0} #changed 4 to 0 + counter = dict(counter) + + # backup bigram stats file + print(" Saving") + + backup_out = open("bigrams_lemma.json", "w+") + backup_out.write(json.dumps(counter)) + backup_out.close() + + print(" Making a wordcount from %i bigrams" % (len(counter))) + w_freq = collections.Counter() + coca = open("_COCA2wfreq_lemma.txt", "r") + for line in tqdm.tqdm(coca): + bgs = json.loads(line) + bgs = [x.strip() for x in bgs if not x.startswith("##")] + w_freq.update(bgs) + + w_freq = dict(w_freq) + coca.close() + + backup_out = open("wfreqs_lemma.json", "w+") + backup_out.write(json.dumps(w_freq)) + backup_out.close() + + + with open("bigrams_lemma.json", "r") as i: + counter = json.loads(i.read()) + with open("wfreqs_lemma.json", "r") as i: + w_freq = json.loads(i.read()) + + w_count = 0 + for item, frq in w_freq.items(): # Sum the word frequency to get the total + w_count += frq + + print("Keeping only nice bigrams") + bad = len(counter) + counter = {k:v for k,v in counter.items() if (k.split()[0] in w_freq) and (k.split()[1] in w_freq)} + print("Kept %i/%i bigrams" % (len(counter), bad)) + + print(" TP-D/TP-B") + + #- Direct transitional probability (TPD.bi.NXT) + #How likely is word x+1 to occur after word x? + #Backwards transitional probability (TPB.bi.NXT) + #How likely is word y-1 to occur before word y? + + forward_pairs = {} + backward_pairs = {} + + print(" Dictionarizing word-pair counts") + #create dict of dicts: how many times is a given word followed by another word and vice versa + + for item, freq in tqdm.tqdm(counter.items()): + x_word = item.split()[0] + y_word = item.split()[1] + + # Check the forward freq dictionary, if word x+1 there, add 1 to freq. + # If word x+1 not there add it with freq = 1. + # If word x not there, create it and add word x+1 as first item with freq = 1. + if x_word in forward_pairs: + if y_word in forward_pairs[x_word]: + forward_pairs[x_word][y_word] += freq + else: + forward_pairs[x_word][y_word] = freq + else: + forward_pairs[x_word] = {y_word : freq} + + # Check the backward freq dictionary, if word y-1 there, add 1 to freq. + # If word y-1 not there add it with freq = 1. + # If word y not there, create it and add word y-1 as first item with freq = 1. + if y_word in backward_pairs: + if x_word in backward_pairs[y_word]: + backward_pairs[y_word][x_word] += freq + else: + backward_pairs[y_word][x_word] = freq + else: + backward_pairs[y_word] = {x_word : freq} + + print(" Pairs collected") + + print(" Counting TPD") + # Calculates forward probabilities and saves them as a Decimal(probability) + # The output variable forward_probs is a dict of dicts, form: {"word_x": {"word_x+1(1)": Decimal(0.73), "word_x+1(2)": Decimal(0.27)}} + forward_probs = {} + for x_word in tqdm.tqdm(forward_pairs): + # total = Decimal(0) + # for item in forward_pairs[x_word]: + # total += Decimal(forward_pairs[x_word][item]) + forward_probs[x_word] = {} + for item in forward_pairs[x_word]: + forward_probs[x_word][item] = float(Decimal(forward_pairs[x_word][item])/w_freq[x_word]) + + del forward_pairs + + print(" Counting TPB") + # Calculates forward probabilities and saves them as a Decimal(probability) + # The output variabe forward_probs is a dict of dicts, form: {"word_x": {"word_x-1(1)": Decimal(0.73), "word_x-1(2)": Decimal(0.27)}} + backward_probs = {} + for y_word in tqdm.tqdm(backward_pairs): + # total = Decimal(0) + # for item in backward_pairs[y_word]: + # total += Decimal(backward_pairs[y_word][item]) + backward_probs[y_word] = {} + for item in backward_pairs[y_word]: + backward_probs[y_word][item] = float(Decimal(backward_pairs[y_word][item])/w_freq[y_word]) + + del backward_pairs + + backup_out = open("fwd_lemma.json", "w+") + backup_out.write(json.dumps(forward_probs)) + backup_out.close() + + backup_out = open("bckw_lemma.json", "w+") + backup_out.write(json.dumps(backward_probs)) + backup_out.close() + + print(" MI/MI3") + + # - Mutual information score (MI.NXT for word i given word i-1; doesn't look beyond!) + #log(Bigram_freq/((item1_freq*item2_freq)/WORDCOUNT)) + + mi_score = {} + mi3_score = {} + + log10_2 = Decimal(log10(2)) # Do not calculate log10(2) every time around + if settings_stats.mi == "BNC" or settings_stats.mi == "BYU": + for bigram in tqdm.tqdm(counter): + + item1, item2 = bigram.split() + item1_freq = Decimal(w_freq[item1]) + item2_freq = Decimal(w_freq[item2]) + + ## Used by BNCweb/BYU + denom = item1_freq*item2_freq + score = Decimal(counter[bigram]*Decimal(w_count))/denom + mi_score[bigram] = float(Decimal(log(score,10))/log10_2) + + score3 = Decimal((counter[bigram]**3)*Decimal(w_count))/denom + mi3_score[bigram] = float(Decimal(log(score3,10))/log10_2) + + else: + ## Based on Wiechmann 2008 + item1, item2 = bigram.split() + item1_freq = Decimal(w_freq[item1]) + item2_freq = Decimal(w_freq[item2]) + + score = Decimal(counter[bigram])/((item1_freq*item2_freq)/Decimal(w_count)) + mi_score[bigram] = float(score.ln()) + + backup_out = open("miscore_lemma.json", "w+") + backup_out.write(json.dumps(mi_score)) + backup_out.close() + del mi_score + backup_out = open("mi3score_lemma.json", "w+") + backup_out.write(json.dumps(mi3_score)) + backup_out.close() + del mi3_score + + print(" z-score") + + # - Z score + + # prob = Wi-1/(w_count-Wi) + # expected = prob * Wi + # z-score = bigram-expected/sqrt(expected*(1-prob)) + + z_score = {} + for bigram in tqdm.tqdm(counter): + + item1, item2 = bigram.split() + item1_freq = Decimal(w_freq[item1]) + item2_freq = Decimal(w_freq[item2]) + + ## Used by BNCweb/BYU + prob = item1_freq/Decimal(w_count-item2_freq) # probability of item1 + expe = prob*item2_freq # expected number of bigrams + numer = Decimal(counter[bigram])-expe # deviation from the expected number + denom = Decimal(sqrt(expe*(Decimal(1)-prob))) # std.deviation (kind of) + z_score[bigram] = float(numer/denom) + + backup_out = open("zscore_lemma.json", "w+") + backup_out.write(json.dumps(z_score)) + backup_out.close() + del z_score + + print(" t-score") + t_score = {} + + dec_w_count = Decimal(w_count) # Do not express the word count as a Decimal every time + for bigram in tqdm.tqdm(counter): + + item1, item2 = bigram.split() + + a = Decimal(counter[bigram]) + b = Decimal(w_freq[item1]) - a + c = Decimal(w_freq[item2]) - a + + expe = ((a+b)*(a+c))/dec_w_count + + # Based on Gries + t_score[bigram]= float((a-expe)/Decimal(sqrt(expe))) + + backup_out = open("tscore_lemma.json", "w+") + backup_out.write(json.dumps(t_score)) + backup_out.close() + del t_score + + print(" delta-p-score") + delta_p21 = {} + delta_p12 = {} + + dec_w_count = Decimal(w_count) # Do not express the word count as a Decimal every time + for bigram in tqdm.tqdm(counter): + + item1, item2 = bigram.split() + + a = Decimal(counter[bigram]) + b = Decimal(w_freq[item1]) - a + c = Decimal(w_freq[item2]) - a + d = dec_w_count-a-b-c + + p1 = Decimal(a)/Decimal(a+b) + p2 = Decimal(c)/Decimal(c+d) + + # Based on Gries + delta_p21[bigram]= float(p1-p2) + + p1 = Decimal(a)/Decimal(a+c) + p2 = Decimal(b)/Decimal(b+d) + + # Based on Gries + delta_p12[bigram]= float(p1-p2) + + backup_out = open("delta_p21_lemma.json", "w+") + backup_out.write(json.dumps(delta_p21)) + backup_out.close() + + backup_out = open("delta_p12_lemma.json", "w+") + backup_out.write(json.dumps(delta_p12)) + backup_out.close() + + del delta_p21 + del delta_p12 + + print(" Dice-score") + # Dice coefficient; using a,b,c,d just like LL + # As used in Sketch Engine + # dice = (2*bigram)/(w1_freq + w2_freq) + + dice_score = {} + for bigram in tqdm.tqdm(counter): + + item1, item2 = bigram.split() + + a = Decimal(counter[bigram]) + b = Decimal(w_freq[item1]) - a + c = Decimal(w_freq[item2]) - a + + score = Decimal(2*a)/(a+b+a+c) + + dice_score[bigram]= float(score) + + backup_out = open("dicescore_lemma.json", "w+") + backup_out.write(json.dumps(dice_score)) + backup_out.close() + del dice_score + + print(" Modified Dice-score") + # modified Dice coefficient; using a,b,c,d just like LL + # Kitamura, M., and Y. Matsumoto. 1996. Automatic Ex-traction of Word Sequence Correspondences in Par-allel Corpora. In Proc. 4'" Workshop on Very Large Cmpora, 79-87. 4 August, Copenhagen, Denmark. + # mod. dice = log2() + + dice_score = {} + for bigram in tqdm.tqdm(counter): + + item1, item2 = bigram.split() + + a = Decimal(counter[bigram]) + b = Decimal(w_freq[item1]) - a + c = Decimal(w_freq[item2]) - a + + score = Decimal(2)*(a*a)/(a+b+a+c) + + dice_score[bigram]= log(float(score),2) + + backup_out = open("moddicescore_lemma.json", "w+") + backup_out.write(json.dumps(dice_score)) + backup_out.close() + del dice_score + + print(" Log-likelihood") + print(" Preparing LL-score calculation") + + lltemp = open("lltemp.bck", "w+") # We'll use a temp file to save memory + + for bigram in tqdm.tqdm(counter): # Prepare the inputs and save them to a temp file - allow multiprocessing without straining RAM + item1, item2 = bigram.split() + + a = counter[bigram] + b = w_freq[item1] - a + c = w_freq[item2] - a + d = w_count-a-b-c + + lltemp.write(json.dumps([bigram, a, b, c, d]) + "\n") + + lltemp.close() # Write access no longer needed + lltemp = open("lltemp.bck", "r") # Open with read access only + lt = lltemp.readlines() # Could be moved to the imap() call, but then the length would be uncertain + + print(" Counting") + worker = Pool(4) + ll_score = [] + for i in tqdm.tqdm(worker.imap_unordered(llscorer, lt), total=len(lt)): # Counting is done on 4 cores, output saved in ll_score + ll_score.append(i) + + del lt + worker.close() + worker.join() + + print(" Dictionarizing and saving") + ll_score = dict(ll_score) + backup_out = open("llscore_lemma.json", "w+") + backup_out.write(json.dumps(ll_score)) + backup_out.close() + del ll_score + + lltemp.close() # We don't need the connection anymore + try: + os.remove("lltemp.bck") + except: + print("Couldn't remove the file lltemp.bck, please do it manually") + print(" G-score") + + # - Lexical gravity G (G.NXT) + + # log((Fbigr * TypeFreqAfterW1)/Fw1) + log((Fbigr * TypeFreqBeforeW2)/Fw2) + + # G-score needs the number of types following/preceding an item + fwd_types = {} + for item in forward_probs: + fwd_types[item] = len(forward_probs[item]) + + bckw_types = {} + for item in backward_probs: + bckw_types[item] = len(backward_probs[item]) + + gtemp = open("gscoretemp.bck", "w+") # We'll use a temp file to save memory + + print(" Preparing G-score calculation") + for bigram in tqdm.tqdm(counter): # The calculation is prepared as a file with JSON-serialized inputs to the llcounter() function + item1, item2 = bigram.split() + item1_3 = w_freq[item1] + item2_3 = w_freq[item2] + item1_2 = fwd_types[item1] + item2_2 = bckw_types[item2] + bf = counter[bigram] + gtemp.write(json.dumps([bigram, bf, item1_2, item2_2, item1_3, item2_3]) + "\n") + + gtemp.close() # We don't need the write access anymore + + gtemp = open("gscoretemp.bck", "r") # Open with read-only + gt = gtemp.readlines() # Could be moved to the imap() call, but then the length would be uncertain + print(" Ready") + worker = Pool(4) + g_score = [] + + for i in tqdm.tqdm(worker.imap_unordered(gscorer, gt), total=len(gt)): # Counting is done on 4 cores, output saved in g_score + g_score.append(i) + + del gt + worker.close() + worker.join() + print(" Dictionarizing and saving") + g_score = dict(g_score) + backup_out = open("gscore_lemma.json", "w+") + backup_out.write(json.dumps(g_score)) + backup_out.close() + del g_score + + gtemp.close() # We don't need the connection anymore + try: + os.remove("gscoretemp.bck") + except: + print("Couldn't remove the file gscoretemp.bck, please do it manually") + + ########### This is a clumsy way of converting the calculated scores into a pandas DataFrame; future versions should get rid of it + # from convert_to_pd import Converter + # worker = Converter() + # worker.convert() + + + ########## This is a clumsy way of calculating dispersion scores; efficient implementation would do that during preprocessing + ####(though memory may be limiting there) + + # from dispersion_counter import DispersionCounter + # worker = DispersionCounter(path=path_to_coca) + # print("Collecting dispersion scores") + # for ext in tqdm.tqdm(["acad", "fic", "news", "mag", "spok"]): + # worker.collect(ext) + # gc.collect() + + # print("Collecting done - preprocessing final data") + # worker.save() + + exit() diff --git a/coca_stats_lemma_simple.py b/coca_stats_lemma_simple.py new file mode 100644 index 0000000..dd21a63 --- /dev/null +++ b/coca_stats_lemma_simple.py @@ -0,0 +1,740 @@ +from os import walk +from bs4 import BeautifulSoup +import json +import psutil +import shutil +import pickle +from decimal import Decimal +import settings_stats +import re +from multiprocessing import Pool, Manager +import collections +import tqdm +from math import log10 +from math import sqrt +from math import log +import os +import sys + +path_to_coca = "/Users/kylamcconnell/Documents/coca_2019_test" +#path_to_coca = "E:/coca_2019_wlp" + +# Helper functions to allow multiprocessing +def gscorer(items): + """Take a string line (JSON serialized list) as input, return bigram + G-score""" + global gscores + bigram, bf, item1_2, item2_2, item1_3, item2_3 = json.loads(items) + + score1 = (Decimal(bf) * Decimal(item1_2))/Decimal(item1_3) + score2 = (Decimal(bf) * Decimal(item2_2))/Decimal(item2_3) + + return([bigram, float(score1.ln() + score2.ln())]) + +# The operations in calculation of Log-likelihood. +# This allows list comprehension to be used instead of searching through the list one item at a time +order = [1, 1, 1, 1, -1, -1, -1, -1, 1] +def llscorer(items): + """Take a string line (JSON serialized list) as input, return bigram + LL-score + + a the frequency of node - collocate pairs + b number of instances where the node does not co-occur with the collocate + c number of instances where the collocate does not co-occur with the node + b the number of words in the corpus minus the number of occurrences of the node and the collocate + + The collocation value is calculated as follows: + + 2*( a*log(a) + b*log(b) + c*log(c) + d*log(d) + - (a+b)*log(a+b) - (a+c)*log(a+c) + - (b+d)*log(b+d) - (c+d)*log(c+d) + + (a+b+c+d)*log(a+b+c+d))""" + + global ll_score + bigram, a,b,c,d = json.loads(items) + + a = Decimal(a) + b = Decimal(b) + c = Decimal(c) + d = Decimal(d) + + base = [a, b, c, d, a+b, a+c, b+d, c+d, a+b+c+d] + + logs = [Decimal(log(x, 10)) if x >0 else Decimal(0) for x in base] + parts = [x*y for x,y in zip(base, logs)] + parts = [x*float(y) for x,y in zip(order, parts)] + + return([bigram, 2*sum(parts)]) + +def preprocess(filename, queue): + with open(filename, "r", errors="replace") as i: + doc = i.read() + doc = re.sub("_.*", "", doc) + doc = re.sub("_", "", doc) + doc = re.sub("\t+", "\t", doc) + doc = doc.split("\n") + doc = [word.split("\t") for word in doc] + + docs = [] + d = [] + declined = re.compile("@_") + for word in doc: + + if len(word) == 4: #changed from 3 to 4 + pos = word[3][0] #take simplified tag (first letter only) + d.append([word[2].lower(), pos]) #changed from 1 20 to 2 31\ + + # elif word[0].startswith("##"): + # d = ["_".join(w).strip() for w in d] + # punctuation = [",", ".", "!", "?", ";", ":"] + # d = [x + " " + y for x,y in zip(d[0:-1], d[1:]) if not (x in punctuation) or (y in punctuation)] + # d = [x for x in d if re.match(declined, x) == None] + # docs.append(d) + # d = [] + + d = ["_".join(w).strip() for w in d if not w[1].endswith("y")] + d = [x + " " + y for x,y in zip(d[0:-1], d[1:]) if re.match("\W", x) == None and re.match("\W", y) == None] + d = [x for x in d if re.match(declined, x) == None] + + docs.append(d) + + docs = [item for sublist in docs for item in sublist] + + queue.put(json.dumps(docs)) + + # except: + # print("Error: " + filename) + +def preprocess_wfreq(filename, queue): + # print(filename) + with open(filename, "r", errors="replace") as i: + try: + doc = i.read() + + #doc = re.sub("_", "", doc) + doc = re.sub("\t+", "\t", doc) + doc = doc.split("\n") + doc = [word.split("\t") for word in doc] + + docs = [] + d = [] + declined = re.compile("@_") + for word in doc: + if len(word) == 4: #changed from 3 to 4 + pos = word[3][0] + d.append([word[2].lower(), pos]) + + # elif word[0].startswith("##"): + # d = ["_".join(w).strip() for w in d] + # d = [x for x in d if not x.endswith("_y")] + # d = [x for x in d if re.match(declined, x) == None] + # docs.append(d) + # d = [] + + else: + pass + + d = ["_".join(w).strip() for w in d] + d = [x for x in d if re.match("\W", x) == None] + d = [x for x in d if re.match(declined, x) == None] + + docs.append(d) + + docs = [item for sublist in docs for item in sublist] + + queue.put(json.dumps(docs)) + + except: + print("Error: " + str(filename)) + + # except UnicodeError as e: + # offending = e.object[e.start:e.end] + # print("This file isn't encoded with", e.encoding) + # print("Illegal bytes:", repr(offending)) + # seen_text = e.object[:e.start] + # line_no = seent_text.count(b'\n') + 1 + # print("Line number: " + line_no) + # raise + +def listener(queue, filename): + f = open(filename, 'w') + while 1: + m = queue.get() + if m == 'kill': + break + f.write(str(m) + '\n') + f.flush() + f.close() + +# Multiprocessing needs this if-statement, otherwise it won't work properly +if __name__ == "__main__": + + ram_present = psutil.virtual_memory()[0] >> 30 + if ram_present < 7: + print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") + sys.exit(0) + + #total, used, free = shutil.disk_usage("\\") + total, used, free = shutil.disk_usage("/") + print("Free drive space: %d/%d GB" % ((free // (2**30)), (total // (2**30)))) + if (free // (2**30)) < 7: + print("WARNING: This is space-intensive operation. It cannot continue if you don't have at least 15 GB of free space on the same drive as the script.\nExiting...") + sys.exit(0) + + print("Initializing...") + files = [] + for dirpath, dirnames, filenames in os.walk(path_to_coca): + files.extend([os.path.join(dirpath, file) for file in filenames]) + + if os.path.isfile("_COCA2_lemma.txt"): + print("A file _COCA2_lemma.txt was found in the script's directory.") + print("This could be the preprocessed corpus. In that case, you can skip the preprocessing.") + print("Otherwise, this script can delete it and preprocess the corpus.") + print("Clean the file _COCA2_lemma.txt?") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + f = open("_COCA2_lemma.txt", 'w+') + f.close() + + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2_lemma.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + else: + pass + + else: + f = open("_COCA2_lemma.txt", 'w+') + f.close() + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2_lemma.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + print(" Bigrams collected") + + files = [] + for dirpath, dirnames, filenames in os.walk(path_to_coca): + files.extend([os.path.join(dirpath, file) for file in filenames]) + + if os.path.isfile("_COCA2wfreq_lemma.txt"): + print("A file _COCA2wfreq_lemma.txt was found in the script's directory.") + print("This could be the preprocessed corpus. In that case, you can skip the preprocessing.") + print("Otherwise, this script can delete it and preprocess the corpus.") + print("Clean the file _COCA2wfreq_lemma.txt?") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + f = open("_COCA2wfreq_lemma.txt", 'w+') + f.close() + + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2wfreq_lemma.txt" )) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess_wfreq, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + else: + pass + + else: + f = open("_COCA2wfreq_lemma.txt", 'w+') + f.close() + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2wfreq_lemma.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess_wfreq, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + print(" Wfreq preprocessing finished") + + print("Preprocessing finished") + print("Counting scores") + + print(" Bigram frequency") + #- Bigram frequency (bi.freq.NXT) + + coca = open("_COCA2_lemma.txt", "r") + counter = collections.Counter() + + for line in tqdm.tqdm(coca): + bgs = json.loads(line) + bgs = [x.strip() for x in bgs if not x.startswith("##")] + counter.update(bgs) + # print("\n\n\n\nEARLY STOPPING") + # break + + coca.close() + print(" Cropping the bigram dict to items with freq > 0") #changed 4 to 0 + counter = {k:v for k,v in counter.items() if v > 0} #changed 4 to 0 + counter = dict(counter) + + # backup bigram stats file + print(" Saving") + + backup_out = open("bigrams_lemma.json", "w+") + backup_out.write(json.dumps(counter)) + backup_out.close() + + print(" Making a wordcount from %i bigrams" % (len(counter))) + w_freq = collections.Counter() + coca = open("_COCA2wfreq_lemma.txt", "r") + for line in tqdm.tqdm(coca): + bgs = json.loads(line) + bgs = [x.strip() for x in bgs if not x.startswith("##")] + w_freq.update(bgs) + + w_freq = dict(w_freq) + coca.close() + + backup_out = open("wfreqs_lemma.json", "w+") + backup_out.write(json.dumps(w_freq)) + backup_out.close() + + + with open("bigrams_lemma.json", "r") as i: + counter = json.loads(i.read()) + with open("wfreqs_lemma.json", "r") as i: + w_freq = json.loads(i.read()) + + w_count = 0 + for item, frq in w_freq.items(): # Sum the word frequency to get the total + w_count += frq + + print("Keeping only nice bigrams") + bad = len(counter) + counter = {k:v for k,v in counter.items() if (k.split()[0] in w_freq) and (k.split()[1] in w_freq)} + print("Kept %i/%i bigrams" % (len(counter), bad)) + + print(" TP-D/TP-B") + + #- Direct transitional probability (TPD.bi.NXT) + #How likely is word x+1 to occur after word x? + #Backwards transitional probability (TPB.bi.NXT) + #How likely is word y-1 to occur before word y? + + forward_pairs = {} + backward_pairs = {} + + print(" Dictionarizing word-pair counts") + #create dict of dicts: how many times is a given word followed by another word and vice versa + + for item, freq in tqdm.tqdm(counter.items()): + x_word = item.split()[0] + y_word = item.split()[1] + + # Check the forward freq dictionary, if word x+1 there, add 1 to freq. + # If word x+1 not there add it with freq = 1. + # If word x not there, create it and add word x+1 as first item with freq = 1. + if x_word in forward_pairs: + if y_word in forward_pairs[x_word]: + forward_pairs[x_word][y_word] += freq + else: + forward_pairs[x_word][y_word] = freq + else: + forward_pairs[x_word] = {y_word : freq} + + # Check the backward freq dictionary, if word y-1 there, add 1 to freq. + # If word y-1 not there add it with freq = 1. + # If word y not there, create it and add word y-1 as first item with freq = 1. + if y_word in backward_pairs: + if x_word in backward_pairs[y_word]: + backward_pairs[y_word][x_word] += freq + else: + backward_pairs[y_word][x_word] = freq + else: + backward_pairs[y_word] = {x_word : freq} + + print(" Pairs collected") + + print(" Counting TPD") + # Calculates forward probabilities and saves them as a Decimal(probability) + # The output variable forward_probs is a dict of dicts, form: {"word_x": {"word_x+1(1)": Decimal(0.73), "word_x+1(2)": Decimal(0.27)}} + forward_probs = {} + for x_word in tqdm.tqdm(forward_pairs): + # total = Decimal(0) + # for item in forward_pairs[x_word]: + # total += Decimal(forward_pairs[x_word][item]) + forward_probs[x_word] = {} + for item in forward_pairs[x_word]: + forward_probs[x_word][item] = float(Decimal(forward_pairs[x_word][item])/w_freq[x_word]) + + del forward_pairs + + print(" Counting TPB") + # Calculates forward probabilities and saves them as a Decimal(probability) + # The output variabe forward_probs is a dict of dicts, form: {"word_x": {"word_x-1(1)": Decimal(0.73), "word_x-1(2)": Decimal(0.27)}} + backward_probs = {} + for y_word in tqdm.tqdm(backward_pairs): + # total = Decimal(0) + # for item in backward_pairs[y_word]: + # total += Decimal(backward_pairs[y_word][item]) + backward_probs[y_word] = {} + for item in backward_pairs[y_word]: + backward_probs[y_word][item] = float(Decimal(backward_pairs[y_word][item])/w_freq[y_word]) + + del backward_pairs + + backup_out = open("fwd_lemma.json", "w+") + backup_out.write(json.dumps(forward_probs)) + backup_out.close() + + backup_out = open("bckw_lemma.json", "w+") + backup_out.write(json.dumps(backward_probs)) + backup_out.close() + + # print(" MI/MI3") + + # # - Mutual information score (MI.NXT for word i given word i-1; doesn't look beyond!) + # #log(Bigram_freq/((item1_freq*item2_freq)/WORDCOUNT)) + + # mi_score = {} + # mi3_score = {} + + # log10_2 = Decimal(log10(2)) # Do not calculate log10(2) every time around + # if settings_stats.mi == "BNC" or settings_stats.mi == "BYU": + # for bigram in tqdm.tqdm(counter): + + # item1, item2 = bigram.split() + # item1_freq = Decimal(w_freq[item1]) + # item2_freq = Decimal(w_freq[item2]) + + # ## Used by BNCweb/BYU + # denom = item1_freq*item2_freq + # score = Decimal(counter[bigram]*Decimal(w_count))/denom + # mi_score[bigram] = float(Decimal(log(score,10))/log10_2) + + # score3 = Decimal((counter[bigram]**3)*Decimal(w_count))/denom + # mi3_score[bigram] = float(Decimal(log(score3,10))/log10_2) + + # else: + # ## Based on Wiechmann 2008 + # item1, item2 = bigram.split() + # item1_freq = Decimal(w_freq[item1]) + # item2_freq = Decimal(w_freq[item2]) + + # score = Decimal(counter[bigram])/((item1_freq*item2_freq)/Decimal(w_count)) + # mi_score[bigram] = float(score.ln()) + + # backup_out = open("miscore_lemma.json", "w+") + # backup_out.write(json.dumps(mi_score)) + # backup_out.close() + # del mi_score + # backup_out = open("mi3score_lemma.json", "w+") + # backup_out.write(json.dumps(mi3_score)) + # backup_out.close() + # del mi3_score + + # print(" z-score") + + # # - Z score + + # # prob = Wi-1/(w_count-Wi) + # # expected = prob * Wi + # # z-score = bigram-expected/sqrt(expected*(1-prob)) + + # z_score = {} + # for bigram in tqdm.tqdm(counter): + + # item1, item2 = bigram.split() + # item1_freq = Decimal(w_freq[item1]) + # item2_freq = Decimal(w_freq[item2]) + + # ## Used by BNCweb/BYU + # prob = item1_freq/Decimal(w_count-item2_freq) # probability of item1 + # expe = prob*item2_freq # expected number of bigrams + # numer = Decimal(counter[bigram])-expe # deviation from the expected number + # denom = Decimal(sqrt(expe*(Decimal(1)-prob))) # std.deviation (kind of) + # z_score[bigram] = float(numer/denom) + + # backup_out = open("zscore_lemma.json", "w+") + # backup_out.write(json.dumps(z_score)) + # backup_out.close() + # del z_score + + # print(" t-score") + # t_score = {} + + # dec_w_count = Decimal(w_count) # Do not express the word count as a Decimal every time + # for bigram in tqdm.tqdm(counter): + + # item1, item2 = bigram.split() + + # a = Decimal(counter[bigram]) + # b = Decimal(w_freq[item1]) - a + # c = Decimal(w_freq[item2]) - a + + # expe = ((a+b)*(a+c))/dec_w_count + + # # Based on Gries + # t_score[bigram]= float((a-expe)/Decimal(sqrt(expe))) + + # backup_out = open("tscore_lemma.json", "w+") + # backup_out.write(json.dumps(t_score)) + # backup_out.close() + # del t_score + + # print(" delta-p-score") + # delta_p21 = {} + # delta_p12 = {} + + # dec_w_count = Decimal(w_count) # Do not express the word count as a Decimal every time + # for bigram in tqdm.tqdm(counter): + + # item1, item2 = bigram.split() + + # a = Decimal(counter[bigram]) + # b = Decimal(w_freq[item1]) - a + # c = Decimal(w_freq[item2]) - a + # d = dec_w_count-a-b-c + + # p1 = Decimal(a)/Decimal(a+b) + # p2 = Decimal(c)/Decimal(c+d) + + # # Based on Gries + # delta_p21[bigram]= float(p1-p2) + + # p1 = Decimal(a)/Decimal(a+c) + # p2 = Decimal(b)/Decimal(b+d) + + # # Based on Gries + # delta_p12[bigram]= float(p1-p2) + + # backup_out = open("delta_p21_lemma.json", "w+") + # backup_out.write(json.dumps(delta_p21)) + # backup_out.close() + + # backup_out = open("delta_p12_lemma.json", "w+") + # backup_out.write(json.dumps(delta_p12)) + # backup_out.close() + + # del delta_p21 + # del delta_p12 + + # print(" Dice-score") + # # Dice coefficient; using a,b,c,d just like LL + # # As used in Sketch Engine + # # dice = (2*bigram)/(w1_freq + w2_freq) + + # dice_score = {} + # for bigram in tqdm.tqdm(counter): + + # item1, item2 = bigram.split() + + # a = Decimal(counter[bigram]) + # b = Decimal(w_freq[item1]) - a + # c = Decimal(w_freq[item2]) - a + + # score = Decimal(2*a)/(a+b+a+c) + + # dice_score[bigram]= float(score) + + # backup_out = open("dicescore_lemma.json", "w+") + # backup_out.write(json.dumps(dice_score)) + # backup_out.close() + # del dice_score + + # print(" Modified Dice-score") + # # modified Dice coefficient; using a,b,c,d just like LL + # # Kitamura, M., and Y. Matsumoto. 1996. Automatic Ex-traction of Word Sequence Correspondences in Par-allel Corpora. In Proc. 4'" Workshop on Very Large Cmpora, 79-87. 4 August, Copenhagen, Denmark. + # # mod. dice = log2() + + # dice_score = {} + # for bigram in tqdm.tqdm(counter): + + # item1, item2 = bigram.split() + + # a = Decimal(counter[bigram]) + # b = Decimal(w_freq[item1]) - a + # c = Decimal(w_freq[item2]) - a + + # score = Decimal(2)*(a*a)/(a+b+a+c) + + # dice_score[bigram]= log(float(score),2) + + # backup_out = open("moddicescore_lemma.json", "w+") + # backup_out.write(json.dumps(dice_score)) + # backup_out.close() + # del dice_score + + # print(" Log-likelihood") + # print(" Preparing LL-score calculation") + + # lltemp = open("lltemp.bck", "w+") # We'll use a temp file to save memory + + # for bigram in tqdm.tqdm(counter): # Prepare the inputs and save them to a temp file - allow multiprocessing without straining RAM + # item1, item2 = bigram.split() + + # a = counter[bigram] + # b = w_freq[item1] - a + # c = w_freq[item2] - a + # d = w_count-a-b-c + + # lltemp.write(json.dumps([bigram, a, b, c, d]) + "\n") + + # lltemp.close() # Write access no longer needed + # lltemp = open("lltemp.bck", "r") # Open with read access only + # lt = lltemp.readlines() # Could be moved to the imap() call, but then the length would be uncertain + + # print(" Counting") + # worker = Pool(4) + # ll_score = [] + # for i in tqdm.tqdm(worker.imap_unordered(llscorer, lt), total=len(lt)): # Counting is done on 4 cores, output saved in ll_score + # ll_score.append(i) + + # del lt + # worker.close() + # worker.join() + + # print(" Dictionarizing and saving") + # ll_score = dict(ll_score) + # backup_out = open("llscore_lemma.json", "w+") + # backup_out.write(json.dumps(ll_score)) + # backup_out.close() + # del ll_score + + # lltemp.close() # We don't need the connection anymore + # try: + # os.remove("lltemp.bck") + # except: + # print("Couldn't remove the file lltemp.bck, please do it manually") + # print(" G-score") + + # # - Lexical gravity G (G.NXT) + + # # log((Fbigr * TypeFreqAfterW1)/Fw1) + log((Fbigr * TypeFreqBeforeW2)/Fw2) + + # # G-score needs the number of types following/preceding an item + # fwd_types = {} + # for item in forward_probs: + # fwd_types[item] = len(forward_probs[item]) + + # bckw_types = {} + # for item in backward_probs: + # bckw_types[item] = len(backward_probs[item]) + + # gtemp = open("gscoretemp.bck", "w+") # We'll use a temp file to save memory + + # print(" Preparing G-score calculation") + # for bigram in tqdm.tqdm(counter): # The calculation is prepared as a file with JSON-serialized inputs to the llcounter() function + # item1, item2 = bigram.split() + # item1_3 = w_freq[item1] + # item2_3 = w_freq[item2] + # item1_2 = fwd_types[item1] + # item2_2 = bckw_types[item2] + # bf = counter[bigram] + # gtemp.write(json.dumps([bigram, bf, item1_2, item2_2, item1_3, item2_3]) + "\n") + + # gtemp.close() # We don't need the write access anymore + + # gtemp = open("gscoretemp.bck", "r") # Open with read-only + # gt = gtemp.readlines() # Could be moved to the imap() call, but then the length would be uncertain + # print(" Ready") + # worker = Pool(4) + # g_score = [] + + # for i in tqdm.tqdm(worker.imap_unordered(gscorer, gt), total=len(gt)): # Counting is done on 4 cores, output saved in g_score + # g_score.append(i) + + # del gt + # worker.close() + # worker.join() + # print(" Dictionarizing and saving") + # g_score = dict(g_score) + # backup_out = open("gscore_lemma.json", "w+") + # backup_out.write(json.dumps(g_score)) + # backup_out.close() + # del g_score + + # gtemp.close() # We don't need the connection anymore + # try: + # os.remove("gscoretemp.bck") + # except: + # print("Couldn't remove the file gscoretemp.bck, please do it manually") + + ########### This is a clumsy way of converting the calculated scores into a pandas DataFrame; future versions should get rid of it + # from convert_to_pd import Converter + # worker = Converter() + # worker.convert() + + + ########## This is a clumsy way of calculating dispersion scores; efficient implementation would do that during preprocessing + ####(though memory may be limiting there) + + # from dispersion_counter import DispersionCounter + # worker = DispersionCounter(path=path_to_coca) + # print("Collecting dispersion scores") + # for ext in tqdm.tqdm(["acad", "fic", "news", "mag", "spok"]): + # worker.collect(ext) + # gc.collect() + + # print("Collecting done - preprocessing final data") + # worker.save() + + exit() diff --git a/coca_stats_simple.py b/coca_stats_simple.py new file mode 100644 index 0000000..8d4956d --- /dev/null +++ b/coca_stats_simple.py @@ -0,0 +1,734 @@ +from os import walk +from bs4 import BeautifulSoup +import json +import psutil +import shutil +import pickle +from decimal import Decimal +import settings_stats +import re +from multiprocessing import Pool, Manager +import collections +import tqdm +from math import log10 +from math import sqrt +from math import log +import os +import sys + +path_to_coca = "/Users/kylamcconnell/Documents/coca_2019_test" +#path_to_coca = "E:/coca_2019_wlp" + +# Helper functions to allow multiprocessing +def gscorer(items): + """Take a string line (JSON serialized list) as input, return bigram + G-score""" + global gscores + bigram, bf, item1_2, item2_2, item1_3, item2_3 = json.loads(items) + + score1 = (Decimal(bf) * Decimal(item1_2))/Decimal(item1_3) + score2 = (Decimal(bf) * Decimal(item2_2))/Decimal(item2_3) + + return([bigram, float(score1.ln() + score2.ln())]) + +# The operations in calculation of Log-likelihood. +# This allows list comprehension to be used instead of searching through the list one item at a time +order = [1, 1, 1, 1, -1, -1, -1, -1, 1] +def llscorer(items): + """Take a string line (JSON serialized list) as input, return bigram + LL-score + + a the frequency of node - collocate pairs + b number of instances where the node does not co-occur with the collocate + c number of instances where the collocate does not co-occur with the node + b the number of words in the corpus minus the number of occurrences of the node and the collocate + + The collocation value is calculated as follows: + + 2*( a*log(a) + b*log(b) + c*log(c) + d*log(d) + - (a+b)*log(a+b) - (a+c)*log(a+c) + - (b+d)*log(b+d) - (c+d)*log(c+d) + + (a+b+c+d)*log(a+b+c+d))""" + + global ll_score + bigram, a,b,c,d = json.loads(items) + + a = Decimal(a) + b = Decimal(b) + c = Decimal(c) + d = Decimal(d) + + base = [a, b, c, d, a+b, a+c, b+d, c+d, a+b+c+d] + + logs = [Decimal(log(x, 10)) if x > 0 else Decimal(0) for x in base] + parts = [x*y for x,y in zip(base, logs)] + parts = [x*float(y) for x,y in zip(order, parts)] + + return([bigram, 2*sum(parts)]) + +def preprocess(filename, queue): + with open(filename, "r", errors="replace") as i: + doc = i.read() + + #doc = re.sub("_.*", "", doc) + #doc = re.sub("\t+", "\t", doc) + doc = doc.split("\n") + doc = [word.split("\t") for word in doc] + + docs = [] + d = [] + declined = re.compile("@_") + for word in doc: + if len(word) == 4: #rows of length 3 were those that had been removed for copyright reasons + pos = re.sub("_.*", "", word[-1]) + d.append([word[1].lower(), pos]) #changed from 0 2 to 1 3 + + d = ["_".join(w).strip() for w in d] + d = [x + " " + y for x,y in zip(d[0:-1], d[1:]) if re.match("\W", x) == None and re.match("\W", y) == None] + d = [x for x in d if re.match(declined, x) == None] + + docs.append(d) + + docs = [item for sublist in docs for item in sublist] + #print(docs) + + queue.put(json.dumps(docs)) + + +def preprocess_wfreq(filename, queue): + # print(filename) + with open(filename, "r", errors="replace") as i: + doc = i.read() + + #doc = re.sub("_.*", "", doc) + #doc = re.sub("_", "", doc) + #doc = re.sub("\t+", "\t", doc) + doc = doc.split("\n") + doc = [word.split("\t") for word in doc] + + docs = [] + d = [] + declined = re.compile("@_") + for word in doc: + if len(word) == 4: + pos = re.sub("_.*", "", word[-1]) + d.append([word[1].lower(), pos]) #changed from 0 2 to 1 3 + + d = ["_".join(w).strip() for w in d] + d = [x for x in d if re.match("\W", x) == None] + d = [x for x in d if re.match(declined, x) == None] + + docs.append(d) + + docs = [item for sublist in docs for item in sublist] + + queue.put(json.dumps(docs)) + + +def listener(queue, filename): + f = open(filename, 'w') + while 1: + m = queue.get() + if m == 'kill': + break + f.write(str(m) + '\n') + f.flush() + f.close() + +# Multiprocessing needs this if-statement, otherwise it won't work properly +if __name__ == "__main__": + + ram_present = psutil.virtual_memory()[0] >> 30 + if ram_present < 7: + print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") + sys.exit(0) + + # total, used, free = shutil.disk_usage("\\") + # print("Free drive space: %d/%d GB" % ((free // (2**30)), (total // (2**30)))) + # if (free // (2**30)) < 7: + # print("WARNING: This is space-intensive operation. It cannot continue if you don't have at least 15 GB of free space on the same drive as the script.\nExiting...") + # sys.exit(0) + + print("Initializing...") + files = [] + for dirpath, dirnames, filenames in os.walk(path_to_coca): + files.extend([os.path.join(dirpath, file) for file in filenames]) + + if os.path.isfile("_COCA2.txt"): + print("A file _COCA2.txt was found in the script's directory.") + print("This could be the preprocessed corpus. In that case, you can skip the preprocessing.") + print("Otherwise, this script can delete it and preprocess the corpus.") + print("Clean the file _COCA2.txt?") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + f = open("_COCA2.txt", 'w+') + f.close() + + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + else: + pass + + else: + f = open("_COCA2.txt", 'w+') + f.close() + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + print(" Bigrams collected") + + files = [] + for dirpath, dirnames, filenames in os.walk(path_to_coca): + files.extend([os.path.join(dirpath, file) for file in filenames]) + + if os.path.isfile("_COCA2wfreq.txt"): + print("A file _COCA2wfreq.txt was found in the script's directory.") + print("This could be the preprocessed corpus. In that case, you can skip the preprocessing.") + print("Otherwise, this script can delete it and preprocess the corpus.") + print("Clean the file _COCA2.txt?") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + f = open("_COCA2wfreq.txt", 'w+') + f.close() + + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2wfreq.txt" )) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess_wfreq, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + else: + pass + + else: + f = open("_COCA2wfreq.txt", 'w+') + f.close() + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2wfreq.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess_wfreq, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + print(" Wfreq preprocessing finished") + + print("Preprocessing finished") + print("Counting scores") + + #skip = input("Skip?").lower() + + skip = "n" + + if skip == "n": + print(" Bigram frequency") + #- Bigram frequency (bi.freq.NXT) + + + coca = open("_COCA2.txt", "r") + counter = collections.Counter() + + for line in tqdm.tqdm(coca): + bgs = json.loads(line) + bgs = [x.strip() for x in bgs if not x.startswith("##")] + counter.update(bgs) + # print("\n\n\n\nEARLY STOPPING") + # break + + coca.close() + print(" Cropping the bigram dict to items with freq > 0") #changed 4 to 0 + counter = {k:v for k,v in counter.items() if v > 0} #changed 4 to 0 + counter = dict(counter) + + # backup bigram stats file + print(" Saving") + + backup_out = open("bigrams.json", "w+") + backup_out.write(json.dumps(counter)) + backup_out.close() + + print(" Making a wordcount from %i bigrams" % (len(counter))) + w_freq = collections.Counter() + coca = open("_COCA2wfreq.txt", "r") + for line in tqdm.tqdm(coca): + bgs = json.loads(line) + bgs = [x.strip() for x in bgs if not x.startswith("##")] + w_freq.update(bgs) + + w_freq = dict(w_freq) + coca.close() + + backup_out = open("wfreqs.json", "w+") + backup_out.write(json.dumps(w_freq)) + backup_out.close() + + with open("bigrams.json", "r") as i: + counter = json.loads(i.read()) + with open("wfreqs.json", "r") as i: + w_freq = json.loads(i.read()) + + w_count = 0 + for item, frq in w_freq.items(): # Sum the word frequency to get the total + w_count += frq + + print("Keeping only nice bigrams") + bad = len(counter) + counter = {k:v for k,v in counter.items() if (k.split()[0] in w_freq) and (k.split()[1] in w_freq)} + print("Kept %i/%i bigrams" % (len(counter), bad)) + + print(" TP-D/TP-B") + + #- Direct transitional probability (TPD.bi.NXT) + #How likely is word x+1 to occur after word x? + #Backwards transitional probability (TPB.bi.NXT) + #How likely is word y-1 to occur before word y? + + forward_pairs = {} + backward_pairs = {} + + print(" Dictionarizing word-pair counts") + #create dict of dicts: how many times is a given word followed by another word and vice versa + + for item, freq in tqdm.tqdm(counter.items()): + x_word = item.split()[0] + y_word = item.split()[1] + + # Check the forward freq dictionary, if word x+1 there, add 1 to freq. + # If word x+1 not there add it with freq = 1. + # If word x not there, create it and add word x+1 as first item with freq = 1. + if x_word in forward_pairs: + if y_word in forward_pairs[x_word]: + forward_pairs[x_word][y_word] += freq + else: + forward_pairs[x_word][y_word] = freq + else: + forward_pairs[x_word] = {y_word : freq} + + # Check the backward freq dictionary, if word y-1 there, add 1 to freq. + # If word y-1 not there add it with freq = 1. + # If word y not there, create it and add word y-1 as first item with freq = 1. + if y_word in backward_pairs: + if x_word in backward_pairs[y_word]: + backward_pairs[y_word][x_word] += freq + else: + backward_pairs[y_word][x_word] = freq + else: + backward_pairs[y_word] = {x_word : freq} + + print(" Pairs collected") + + if skip == "n": + + print(" Counting TPD") + # Calculates forward probabilities and saves them as a Decimal(probability) + # The output variable forward_probs is a dict of dicts, form: {"word_x": {"word_x+1(1)": Decimal(0.73), "word_x+1(2)": Decimal(0.27)}} + forward_probs = {} + for x_word in tqdm.tqdm(forward_pairs): + # total = Decimal(0) + # for item in forward_pairs[x_word]: + # total += Decimal(forward_pairs[x_word][item]) + forward_probs[x_word] = {} + for item in forward_pairs[x_word]: + forward_probs[x_word][item] = float(Decimal(forward_pairs[x_word][item])/w_freq[x_word]) + + del forward_pairs + + print(" Counting TPB") + # Calculates forward probabilities and saves them as a Decimal(probability) + # The output variabe forward_probs is a dict of dicts, form: {"word_x": {"word_x-1(1)": Decimal(0.73), "word_x-1(2)": Decimal(0.27)}} + backward_probs = {} + for y_word in tqdm.tqdm(backward_pairs): + # total = Decimal(0) + # for item in backward_pairs[y_word]: + # total += Decimal(backward_pairs[y_word][item]) + backward_probs[y_word] = {} + for item in backward_pairs[y_word]: + backward_probs[y_word][item] = float(Decimal(backward_pairs[y_word][item])/w_freq[y_word]) + + del backward_pairs + + backup_out = open("fwd.json", "w+") + backup_out.write(json.dumps(forward_probs)) + backup_out.close() + + backup_out = open("bckw.json", "w+") + backup_out.write(json.dumps(backward_probs)) + backup_out.close() + + # print(" MI/MI3") + + # # - Mutual information score (MI.NXT for word i given word i-1; doesn't look beyond!) + # #log(Bigram_freq/((item1_freq*item2_freq)/WORDCOUNT)) + + # mi_score = {} + # mi3_score = {} + + # log10_2 = Decimal(log10(2)) # Do not calculate log10(2) every time around + # if settings_stats.mi == "BNC" or settings_stats.mi == "BYU": + # for bigram in tqdm.tqdm(counter): + + # try: + # item1, item2 = bigram.split() + # item1_freq = Decimal(w_freq[item1]) + # item2_freq = Decimal(w_freq[item2]) + + # ## Used by BNCweb/BYU + # denom = item1_freq*item2_freq + # score = Decimal(counter[bigram]*Decimal(w_count))/denom + # mi_score[bigram] = float(Decimal(log(score,10))/log10_2) + + # score3 = Decimal((counter[bigram]**3)*Decimal(w_count))/denom + # mi3_score[bigram] = float(Decimal(log(score3,10))/log10_2) + # except ValueError: + # print("Line 465 ValueError" + bigram) + + # else: + # try: + # ## Based on Wiechmann 2008 + # item1, item2 = bigram.split() + # item1_freq = Decimal(w_freq[item1]) + # item2_freq = Decimal(w_freq[item2]) + + # score = Decimal(counter[bigram])/((item1_freq*item2_freq)/Decimal(w_count)) + # mi_score[bigram] = float(score.ln()) + # except ValueError: + # print("Line 477 ValueError: " + bigram) + + # backup_out = open("miscore.json", "w+") + # backup_out.write(json.dumps(mi_score)) + # backup_out.close() + # del mi_score + # backup_out = open("mi3score.json", "w+") + # backup_out.write(json.dumps(mi3_score)) + # backup_out.close() + # del mi3_score + + # print(" z-score") + + # # - Z score + + # # prob = Wi-1/(w_count-Wi) + # # expected = prob * Wi + # # z-score = bigram-expected/sqrt(expected*(1-prob)) + + # z_score = {} + # for bigram in tqdm.tqdm(counter): + + # try: + # item1, item2 = bigram.split() + # item1_freq = Decimal(w_freq[item1]) + # item2_freq = Decimal(w_freq[item2]) + + # ## Used by BNCweb/BYU + # prob = item1_freq/Decimal(w_count-item2_freq) # probability of item1 + # expe = prob*item2_freq # expected number of bigrams + # numer = Decimal(counter[bigram])-expe # deviation from the expected number + # denom = Decimal(sqrt(expe*(Decimal(1)-prob))) # std.deviation (kind of) + # z_score[bigram] = float(numer/denom) + # except ValueError: + # print("Line 508 ValueError: " + bigram) + + # backup_out = open("zscore.json", "w+") + # backup_out.write(json.dumps(z_score)) + # backup_out.close() + # del z_score + + # print(" t-score") + # t_score = {} + + # dec_w_count = Decimal(w_count) # Do not express the word count as a Decimal every time + # for bigram in tqdm.tqdm(counter): + + # item1, item2 = bigram.split() + + # a = Decimal(counter[bigram]) + # b = Decimal(w_freq[item1]) - a + # c = Decimal(w_freq[item2]) - a + + # expe = ((a+b)*(a+c))/dec_w_count + + # # Based on Gries + # t_score[bigram]= float((a-expe)/Decimal(sqrt(expe))) + + # backup_out = open("tscore.json", "w+") + # backup_out.write(json.dumps(t_score)) + # backup_out.close() + # del t_score + + # print(" delta-p-score") + # delta_p21 = {} + # delta_p12 = {} + + # dec_w_count = Decimal(w_count) # Do not express the word count as a Decimal every time + # for bigram in tqdm.tqdm(counter): + + # item1, item2 = bigram.split() + + # a = Decimal(counter[bigram]) + # b = Decimal(w_freq[item1]) - a + # c = Decimal(w_freq[item2]) - a + # d = dec_w_count-a-b-c + + # p1 = Decimal(a)/Decimal(a+b) + # p2 = Decimal(c)/Decimal(c+d) + + # # Based on Gries + # delta_p21[bigram]= float(p1-p2) + + # p1 = Decimal(a)/Decimal(a+c) + # p2 = Decimal(b)/Decimal(b+d) + + # # Based on Gries + # delta_p12[bigram]= float(p1-p2) + + # backup_out = open("delta_p21.json", "w+") + # backup_out.write(json.dumps(delta_p21)) + # backup_out.close() + + # backup_out = open("delta_p12.json", "w+") + # backup_out.write(json.dumps(delta_p12)) + # backup_out.close() + + # del delta_p21 + # del delta_p12 + + # print(" Dice-score") + # # Dice coefficient; using a,b,c,d just like LL + # # As used in Sketch Engine + # # dice = (2*bigram)/(w1_freq + w2_freq) + + # dice_score = {} + # for bigram in tqdm.tqdm(counter): + + # item1, item2 = bigram.split() + + # a = Decimal(counter[bigram]) + # b = Decimal(w_freq[item1]) - a + # c = Decimal(w_freq[item2]) - a + + # score = Decimal(2*a)/(a+b+a+c) + + # dice_score[bigram]= float(score) + + # backup_out = open("dicescore.json", "w+") + # backup_out.write(json.dumps(dice_score)) + # backup_out.close() + # del dice_score + + # print(" Modified Dice-score") + # # modified Dice coefficient; using a,b,c,d just like LL + # # Kitamura, M., and Y. Matsumoto. 1996. Automatic Ex-traction of Word Sequence Correspondences in Par-allel Corpora. In Proc. 4'" Workshop on Very Large Cmpora, 79-87. 4 August, Copenhagen, Denmark. + # # mod. dice = log2() + + # dice_score = {} + # for bigram in tqdm.tqdm(counter): + + # item1, item2 = bigram.split() + + # a = Decimal(counter[bigram]) + # b = Decimal(w_freq[item1]) - a + # c = Decimal(w_freq[item2]) - a + + # score = Decimal(2)*(a*a)/(a+b+a+c) + + # dice_score[bigram]= log(float(score),2) + + # backup_out = open("moddicescore.json", "w+") + # backup_out.write(json.dumps(dice_score)) + # backup_out.close() + # del dice_score + + # print(" Log-likelihood") + # print(" Preparing LL-score calculation") + + # lltemp = open("lltemp.bck", "w+") # We'll use a temp file to save memory + + # for bigram in tqdm.tqdm(counter): # Prepare the inputs and save them to a temp file - allow multiprocessing without straining RAM + # item1, item2 = bigram.split() + + # a = counter[bigram] + # b = w_freq[item1] - a + # c = w_freq[item2] - a + # d = w_count-a-b-c + + # lltemp.write(json.dumps([bigram, a, b, c, d]) + "\n") + + # lltemp.close() # Write access no longer needed + # lltemp = open("lltemp.bck", "r") # Open with read access only + # lt = lltemp.readlines() # Could be moved to the imap() call, but then the length would be uncertain + + # print(" Counting") + # worker = Pool(4) + # ll_score = [] + # for i in tqdm.tqdm(worker.imap_unordered(llscorer, lt), total=len(lt)): # Counting is done on 4 cores, output saved in ll_score + # ll_score.append(i) + + # del lt + # worker.close() + # worker.join() + + # print(" Dictionarizing and saving") + # ll_score = dict(ll_score) + # backup_out = open("llscore.json", "w+") + # backup_out.write(json.dumps(ll_score)) + # backup_out.close() + # del ll_score + + # lltemp.close() # We don't need the connection anymore + # try: + # os.remove("lltemp.bck") + # except: + # print("Couldn't remove the file lltemp.bck, please do it manually") + # print(" G-score") + + # # - Lexical gravity G (G.NXT) + + # # log((Fbigr * TypeFreqAfterW1)/Fw1) + log((Fbigr * TypeFreqBeforeW2)/Fw2) + + # # G-score needs the number of types following/preceding an item + # backup_out = open("fwd.json", "r") + # forward_probs = json.loads(backup_out.read()) + # backup_out.close() + + # backup_out = open("bckw.json", "r") + # backward_probs = json.loads(backup_out.read()) + # backup_out.close() + + # fwd_types = {} + # for item in forward_probs: + # fwd_types[item] = len(forward_probs[item]) + + # bckw_types = {} + # for item in backward_probs: + # bckw_types[item] = len(backward_probs[item]) + + # gtemp = open("gscoretemp.bck", "w+") # We'll use a temp file to save memory + + # print(" Preparing G-score calculation") + + + # for bigram in tqdm.tqdm(counter): # The calculation is prepared as a file with JSON-serialized inputs to the llcounter() function + # item1, item2 = bigram.split() + # item1_3 = w_freq[item1] + # item2_3 = w_freq[item2] + # item1_2 = fwd_types[item1] + # item2_2 = bckw_types[item2] + # bf = counter[bigram] + # gtemp.write(json.dumps([bigram, bf, item1_2, item2_2, item1_3, item2_3]) + "\n") + + # gtemp.close() # We don't need the write access anymore + + # gtemp = open("gscoretemp.bck", "r") # Open with read-only + # gt = gtemp.readlines() # Could be moved to the imap() call, but then the length would be uncertain + # print(" Ready") + # worker = Pool(4) + # g_score = [] + + # for i in tqdm.tqdm(worker.imap_unordered(gscorer, gt), total=len(gt)): # Counting is done on 4 cores, output saved in g_score + # g_score.append(i) + + # del gt + # worker.close() + # worker.join() + # print(" Dictionarizing and saving") + # g_score = dict(g_score) + # backup_out = open("gscore.json", "w+") + # backup_out.write(json.dumps(g_score)) + # backup_out.close() + # del g_score + + # gtemp.close() # We don't need the connection anymore + # try: + # os.remove("gscoretemp.bck") + # except: + # print("Couldn't remove the file gscoretemp.bck, please do it manually") + + ########### This is a clumsy way of converting the calculated scores into a pandas DataFrame; future versions should get rid of it + # from convert_to_pd import Converter + # worker = Converter() + # worker.convert() + + + ########## This is a clumsy way of calculating dispersion scores; efficient implementation would do that during preprocessing + ####(though memory may be limiting there) + + # from dispersion_counter import DispersionCounter + # worker = DispersionCounter(path=path_to_coca) + # print("Collecting dispersion scores") + # for ext in tqdm.tqdm(["acad", "fic", "news", "mag", "spok"]): + # worker.collect(ext) + # gc.collect() + + # print("Collecting done - preprocessing final data") + # worker.save() + + exit() diff --git a/settings_stats.py b/settings_stats.py new file mode 100644 index 0000000..35d4e25 --- /dev/null +++ b/settings_stats.py @@ -0,0 +1,3 @@ +debug = True +skip = True +mi = "BNC" \ No newline at end of file diff --git a/single_word_freq.py b/single_word_freq.py new file mode 100644 index 0000000..a18d3b0 --- /dev/null +++ b/single_word_freq.py @@ -0,0 +1,317 @@ +import sys +import csv +import tqdm +import psutil +import random +import time +from numpy import array, digitize, zeros, amin, amax, arange, percentile, histogram, argmin, transpose, delete, log, exp, nanmedian +from numpy import random as nrandom +from numpy import sum as nsum +from math import floor, ceil +from nltk import word_tokenize, pos_tag +import matplotlib.pyplot as plt +from copy import deepcopy +from re import match, sub, compile +import json +import os +import pandas as pd + +import gc +gc.enable() + +score_path = "E:/stimuli_coca_exp19/scores/" +mapping_path = "E:/stimuli_coca_exp19/mapping.json" + + +class Tagger: + def __init__(self, iterable_of_tagged): + self.tagging_map = {} + for tagged_bigram in iterable_of_tagged: + tagged_word = tagged_bigram.split()[0].lower().strip() + untagged_word = sub("_[^ ]+", "", tagged_word) + tagged_word = untagged_word + "_" + tagged_word.split("_")[1][0] + if untagged_word in self.tagging_map: + if tagged_word not in self.tagging_map.get(untagged_word): + self.tagging_map[untagged_word].append(tagged_word) + else: + self.tagging_map[untagged_word] = [tagged_word] + print(self.tagging_map) + + def tag(self, item_to_tag): + """Tag an item. If only one option is know, this is assumed to be the correct option. + If multiple options are known, the correct one can be picked or specified manually. + If no option is known, the tags should be written manually.""" + try: + tags = self.tagging_map[item_to_tag] + if len(tags) == 1: + return tags[0] + elif len(tags) > 1: + return self.pick_a_tag_from_selection(item_to_tag, tags) + except: + return self.tag_manually(item_to_tag) + + def pick_a_tag_from_selection(self, item_to_tag, tags): + """Allow the user to select one of multiple offered tags or specify their own tagging""" + print("_"*10) + print("There are multiple possible tags for '{}'".format(item_to_tag)) + for index, tag in enumerate(tags): + print("\t[{}] {}".format(index, tag)) + print("\n\t[m] a manual tag") + + choice = None; + while choice == None: + new_choice = input("\nPick the relevant tag\n> ") + if new_choice.lower() == "m": + return self.tag_manually(item_to_tag) + try: + new_choice = int(new_choice) + if (new_choice >= 0) and (new_choice < len(tags)): + choice = new_choice + except: + pass + + return tags[choice] + + def tag_manually(self, item_to_tag): + """Allow manual tagging of items not found in the simple tagging map. + Expects the user to input valid tags""" + print("_"*10) + print("The item '{}' cannot be tagged semi-automatically. Please tag it manually.".format(item_to_tag)) + #w1, w2 = item_to_tag.split() + + # TODO: input validation/formatting (lowercase?) + w1_tag = input("{}_".format(item_to_tag)) + + return "{}_{}".format(item_to_tag,w1_tag) + + + +class Scorer(object): + """Loads all the score lists, can then be used to assign the scores on a per-item base with the score method. + It is memory-heavy, but could be included in functions which allow interactive collocation input.""" + + def __init__(self): + """Load the required score files. If they are not present in the folder, throw an exception.""" + import os + + print("\nLoading the saved scores") + print(" unigram frequency") + + with open(score_path + "wfreqs_lemma.json", "r") as i: + self.wfreq = json.loads(i.read()) + + print("_________________________________") + + + def score(self, items): + """Score single words. Input is a list items on separate lines. If a bigram is not in the score list, return NA.""" + + items_out = [] + for word in items: + try: + w1_frq = self.wfreq[word] + except: + w1_frq = "NA" + + items_out.append([word, w1_frq]) + + return(items_out) + +class ScorerLemma(object): + """Loads all the score lists, can then be used to assign the scores on a per-item base with the score method. + It is memory-heavy, but could be included in functions which allow interactive collocation input.""" + + def __init__(self): + """Load the required score files. If they are not present in the folder, throw an exception.""" + import os + + print("\nLoading the saved scores") + print(" unigram frequency") + + with open(score_path + "wfreqs_lemma.json", "r") as i: + self.wfreq = json.loads(i.read()) + self.wfreq_lemma = {} + for key, value in self.wfreq.items(): + new_key = sub("_[^ ]+", "", key) + if new_key in self.wfreq_lemma: + new_value = self.wfreq_lemma[new_key] + value + self.wfreq_lemma[new_key] = new_value + else: + self.wfreq_lemma[new_key] = value + + print("_________________________________") + + + def score(self, items): + """Score single words. Input is a list items on separate lines. If a bigram is not in the score list, return NA.""" + + items_out = [] + for word in items: + try: + lemma_freq = self.wfreq_lemma[word] + except: + lemma_freq = "NA" + + items_out.append([word, lemma_freq]) + + return(items_out) + +if __name__ == "__main__": + + inpath = input("Where is the file to load?\n ") + + with open(inpath, "r") as infile: + raw_items = infile.readlines() + raw_items = [item.replace('\n', '') for item in raw_items] + print("First read in items: " + str(raw_items)) + + #very VERY VERY crude lemmatization, specific to my stimuli list + items = [] + for item in raw_items: + if item in ["species_n", "genius_n", "awareness_n", "dress_n", "access_n", "basis_n", "witness_n", "mathmatics_n", "success_n", "business_n", "ms_n", "works_n", "mathematics_n", "changes_n"]: + items.append(item) + elif item.endswith("es_n"): + if item in ["colleagues_n", "languages_n", "values_n", "places_n", "prices_n", "forces_n", "faces_n", "temperatures_n", "candles_n", "everyones_n", "pressures_n", "circumstances_n", "scenes_n"]: + items.append(item.replace("es_n", "e_n")) + else: + items.append(item.replace("es_n", "_n")) + elif item.endswith("s_n"): + items.append(item.replace("s_n", "_n")) + elif item == "led_v": + items.append("lead_v") + elif item == "became_v": + items.append("become_v") + elif item == "need_v": + items.append("need_v") + elif item == "chose_v": + items.append("choose_v") + elif item == "found_v": + items.append("find_v") + elif item in ["stared_v", "discouraged_v", "changed_v", "saved_v", "ensured_v", "supposed_v", "collapsed_v", "continued_v", "saturated_v", "figured_v", "created_v", "noted_v"]: + items.append(item.replace("ed_v", "e_v")) + elif item in ["carried_v", "studied_v"]: + items.append(item.replace("ied_v", "y_v")) + elif item == "deteriorating_v": + items.append("deteriorate_v") + elif item.endswith("ing_v"): + if item in ["locating_v", "noticing_v", "defining_v", "deteriorating_v", "consuming_v"]: + items.append(item.replace("ing_v", "e_v")) + elif item == "getting_v": + items.append("get_v") + else: + items.append(item.replace("ing_v", "_v")) + elif item == "written_v": + items.append("write_v") + elif item == "given_v": + items.append("give_v") + elif item == "heard_v": + items.append("hear_v") + elif item == "came_v": + items.append("come_v") + elif item == "got_v": + items.append("get_v") + elif item == "saw_v": + items.append("see_v") + elif item == "went_v": + items.append("go_v") + elif item in ["been_v", "are_v", "is_v", "were_v", "was_v"]: + items.append("be_v") + elif item == "blew_v": + items.append("blow_v") + elif item == "caught_v": + items.append("catch_v") + elif item == "ran_v": + items.append("run_v") + elif item == "felt_v": + items.append("feel_v") + elif item == "does_v": + items.append("do_v") + elif item == "meant_v": + items.append("mean_v") + elif item == "took_v": + items.append("take_v") + elif item == "made_v": + items.append("make_v") + elif item in ["has_v", "had_v", "has_v"]: + items.append("have_v") + elif item.endswith("ed_v"): + items.append(item.replace("ed_v", "_v")) + elif item.endswith("s_v"): + if item not in []: + items.append(item.replace("s_v", "_v")) + else: + items.append(item) + print("Lemmatized items: " + str(items)) + + + ram_present = psutil.virtual_memory()[0] >> 30 + ram_available = psutil.virtual_memory()[1] >> 30 + + # Check the RAM installed and available, if sufficient use the default scorer, otherwise use the lite version + if ram_present > 7 and ram_available > 5: + pass + else: + print("This is a RAM-intensive operation. You need at least 6 GB of free RAM.") + print("Exiting...") + sys.exit(0) + + if match("[^_]+_", items[0]) == None: + with open(mapping_path, "r") as f: + print("Loading mapping file.") + mapping = json.load(f) + tagger = Tagger(mapping) + print("-----------") + tagged_items = [] + print("Tagging items.") + for item in items: + try: + tagged_item = tagger.tag(item) + tagged_items.append(tagged_item) + except Exception as e: + print(str(e) + str(item)) + else: + tagged_items = items + + print("Tagged items: " + str(tagged_items)) + + # with open('tagged_list.json', "a") as fp: + # for item in tagged_items: + # fp.write(item + "\n") + + scorer = Scorer() + items = scorer.score(tagged_items) + print("Third step items: " + str(items)) + + scores = pd.DataFrame(items, columns=["tagged_word", "freq"]) + + scores["lemma_stem"] = [sub("_[^ ]+","",x) for x in scores["tagged_word"]] + scores["word"] = [sub("_[^ ]+","",x) for x in raw_items] + + del scorer + del items + + lemma_scorer = ScorerLemma() + lemma_scored_items = lemma_scorer.score(scores["lemma_stem"]) + + lemmascores = pd.DataFrame(lemma_scorer.score(scores["lemma_stem"]), columns=["lemma_stem", "lemma_freq"]) + + scores = scores.merge(lemmascores, on="lemma_stem", how="outer") + + scores = scores[["word", "lemma_stem", "tagged_word", "freq", "lemma_freq"]] + + scores.drop_duplicates().reset_index(drop=True) + + print(scores) + + print("Saving") + + if len(sys.argv) > 2: + outpath = sys.argv[2] + else: + outpath = input("Where should the results be saved?\n ") + + print("Saving") + scores.to_csv(outpath, index=False) + print("Done. Press RETURN to exit") + wait = input() + sys.exit(0) diff --git a/string_to_lemma.py b/string_to_lemma.py new file mode 100644 index 0000000..36230a4 --- /dev/null +++ b/string_to_lemma.py @@ -0,0 +1,180 @@ +from os import walk +from bs4 import BeautifulSoup +import json +import psutil +#import shutil +import pickle +from decimal import Decimal +import settings_stats +import re +from multiprocessing import Pool, Manager +import collections +import tqdm +from math import log10 +from math import sqrt +from math import log +import os +import sys + +#path_to_coca = "E:/coca_2019_wlp" +path_to_coca = "/Users/kylamcconnell/Documents/coca_2019_test" + +def preprocess(filename, queue): + with open(filename, "r", errors="replace") as i: + doc = i.read() + + #doc = re.sub("_", "", doc) + #doc = re.sub("\t+", "\t", doc) + doc = doc.split("\n") + doc = [word.split("\t") for word in doc] + + docs = [] + d = [] + declined = re.compile("@_") + for word in doc: + if len(word) == 4: #changed from 3 to 4 + pos = re.sub("_.*", "", word[3]) + pos_lemma = pos[0] if len(pos) > 0 else pos + + d.append([[word[1].lower(), pos],[word[2].lower(), pos_lemma]]) #changed from 0 2 1 20 to 1 3 2 pos (defined above) + + d = [["_".join(w).strip(), "_".join(l).strip()] for w,l in d] + d = [[x[0] + " " + y[0], x[1] + " " + y[1]] for x,y in zip(d[0:-1], d[1:]) if re.match("\W", x[0]) == None and re.match("\W", y[0]) == None] + d = [[x,y] for x,y in d if re.match(declined, x) == None] + + docs.append(d) + + docs = [item for sublist in docs for item in sublist] + + queue.put(json.dumps(docs)) + + +def listener(queue, filename): + f = open(filename, 'w') + while 1: + m = queue.get() + if m == 'kill': + break + f.write(str(m) + '\n') + f.flush() + f.close() + +# Multiprocessing needs this if-statement, otherwise it won't work properly +if __name__ == "__main__": + + ram_present = psutil.virtual_memory()[0] >> 30 + if ram_present < 7: + print("WARNING: This is RAM-intensive operation. It cannot continue if you don't have at least 8 GB of RAM.\nExiting...") + sys.exit(0) + + #total, used, free = shutil.disk_usage("\\") + #print("Free drive space: %d/%d GB" % ((free // (2**30)), (total // (2**30)))) + #if (free // (2**30)) < 7: + #print("WARNING: This is space-intensive operation. It cannot continue if you don't have at least 15 GB of free space on the same drive as the script.\nExiting...") + #sys.exit(0) + + print("Initializing...") + files = [] + for dirpath, dirnames, filenames in os.walk(path_to_coca): + files.extend([os.path.join(dirpath, file) for file in filenames]) + + if os.path.isfile("_COCA2_mapping.txt"): + print("A file _COCA2_mapping.txt was found in the script's directory.") + print("This could be the preprocessed corpus. In that case, you can skip the preprocessing.") + print("Otherwise, this script can delete it and preprocess the corpus.") + print("Clean the file _COCA2_mapping.txt?") + dec = "" + while dec.lower() not in set(["y", "n"]): + dec = input("[y/N]") or "N" + + if dec.lower() == "y": + f = open("_COCA2_mapping.txt", 'w+') + f.close() + + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2_mapping.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + else: + pass + + else: + f = open("_COCA2_mapping.txt", 'w+') + f.close() + manager = Manager() + queue = manager.Queue() + pool = Pool(4) + + #put listener to work first + watcher = pool.apply_async(listener, (queue, "_COCA2_mapping.txt")) + + #fire off workers + jobs = [] + for filename in files: + job = pool.apply_async(preprocess, (filename, queue)) + jobs.append(job) + + # collect results from the workers through the pool result queue + for job in tqdm.tqdm(jobs): + job.get() + + #now we are done, kill the listener + queue.put('kill') + pool.close() + + print(" Bigrams collected") + + + print("Preprocessing finished") + print("Counting scores") + + print(" Bigram frequency") + #- Bigram frequency (bi.freq.NXT) + + with open("bigrams.json", "r") as f: + frequent = json.load(f) + frequent = set([x for x in frequent]) + + coca = open("_COCA2_mapping.txt", "r") + mapping = {} + log = open("conversionlog.txt", "w+") + + for line in coca: + bgs = json.loads(line) + bgs = [[x.strip(), y.strip()] for x,y in bgs] #if not x.startswith("##")] + for string, lemma in bgs: + if string in frequent: + if string in mapping: + if mapping[string] == lemma: + pass + else: + log.write('%s : %s versus %s\n' % (string, lemma, mapping[string])) + else: + mapping[string] = lemma + else: + print(string) + del bgs + coca.close() + + import gc + gc.collect() + + with open("mapping.json", "w+") as f: + json.dump(mapping, f)