Reestructured project for ordered growth and scaling
diff --git a/TIGER/tiger_evaluate.py b/TIGER/tiger_evaluate.py
new file mode 100644
index 0000000..e89e277
--- /dev/null
+++ b/TIGER/tiger_evaluate.py
@@ -0,0 +1,96 @@
+from CoNLL_Annotation import *
+from collections import Counter
+import pandas as pd
+import numpy as np
+from sklearn.metrics import precision_recall_fscore_support as eval_f1
+from tabulate import tabulate
+
+
+def eval_lemma(sys, gld):
+ match, err, symbol = 0, 0, []
+ mistakes = []
+ for i, gld_tok in enumerate(gld.tokens):
+ if gld_tok.lemma == sys.tokens[i].lemma:
+ match += 1
+ elif not sys.tokens[i].lemma.isalnum(): # This was added because Turku does not lemmatize symbols (it only copies them) => ERR ((',', '--', ','), 43642)
+ symbol.append(sys.tokens[i].lemma)
+ if sys.tokens[i].word == sys.tokens[i].lemma:
+ match += 1
+ else:
+ err += 1
+ else:
+ err += 1
+ mistakes.append((gld_tok.word, gld_tok.lemma, sys.tokens[i].lemma))
+ return match, err, symbol, mistakes
+
+
+def eval_pos(sys, gld):
+ match, mistakes = 0, []
+ y_gld, y_pred = [], []
+ for i, gld_tok in enumerate(gld.tokens):
+ y_gld.append(gld_tok.pos_tag)
+ y_pred.append(sys.tokens[i].pos_tag)
+ all_pos_labels.add(gld_tok.pos_tag)
+ all_pos_labels.add(sys.tokens[i].pos_tag)
+ if gld_tok.pos_tag == sys.tokens[i].pos_tag:
+ match += 1
+ else:
+ mistakes.append((gld_tok.word, gld_tok.pos_tag, sys.tokens[i].pos_tag))
+ return y_gld, y_pred, match, mistakes
+
+
+
+if __name__ == "__main__":
+
+ # Read the Original TiGeR Annotations
+ gld_filename = "/home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09"
+ gld_generator = read_conll_generator(gld_filename, token_class=CoNLL09_Token)
+
+ # Read the Annotations Generated by the Automatic Parser [Turku]
+ sys_filename = "/home/daza/datasets/TIGER_conll/tiger_turku_parsed.conllu"
+ sys_generator = read_conll_generator(sys_filename, token_class=CoNLLUP_Token)
+
+ lemma_all_match, lemma_all_err, lemma_all_mistakes = 0, 0, []
+ lemma_all_symbols = []
+ pos_all_match, pos_all_err, pos_all_mistakes = 0, 0, []
+ pos_all_pred, pos_all_gld = [], []
+ all_pos_labels = set()
+
+ for i, (s,g) in enumerate(zip(sys_generator, gld_generator)):
+ assert len(s.tokens) == len(g.tokens), "Token Mismatch!"
+ # Lemmas ...
+ lemma_match, lemma_err, lemma_sym, mistakes = eval_lemma(s,g)
+ lemma_all_match += lemma_match
+ lemma_all_err += lemma_err
+ lemma_all_mistakes += mistakes
+ lemma_all_symbols += lemma_sym
+ # POS Tags ...
+ pos_gld, pos_pred, pos_match, pos_mistakes = eval_pos(s, g)
+ pos_all_pred += pos_pred
+ pos_all_gld += pos_gld
+ pos_all_match += pos_match
+ pos_all_err += len(pos_mistakes)
+ pos_all_mistakes += pos_mistakes
+
+ # Lemmas ...
+ print(f"Lemma Matches = {lemma_all_match} || Errors = {lemma_all_err} || Symbol Chars = {len(lemma_all_symbols)}")
+ print(f"Lemma Accuracy = {(lemma_all_match*100/(lemma_all_match + lemma_all_err)):.2f}%\n")
+ lemma_miss_df = pd.DataFrame(lemma_all_mistakes, columns =['Gold_Word', 'Gold_Lemma', 'Sys_Lemma']).value_counts()
+ lemma_miss_df.to_csv(path_or_buf="LemmaErrors.tsv", sep="\t")
+
+ # POS Tags ...
+ print(f"POS Matches = {pos_all_match} || Errors = {pos_all_err}")
+ print(f"POS Tagging Accuracy = {(pos_all_match*100/(pos_all_match + pos_all_err)):.2f}%\n")
+ pos_miss_df = pd.DataFrame(pos_all_mistakes, columns =['Gold_Word', 'Gold_POS', 'Sys_POS']).value_counts()
+ pos_miss_df.to_csv(path_or_buf="POS-Errors.tsv", sep="\t")
+
+ ordered_labels = sorted(all_pos_labels)
+ p_labels, r_labels, f_labels, support = eval_f1(y_true=pos_all_gld, y_pred=pos_all_pred, labels=ordered_labels , average=None)
+ scores_per_label = zip(ordered_labels, [x*100 for x in p_labels], [x*100 for x in r_labels], [x*100 for x in f_labels])
+ print("\n\n")
+ print(tabulate(scores_per_label, headers=["POS Tag","Precision", "Recall", "F1"], floatfmt=".2f"))
+ print("\n Total Prec, Rec, and F1 Score: ")
+ p_labels, r_labels, f_labels, support = eval_f1(y_true=np.array(pos_all_gld), y_pred=np.array(pos_all_pred), average='micro', zero_division=0)
+ print(p_labels*100, r_labels*100, f_labels*100)
+
+
\ No newline at end of file