| from lib.CoNLL_Annotation import * |
| from collections import Counter, defaultdict |
| import pandas as pd |
| import numpy as np |
| from sklearn.metrics import precision_recall_fscore_support as eval_f1 |
| from tabulate import tabulate |
| import logging, argparse, sys |
| from datetime import datetime |
| |
| |
| tree_tagger_fixes = { |
| "die": "der", |
| "eine": "ein", |
| "dass": "daß", |
| "keine": "kein", |
| "dies": "dieser", |
| "erst": "erster", |
| "andere": "anderer", |
| "alle": "aller", |
| "Sie": "sie", |
| "wir": "uns", |
| "alle": "aller", |
| "wenige": "wenig" |
| } |
| |
| |
| def save_evaluated(all_sys, all_gld, out_path, print_gold=True): |
| with open(out_path, "w") as out: |
| if print_gold: |
| out.write(f"ORIGINAL_CORPUS_TAGS\n\nTAG\tGLD_COUNT\tSYS_COUNT\n") |
| for g_tag,g_count in sorted(all_gld.items()): |
| s_count = all_sys.get(g_tag, 0) |
| out.write(f"{g_tag}\t{g_count}\t{s_count}\n") |
| |
| out.write("\n\nSYSTEM_ONLY_TAGS\n\nTAG\tG_COUNT\tSYS_COUNT\n") |
| for s_tag,s_count in sorted(all_sys.items()): |
| g_count = all_gld.get(s_tag, 0) |
| if g_count == 0: |
| out.write(f"{s_tag}\t{g_count}\t{s_count}\n") |
| |
| |
| |
| def eval_lemma(sys, gld): |
| match, err, symbol = 0, 0, [] |
| y_gld, y_pred, mistakes = [], [], [] |
| for i, gld_tok in enumerate(gld.tokens): |
| sys_lemma = sys.tokens[i].lemma |
| # sys_lemma = tree_tagger_fixes.get(sys.tokens[i].lemma, sys.tokens[i].lemma) # Omit TreeTagger "errors" because of article lemma disagreement |
| y_gld.append(gld_tok.pos_tag) |
| y_pred.append(sys_lemma) |
| if gld_tok.lemma == sys_lemma: |
| match += 1 |
| elif not sys.tokens[i].lemma.isalnum(): # Turku does not lemmatize symbols (it only copies them) => ERR ((',', '--', ','), 43642) |
| symbol.append(sys.tokens[i].lemma) |
| if sys.tokens[i].word == sys.tokens[i].lemma: |
| match += 1 |
| else: |
| err += 1 |
| else: |
| err += 1 |
| mistakes.append((gld_tok.word, gld_tok.lemma, sys.tokens[i].lemma)) |
| return y_gld, y_pred, match, err, symbol, mistakes |
| |
| |
| def eval_pos(sys, gld): |
| match, mistakes = 0, [] |
| y_gld, y_pred = [], [] |
| for i, gld_tok in enumerate(gld.tokens): |
| y_gld.append(gld_tok.pos_tag) |
| y_pred.append(sys.tokens[i].pos_tag) |
| # pos_all_pred[gld_tok.pos_tag] += 1 |
| # pos_all_gold[sys.tokens[i].pos_tag] += 1 |
| if gld_tok.pos_tag == sys.tokens[i].pos_tag: |
| match += 1 |
| elif gld_tok.pos_tag == "$." and sys.tokens[i].pos_tag == "$": |
| match += 1 |
| y_pred = y_pred[:-1] + ["$."] |
| else: |
| mistakes.append((gld_tok.word, gld_tok.pos_tag, sys.tokens[i].pos_tag)) |
| return y_gld, y_pred, match, mistakes |
| |
| |
| |
| if __name__ == "__main__": |
| """ |
| EVALUATIONS: |
| |
| ********** TIGER CORPUS ALL ************ |
| |
| python systems/evaluate.py -t Turku --corpus_name Tiger --gld_token_type CoNLL09_Token \ |
| --sys_file /home/daza/datasets/TIGER_conll/tiger_turku_parsed.conllu \ |
| --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09 |
| |
| python systems/evaluate.py -t SpaCy --corpus_name Tiger --gld_token_type CoNLL09_Token \ |
| --sys_file /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu \ |
| --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09 |
| |
| python systems/evaluate.py -t RNNTagger --corpus_name Tiger --gld_token_type CoNLL09_Token \ |
| --sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.RNNTagger.conll \ |
| --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09 |
| |
| python systems/evaluate.py -t TreeTagger --corpus_name Tiger --gld_token_type CoNLL09_Token \ |
| --sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.TreeTagger.conll \ |
| --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09 |
| |
| |
| ********** TIGER CORPUS TEST ************ |
| |
| python systems/evaluate.py -t SpaCy --corpus_name TigerTestOld \ |
| --sys_file /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.test.conllu \ |
| --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.OldOrth.test.conll |
| |
| python systems/evaluate.py -t SpaCy --corpus_name TigerTestNew \ |
| --sys_file /home/daza/datasets/TIGER_conll/Tiger.NewOrth.test.spacy_parsed.conllu\ |
| --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll |
| |
| |
| python systems/evaluate.py -t Turku --corpus_name TigerTestNew \ |
| --sys_file /home/daza/datasets/TIGER_conll/sys_outputs/Tiger.NewOrth.test.turku_parsed.conllu \ |
| --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll |
| |
| ********** UNIVERSAL DEPENDENCIES TEST-SET ************ |
| |
| python systems/evaluate.py -t Turku --corpus_name DE_GSD \ |
| --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu.parsed.0.conllu \ |
| --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu |
| |
| python systems/evaluate.py -t SpaCyGL --corpus_name DE_GSD \ |
| --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.germalemma.conllu \ |
| --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu |
| |
| python systems/evaluate.py -t SpaCy --corpus_name DE_GSD \ |
| --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.conllu \ |
| --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu |
| |
| python systems/evaluate.py -t RNNTagger --corpus_name DE_GSD \ |
| --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.RNNtagger.parsed.conll \ |
| --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu |
| |
| python systems/evaluate.py -t TreeTagger --corpus_name DE_GSD \ |
| --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.treetagger.parsed.conll \ |
| --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu |
| |
| """ |
| |
| # ===================================================================================== |
| # INPUT PARAMS |
| # ===================================================================================== |
| parser = argparse.ArgumentParser() |
| parser.add_argument("-s", "--sys_file", help="System output in CoNLL-U Format", required=True) |
| parser.add_argument("-g", "--gld_file", help="Gold Labels to evaluate in CoNLL-U Format", required=True) |
| parser.add_argument("-c", "--corpus_name", help="Corpus Name for Gold Labels", required=True) |
| parser.add_argument("-t", "--type_sys", help="Which system produced the outputs", default="system") |
| parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLLUP_Token") |
| parser.add_argument("-cs", "--comment_str", help="CoNLL Format of comentaries inside the file", default="#") |
| args = parser.parse_args() |
| |
| # ===================================================================================== |
| # LOGGING INFO ... |
| # ===================================================================================== |
| logger = logging.getLogger(__name__) |
| console_hdlr = logging.StreamHandler(sys.stdout) |
| file_hdlr = logging.FileHandler(filename=f"logs/Eval_{args.corpus_name}.{args.type_sys}.log") |
| logging.basicConfig(level=logging.INFO, handlers=[console_hdlr, file_hdlr]) |
| now_is = datetime.now().strftime('%Y-%m-%d %H:%M:%S') |
| logger.info(f"\n\nEvaluating {args.corpus_name} Corpus {now_is}") |
| |
| # Read the Original GOLD Annotations [CoNLL09, CoNLLUP] |
| gld_generator = read_conll_generator(args.gld_file, token_class=get_token_type(args.gld_token_type), comment_str=args.comment_str) |
| # Read the Annotations Generated by the Automatic Parser [Turku, SpaCy, RNNTagger] |
| if args.type_sys == "RNNTagger": |
| sys_generator = read_conll_generator(args.sys_file, token_class=RNNTagger_Token, comment_str="#") |
| elif args.type_sys == "TreeTagger": |
| sys_generator = read_conll_generator(args.sys_file, token_class=RNNTagger_Token, sent_sep="</S>", comment_str="#") |
| else: |
| sys_generator = read_conll_generator(args.sys_file, token_class=CoNLLUP_Token, comment_str="#") |
| |
| lemma_all_match, lemma_all_err, lemma_all_mistakes = 0, 0, [] |
| lemma_all_symbols, sys_only_lemmas = [], [] |
| pos_all_match, pos_all_err, pos_all_mistakes = 0, 0, [] |
| pos_all_pred, pos_all_gld = [], [] |
| lemma_all_pred, lemma_all_gld = [], [] |
| n_sents = 0 |
| |
| for i, (s,g) in enumerate(zip(sys_generator, gld_generator)): |
| # print([x.word for x in s.tokens]) |
| # print([x.word for x in g.tokens]) |
| assert len(s.tokens) == len(g.tokens), f"Token Mismatch! S={len(s.tokens)} G={len(g.tokens)} IX={i+1}" |
| n_sents += 1 |
| # Lemmas ... |
| lemma_gld, lemma_pred, lemma_match, lemma_err, lemma_sym, mistakes = eval_lemma(s,g) |
| lemma_all_match += lemma_match |
| lemma_all_err += lemma_err |
| lemma_all_mistakes += mistakes |
| lemma_all_symbols += lemma_sym |
| lemma_all_pred += lemma_pred |
| lemma_all_gld += lemma_gld |
| # POS Tags ... |
| pos_gld, pos_pred, pos_match, pos_mistakes = eval_pos(s, g) |
| pos_all_pred += pos_pred |
| pos_all_gld += pos_gld |
| pos_all_match += pos_match |
| pos_all_err += len(pos_mistakes) |
| pos_all_mistakes += pos_mistakes |
| |
| logger.info(f"A total of {n_sents} sentences were analyzed") |
| |
| # Lemmas ... |
| logger.info(f"Lemma Matches = {lemma_all_match} || Errors = {lemma_all_err} || Symbol Chars = {len(lemma_all_symbols)}") |
| logger.info(f"Lemma Accuracy = {(lemma_all_match*100/(lemma_all_match + lemma_all_err)):.2f}%\n") |
| lemma_miss_df = pd.DataFrame(lemma_all_mistakes, columns =['Gold_Word', 'Gold_Lemma', 'Sys_Lemma']).value_counts() |
| lemma_miss_df.to_csv(path_or_buf=f"outputs/LemmaErrors.{args.corpus_name}.{args.type_sys}.tsv", sep="\t") |
| save_evaluated(Counter(lemma_all_pred), Counter(lemma_all_gld), |
| f"outputs/Lemma-Catalogue.{args.corpus_name}.{args.type_sys}.txt", print_gold=False) |
| |
| # POS Tags ... |
| logger.info(f"POS Matches = {pos_all_match} || Errors = {pos_all_err}") |
| logger.info(f"POS Tagging Accuracy = {(pos_all_match*100/(pos_all_match + pos_all_err)):.2f}%\n") |
| pos_miss_df = pd.DataFrame(pos_all_mistakes, columns =['Gold_Word', 'Gold_POS', 'Sys_POS']).value_counts() |
| pos_miss_df.to_csv(path_or_buf=f"outputs/POS-Errors.{args.corpus_name}.{args.type_sys}.tsv", sep="\t") |
| save_evaluated(Counter(pos_all_pred), Counter(pos_all_gld), f"outputs/POS-Catalogue.{args.corpus_name}.{args.type_sys}.txt") |
| |
| ordered_labels = sorted(set(pos_all_gld)) |
| p_labels, r_labels, f_labels, support = eval_f1(y_true=pos_all_gld, y_pred=pos_all_pred, labels=ordered_labels , average=None) |
| scores_per_label = zip(ordered_labels, [x*100 for x in p_labels], [x*100 for x in r_labels], [x*100 for x in f_labels]) |
| logger.info("\n\n") |
| logger.info(tabulate(scores_per_label, headers=["POS Tag","Precision", "Recall", "F1"], floatfmt=".2f")) |
| p_labels, r_labels, f_labels, support = eval_f1(y_true=np.array(pos_all_gld), y_pred=np.array(pos_all_pred), average='macro', zero_division=0) |
| logger.info(f"Total Prec = {p_labels*100}\tRec = {r_labels*100}\tF1 = {f_labels*100}") |
| |
| |