Gitiles
Code Review Sign In
korap.ids-mannheim.de / KorAP / sota-pos-lemmatizers / fb308a2618feb359ea06335c4d4cf01a2906e2a9 / . / systems_eval / evaluate.py
blob: b87e203f8f90b84196b1c56179e55bccd2da0d54 [file] [log] [blame]
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]1from lib.CoNLL_Annotation import *
2from collections import Counter, defaultdict
3import pandas as pd
4import numpy as np
5from sklearn.metrics import precision_recall_fscore_support as eval_f1
6from tabulate import tabulate
7import logging, argparse, sys
8from datetime import datetime
9
10
11tree_tagger_fixes = {
12 "die": "der",
13 "eine": "ein",
14 "dass": "daß",
15 "keine": "kein",
16 "dies": "dieser",
17 "erst": "erster",
18 "andere": "anderer",
19 "alle": "aller",
20 "Sie": "sie",
21 "wir": "uns",
22 "alle": "aller",
23 "wenige": "wenig"
24}
25
26
27def save_evaluated(all_sys, all_gld, out_path, print_gold=True):
28 with open(out_path, "w") as out:
29 if print_gold:
30 out.write(f"ORIGINAL_CORPUS_TAGS\n\nTAG\tGLD_COUNT\tSYS_COUNT\n")
31 for g_tag,g_count in sorted(all_gld.items()):
32 s_count = all_sys.get(g_tag, 0)
33 out.write(f"{g_tag}\t{g_count}\t{s_count}\n")
34
35 out.write("\n\nSYSTEM_ONLY_TAGS\n\nTAG\tG_COUNT\tSYS_COUNT\n")
36 for s_tag,s_count in sorted(all_sys.items()):
37 g_count = all_gld.get(s_tag, 0)
38 if g_count == 0:
39 out.write(f"{s_tag}\t{g_count}\t{s_count}\n")
40
41
42
43def eval_lemma(sys, gld):
44 match, err, symbol = 0, 0, []
45 y_gld, y_pred, mistakes = [], [], []
46 for i, gld_tok in enumerate(gld.tokens):
dazafb308a22021-01-27 16:20:08 +0100[diff] [blame^]47<<<<<<< HEAD
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]48 sys_lemma = sys.tokens[i].lemma
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]49 # sys_lemma = tree_tagger_fixes.get(sys.tokens[i].lemma, sys.tokens[i].lemma) # Omit TreeTagger "errors" because of article lemma disagreement
dazafb308a22021-01-27 16:20:08 +0100[diff] [blame^]50=======
daza54e072e2020-11-04 11:06:26 +0100[diff] [blame]51 # sys_lemma = tree_tagger_fixes.get(sys.tokens[i].lemma, sys.tokens[i].lemma) # Omit TreeTagger "errors" because of article lemma disagreement
52 sys_lemma = sys.tokens[i].lemma
dazafb308a22021-01-27 16:20:08 +0100[diff] [blame^]53>>>>>>> 54e072e61c24a3ce12a7f47e17e9d8d0d1583236
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]54 y_gld.append(gld_tok.pos_tag)
55 y_pred.append(sys_lemma)
56 if gld_tok.lemma == sys_lemma:
57 match += 1
58 elif not sys.tokens[i].lemma.isalnum(): # Turku does not lemmatize symbols (it only copies them) => ERR ((',', '--', ','), 43642)
59 symbol.append(sys.tokens[i].lemma)
60 if sys.tokens[i].word == sys.tokens[i].lemma:
61 match += 1
62 else:
63 err += 1
64 else:
65 err += 1
66 mistakes.append((gld_tok.word, gld_tok.lemma, sys.tokens[i].lemma))
67 return y_gld, y_pred, match, err, symbol, mistakes
68
69
70def eval_pos(sys, gld):
71 match, mistakes = 0, []
72 y_gld, y_pred = [], []
73 for i, gld_tok in enumerate(gld.tokens):
74 y_gld.append(gld_tok.pos_tag)
75 y_pred.append(sys.tokens[i].pos_tag)
76 # pos_all_pred[gld_tok.pos_tag] += 1
77 # pos_all_gold[sys.tokens[i].pos_tag] += 1
78 if gld_tok.pos_tag == sys.tokens[i].pos_tag:
79 match += 1
80 elif gld_tok.pos_tag == "$." and sys.tokens[i].pos_tag == "$":
81 match += 1
82 y_pred = y_pred[:-1] + ["$."]
83 else:
84 mistakes.append((gld_tok.word, gld_tok.pos_tag, sys.tokens[i].pos_tag))
85 return y_gld, y_pred, match, mistakes
86
87
88
89if __name__ == "__main__":
90 """
91 EVALUATIONS:
92
93 ********** TIGER CORPUS ALL ************
94
dazafb308a22021-01-27 16:20:08 +0100[diff] [blame^]95<<<<<<< HEAD
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]96 python systems/evaluate.py -t Turku --corpus_name Tiger --gld_token_type CoNLL09_Token \
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]97 --sys_file /home/daza/datasets/TIGER_conll/tiger_turku_parsed.conllu \
98 --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
99
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]100 python systems/evaluate.py -t SpaCy --corpus_name Tiger --gld_token_type CoNLL09_Token \
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]101 --sys_file /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu \
102 --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
103
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]104 python systems/evaluate.py -t RNNTagger --corpus_name Tiger --gld_token_type CoNLL09_Token \
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]105 --sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.RNNTagger.conll \
106 --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
107
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]108 python systems/evaluate.py -t TreeTagger --corpus_name Tiger --gld_token_type CoNLL09_Token \
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]109 --sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.TreeTagger.conll \
110 --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
111
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]112
113 ********** TIGER CORPUS TEST ************
114
115 python systems/evaluate.py -t SpaCy --corpus_name TigerTestOld \
116 --sys_file /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.test.conllu \
117 --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.OldOrth.test.conll
118
119 python systems/evaluate.py -t SpaCy --corpus_name TigerTestNew \
120 --sys_file /home/daza/datasets/TIGER_conll/Tiger.NewOrth.test.spacy_parsed.conllu\
121 --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll
122
123
124 python systems/evaluate.py -t Turku --corpus_name TigerTestNew \
125 --sys_file /home/daza/datasets/TIGER_conll/sys_outputs/Tiger.NewOrth.test.turku_parsed.conllu \
126 --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll
127
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]128 ********** UNIVERSAL DEPENDENCIES TEST-SET ************
129
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]130 python systems/evaluate.py -t Turku --corpus_name DE_GSD \
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]131 --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu.parsed.0.conllu \
132 --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
133
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]134 python systems/evaluate.py -t SpaCyGL --corpus_name DE_GSD \
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]135 --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.germalemma.conllu \
136 --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
137
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]138 python systems/evaluate.py -t SpaCy --corpus_name DE_GSD \
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]139 --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.conllu \
140 --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
141
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]142 python systems/evaluate.py -t RNNTagger --corpus_name DE_GSD \
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]143 --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.RNNtagger.parsed.conll \
144 --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
145
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]146 python systems/evaluate.py -t TreeTagger --corpus_name DE_GSD \
dazafb308a22021-01-27 16:20:08 +0100[diff] [blame^]147=======
daza54e072e2020-11-04 11:06:26 +0100[diff] [blame]148 python systems/evaluate.py -t Turku --corpus_name Tiger\
149 --sys_file /home/daza/datasets/TIGER_conll/tiger_turku_parsed.conllu \
150 --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
151
152 python systems/evaluate.py -t SpaCy --corpus_name Tiger\
153 --sys_file /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu \
154 --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
155
156 python systems/evaluate.py -t RNNTagger --corpus_name Tiger\
157 --sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.RNNTagger.conll \
158 --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
159
160 python systems/evaluate.py -t TreeTagger --corpus_name Tiger\
161 --sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.TreeTagger.conll \
162 --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
163
164 ********** UNIVERSAL DEPENDENCIES TEST-SET ************
165
166 python systems/evaluate.py -t Turku --gld_token_type CoNLLUP_Token --corpus_name DE_GSD\
167 --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu.parsed.0.conllu \
168 --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
169
170 python systems/evaluate.py -t SpaCyGL --gld_token_type CoNLLUP_Token --corpus_name DE_GSD\
171 --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.germalemma.conllu \
172 --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
173
174 python systems/evaluate.py -t SpaCy --gld_token_type CoNLLUP_Token --corpus_name DE_GSD\
175 --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.conllu \
176 --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
177
178 python systems/evaluate.py -t RNNTagger --gld_token_type CoNLLUP_Token --corpus_name DE_GSD\
179 --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.RNNtagger.parsed.conll \
180 --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
181
182 python systems/evaluate.py -t TreeTagger --gld_token_type CoNLLUP_Token --corpus_name DE_GSD\
dazafb308a22021-01-27 16:20:08 +0100[diff] [blame^]183>>>>>>> 54e072e61c24a3ce12a7f47e17e9d8d0d1583236
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]184 --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.treetagger.parsed.conll \
185 --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
186
187 """
188
189 # =====================================================================================
190 # INPUT PARAMS
191 # =====================================================================================
192 parser = argparse.ArgumentParser()
193 parser.add_argument("-s", "--sys_file", help="System output in CoNLL-U Format", required=True)
194 parser.add_argument("-g", "--gld_file", help="Gold Labels to evaluate in CoNLL-U Format", required=True)
dazafb308a22021-01-27 16:20:08 +0100[diff] [blame^]195<<<<<<< HEAD
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]196 parser.add_argument("-c", "--corpus_name", help="Corpus Name for Gold Labels", required=True)
dazad7d70752021-01-12 18:17:49 +0100[diff] [blame]197 parser.add_argument("-t", "--type_sys", help="Which system produced the outputs", default="system")
198 parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLLUP_Token")
dazafb308a22021-01-27 16:20:08 +0100[diff] [blame^]199=======
daza54e072e2020-11-04 11:06:26 +0100[diff] [blame]200 parser.add_argument("-t", "--type_sys", help="Which system produced the outputs", default="system")
201 parser.add_argument("-c", "--corpus_name", help="Corpus Name for Gold Labels", required=True)
202 parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLL09_Token")
dazafb308a22021-01-27 16:20:08 +0100[diff] [blame^]203>>>>>>> 54e072e61c24a3ce12a7f47e17e9d8d0d1583236
dazae3bc92e2020-11-04 11:06:26 +0100[diff] [blame]204 parser.add_argument("-cs", "--comment_str", help="CoNLL Format of comentaries inside the file", default="#")
205 args = parser.parse_args()
206
207 # =====================================================================================
208 # LOGGING INFO ...
209 # =====================================================================================
210 logger = logging.getLogger(__name__)
211 console_hdlr = logging.StreamHandler(sys.stdout)
212 file_hdlr = logging.FileHandler(filename=f"logs/Eval_{args.corpus_name}.{args.type_sys}.log")
213 logging.basicConfig(level=logging.INFO, handlers=[console_hdlr, file_hdlr])
214 now_is = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
215 logger.info(f"\n\nEvaluating {args.corpus_name} Corpus {now_is}")
216
217 # Read the Original GOLD Annotations [CoNLL09, CoNLLUP]
218 gld_generator = read_conll_generator(args.gld_file, token_class=get_token_type(args.gld_token_type), comment_str=args.comment_str)
219 # Read the Annotations Generated by the Automatic Parser [Turku, SpaCy, RNNTagger]
220 if args.type_sys == "RNNTagger":
221 sys_generator = read_conll_generator(args.sys_file, token_class=RNNTagger_Token, comment_str="#")
222 elif args.type_sys == "TreeTagger":
223 sys_generator = read_conll_generator(args.sys_file, token_class=RNNTagger_Token, sent_sep="</S>", comment_str="#")
224 else:
225 sys_generator = read_conll_generator(args.sys_file, token_class=CoNLLUP_Token, comment_str="#")
226
227 lemma_all_match, lemma_all_err, lemma_all_mistakes = 0, 0, []
228 lemma_all_symbols, sys_only_lemmas = [], []
229 pos_all_match, pos_all_err, pos_all_mistakes = 0, 0, []
230 pos_all_pred, pos_all_gld = [], []
231 lemma_all_pred, lemma_all_gld = [], []
232 n_sents = 0
233
234 for i, (s,g) in enumerate(zip(sys_generator, gld_generator)):
235 # print([x.word for x in s.tokens])
236 # print([x.word for x in g.tokens])
237 assert len(s.tokens) == len(g.tokens), f"Token Mismatch! S={len(s.tokens)} G={len(g.tokens)} IX={i+1}"
238 n_sents += 1
239 # Lemmas ...
240 lemma_gld, lemma_pred, lemma_match, lemma_err, lemma_sym, mistakes = eval_lemma(s,g)
241 lemma_all_match += lemma_match
242 lemma_all_err += lemma_err
243 lemma_all_mistakes += mistakes
244 lemma_all_symbols += lemma_sym
245 lemma_all_pred += lemma_pred
246 lemma_all_gld += lemma_gld
247 # POS Tags ...
248 pos_gld, pos_pred, pos_match, pos_mistakes = eval_pos(s, g)
249 pos_all_pred += pos_pred
250 pos_all_gld += pos_gld
251 pos_all_match += pos_match
252 pos_all_err += len(pos_mistakes)
253 pos_all_mistakes += pos_mistakes
254
255 logger.info(f"A total of {n_sents} sentences were analyzed")
256
257 # Lemmas ...
258 logger.info(f"Lemma Matches = {lemma_all_match} || Errors = {lemma_all_err} || Symbol Chars = {len(lemma_all_symbols)}")
259 logger.info(f"Lemma Accuracy = {(lemma_all_match*100/(lemma_all_match + lemma_all_err)):.2f}%\n")
260 lemma_miss_df = pd.DataFrame(lemma_all_mistakes, columns =['Gold_Word', 'Gold_Lemma', 'Sys_Lemma']).value_counts()
261 lemma_miss_df.to_csv(path_or_buf=f"outputs/LemmaErrors.{args.corpus_name}.{args.type_sys}.tsv", sep="\t")
262 save_evaluated(Counter(lemma_all_pred), Counter(lemma_all_gld),
263 f"outputs/Lemma-Catalogue.{args.corpus_name}.{args.type_sys}.txt", print_gold=False)
264
265 # POS Tags ...
266 logger.info(f"POS Matches = {pos_all_match} || Errors = {pos_all_err}")
267 logger.info(f"POS Tagging Accuracy = {(pos_all_match*100/(pos_all_match + pos_all_err)):.2f}%\n")
268 pos_miss_df = pd.DataFrame(pos_all_mistakes, columns =['Gold_Word', 'Gold_POS', 'Sys_POS']).value_counts()
269 pos_miss_df.to_csv(path_or_buf=f"outputs/POS-Errors.{args.corpus_name}.{args.type_sys}.tsv", sep="\t")
270 save_evaluated(Counter(pos_all_pred), Counter(pos_all_gld), f"outputs/POS-Catalogue.{args.corpus_name}.{args.type_sys}.txt")
271
272 ordered_labels = sorted(set(pos_all_gld))
273 p_labels, r_labels, f_labels, support = eval_f1(y_true=pos_all_gld, y_pred=pos_all_pred, labels=ordered_labels , average=None)
274 scores_per_label = zip(ordered_labels, [x*100 for x in p_labels], [x*100 for x in r_labels], [x*100 for x in f_labels])
275 logger.info("\n\n")
276 logger.info(tabulate(scores_per_label, headers=["POS Tag","Precision", "Recall", "F1"], floatfmt=".2f"))
277 p_labels, r_labels, f_labels, support = eval_f1(y_true=np.array(pos_all_gld), y_pred=np.array(pos_all_pred), average='macro', zero_division=0)
278 logger.info(f"Total Prec = {p_labels*100}\tRec = {r_labels*100}\tF1 = {f_labels*100}")
279
280