Stable tested version
diff --git a/systems/Run_Tree-RNN_Taggers.txt b/systems/Run_Tree-RNN_Taggers.txt
index 98a9dd2..fe8be10 100644
--- a/systems/Run_Tree-RNN_Taggers.txt
+++ b/systems/Run_Tree-RNN_Taggers.txt
@@ -14,4 +14,4 @@
# RNN Tagger:
time cmd/rnn-tagger-german-notokenize.sh /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu.tok > /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.RNNtagger.parsed.conll
-time cmd/rnn-tagger-german-notokenize.sh /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll.tok > /home/daza/datasets/TIGER_conll/sys_outputs/Tiger.NewOrth.test.RNNTagger.conll
+time cmd/rnn-tagger-german-notokenize.sh /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll.tok > /home/daza/datasets/TIGER_conll/sys_outputs/Tiger.NewOrth.test.RNNTagger.conll
\ No newline at end of file
diff --git a/systems/eval_old_vs_new_tiger.py b/systems/eval_old_vs_new_tiger.py
deleted file mode 100644
index 38dc597..0000000
--- a/systems/eval_old_vs_new_tiger.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import my_utils.file_utils as fu
-from lib.CoNLL_Annotation import read_conll, CoNLLUP_Token
-from collections import Counter
-from germalemma import GermaLemma
-
-SPACY_NEW = "/home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.spacy_parsed.conllu"
-CASES = "/home/daza/datasets/TIGER_conll/NewOrthProblems_Indices.train.txt"
-
-orth_dict = fu.file_to_dict("/vol/netapp/daza/datasets/TIGER_conll/TigerOrthMapping.train.json")
-new_to_old = {v:k for k,v in orth_dict.items()}
-
-
-if __name__ == "__main__":
- line_generator = fu.file_generator(SPACY_NEW)
- conll_sents, _ = read_conll(line_generator, chunk_size=60000, token_class=CoNLLUP_Token, comment_str="#")
- special_cases = [int(line) for line in open(CASES).read().splitlines()]
- checked_cases = []
-
- lemmatizer = GermaLemma()
-
- for ix, sent in enumerate(conll_sents):
- if ix in special_cases:
- for tok in sent.tokens:
- old_word_change = new_to_old.get(tok.word)
- if old_word_change:
- try:
- old_lemma = lemmatizer.find_lemma(old_word_change, tok.pos_tag)
- except:
- old_lemma = f"UNK_{tok.pos_tag}"
- checked_cases.append((old_word_change, tok.word, old_lemma, tok.lemma))
-
- print(f"Cases checked: {len(checked_cases)}")
- case_count = Counter(checked_cases).most_common()
- fu.counter_to_file(case_count, "/home/daza/datasets/TIGER_conll/TigerLemmas_Old_New.tsv")
-
\ No newline at end of file
diff --git a/systems/evaluate.py b/systems/evaluate.py
deleted file mode 100644
index 839cd37..0000000
--- a/systems/evaluate.py
+++ /dev/null
@@ -1,231 +0,0 @@
-from lib.CoNLL_Annotation import *
-from collections import Counter, defaultdict
-import pandas as pd
-import numpy as np
-from sklearn.metrics import precision_recall_fscore_support as eval_f1
-from tabulate import tabulate
-import logging, argparse, sys
-from datetime import datetime
-
-
-tree_tagger_fixes = {
- "die": "der",
- "eine": "ein",
- "dass": "daß",
- "keine": "kein",
- "dies": "dieser",
- "erst": "erster",
- "andere": "anderer",
- "alle": "aller",
- "Sie": "sie",
- "wir": "uns",
- "alle": "aller",
- "wenige": "wenig"
-}
-
-
-def save_evaluated(all_sys, all_gld, out_path, print_gold=True):
- with open(out_path, "w") as out:
- if print_gold:
- out.write(f"ORIGINAL_CORPUS_TAGS\n\nTAG\tGLD_COUNT\tSYS_COUNT\n")
- for g_tag,g_count in sorted(all_gld.items()):
- s_count = all_sys.get(g_tag, 0)
- out.write(f"{g_tag}\t{g_count}\t{s_count}\n")
-
- out.write("\n\nSYSTEM_ONLY_TAGS\n\nTAG\tG_COUNT\tSYS_COUNT\n")
- for s_tag,s_count in sorted(all_sys.items()):
- g_count = all_gld.get(s_tag, 0)
- if g_count == 0:
- out.write(f"{s_tag}\t{g_count}\t{s_count}\n")
-
-
-
-def eval_lemma(sys, gld):
- match, err, symbol = 0, 0, []
- y_gld, y_pred, mistakes = [], [], []
- for i, gld_tok in enumerate(gld.tokens):
- sys_lemma = sys.tokens[i].lemma
- # sys_lemma = tree_tagger_fixes.get(sys.tokens[i].lemma, sys.tokens[i].lemma) # Omit TreeTagger "errors" because of article lemma disagreement
- y_gld.append(gld_tok.pos_tag)
- y_pred.append(sys_lemma)
- if gld_tok.lemma == sys_lemma:
- match += 1
- elif not sys.tokens[i].lemma.isalnum(): # Turku does not lemmatize symbols (it only copies them) => ERR ((',', '--', ','), 43642)
- symbol.append(sys.tokens[i].lemma)
- if sys.tokens[i].word == sys.tokens[i].lemma:
- match += 1
- else:
- err += 1
- else:
- err += 1
- mistakes.append((gld_tok.word, gld_tok.lemma, sys.tokens[i].lemma))
- return y_gld, y_pred, match, err, symbol, mistakes
-
-
-def eval_pos(sys, gld):
- match, mistakes = 0, []
- y_gld, y_pred = [], []
- for i, gld_tok in enumerate(gld.tokens):
- y_gld.append(gld_tok.pos_tag)
- y_pred.append(sys.tokens[i].pos_tag)
- # pos_all_pred[gld_tok.pos_tag] += 1
- # pos_all_gold[sys.tokens[i].pos_tag] += 1
- if gld_tok.pos_tag == sys.tokens[i].pos_tag:
- match += 1
- elif gld_tok.pos_tag == "$." and sys.tokens[i].pos_tag == "$":
- match += 1
- y_pred = y_pred[:-1] + ["$."]
- else:
- mistakes.append((gld_tok.word, gld_tok.pos_tag, sys.tokens[i].pos_tag))
- return y_gld, y_pred, match, mistakes
-
-
-
-if __name__ == "__main__":
- """
- EVALUATIONS:
-
- ********** TIGER CORPUS ALL ************
-
- python systems/evaluate.py -t Turku --corpus_name Tiger --gld_token_type CoNLL09_Token \
- --sys_file /home/daza/datasets/TIGER_conll/tiger_turku_parsed.conllu \
- --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
-
- python systems/evaluate.py -t SpaCy --corpus_name Tiger --gld_token_type CoNLL09_Token \
- --sys_file /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu \
- --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
-
- python systems/evaluate.py -t RNNTagger --corpus_name Tiger --gld_token_type CoNLL09_Token \
- --sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.RNNTagger.conll \
- --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
-
- python systems/evaluate.py -t TreeTagger --corpus_name Tiger --gld_token_type CoNLL09_Token \
- --sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.TreeTagger.conll \
- --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
-
-
- ********** TIGER CORPUS TEST ************
-
- python systems/evaluate.py -t SpaCy --corpus_name TigerTestOld \
- --sys_file /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.test.conllu \
- --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.OldOrth.test.conll
-
- python systems/evaluate.py -t SpaCy --corpus_name TigerTestNew \
- --sys_file /home/daza/datasets/TIGER_conll/Tiger.NewOrth.test.spacy_parsed.conllu\
- --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll
-
-
- python systems/evaluate.py -t Turku --corpus_name TigerTestNew \
- --sys_file /home/daza/datasets/TIGER_conll/sys_outputs/Tiger.NewOrth.test.turku_parsed.conllu \
- --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll
-
- ********** UNIVERSAL DEPENDENCIES TEST-SET ************
-
- python systems/evaluate.py -t Turku --corpus_name DE_GSD \
- --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu.parsed.0.conllu \
- --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
-
- python systems/evaluate.py -t SpaCyGL --corpus_name DE_GSD \
- --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.germalemma.conllu \
- --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
-
- python systems/evaluate.py -t SpaCy --corpus_name DE_GSD \
- --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.conllu \
- --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
-
- python systems/evaluate.py -t RNNTagger --corpus_name DE_GSD \
- --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.RNNtagger.parsed.conll \
- --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
-
- python systems/evaluate.py -t TreeTagger --corpus_name DE_GSD \
- --sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.treetagger.parsed.conll \
- --gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
-
- """
-
- # =====================================================================================
- # INPUT PARAMS
- # =====================================================================================
- parser = argparse.ArgumentParser()
- parser.add_argument("-s", "--sys_file", help="System output in CoNLL-U Format", required=True)
- parser.add_argument("-g", "--gld_file", help="Gold Labels to evaluate in CoNLL-U Format", required=True)
- parser.add_argument("-c", "--corpus_name", help="Corpus Name for Gold Labels", required=True)
- parser.add_argument("-t", "--type_sys", help="Which system produced the outputs", default="system")
- parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLLUP_Token")
- parser.add_argument("-cs", "--comment_str", help="CoNLL Format of comentaries inside the file", default="#")
- args = parser.parse_args()
-
- # =====================================================================================
- # LOGGING INFO ...
- # =====================================================================================
- logger = logging.getLogger(__name__)
- console_hdlr = logging.StreamHandler(sys.stdout)
- file_hdlr = logging.FileHandler(filename=f"logs/Eval_{args.corpus_name}.{args.type_sys}.log")
- logging.basicConfig(level=logging.INFO, handlers=[console_hdlr, file_hdlr])
- now_is = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
- logger.info(f"\n\nEvaluating {args.corpus_name} Corpus {now_is}")
-
- # Read the Original GOLD Annotations [CoNLL09, CoNLLUP]
- gld_generator = read_conll_generator(args.gld_file, token_class=get_token_type(args.gld_token_type), comment_str=args.comment_str)
- # Read the Annotations Generated by the Automatic Parser [Turku, SpaCy, RNNTagger]
- if args.type_sys == "RNNTagger":
- sys_generator = read_conll_generator(args.sys_file, token_class=RNNTagger_Token, comment_str="#")
- elif args.type_sys == "TreeTagger":
- sys_generator = read_conll_generator(args.sys_file, token_class=RNNTagger_Token, sent_sep="</S>", comment_str="#")
- else:
- sys_generator = read_conll_generator(args.sys_file, token_class=CoNLLUP_Token, comment_str="#")
-
- lemma_all_match, lemma_all_err, lemma_all_mistakes = 0, 0, []
- lemma_all_symbols, sys_only_lemmas = [], []
- pos_all_match, pos_all_err, pos_all_mistakes = 0, 0, []
- pos_all_pred, pos_all_gld = [], []
- lemma_all_pred, lemma_all_gld = [], []
- n_sents = 0
-
- for i, (s,g) in enumerate(zip(sys_generator, gld_generator)):
- # print([x.word for x in s.tokens])
- # print([x.word for x in g.tokens])
- assert len(s.tokens) == len(g.tokens), f"Token Mismatch! S={len(s.tokens)} G={len(g.tokens)} IX={i+1}"
- n_sents += 1
- # Lemmas ...
- lemma_gld, lemma_pred, lemma_match, lemma_err, lemma_sym, mistakes = eval_lemma(s,g)
- lemma_all_match += lemma_match
- lemma_all_err += lemma_err
- lemma_all_mistakes += mistakes
- lemma_all_symbols += lemma_sym
- lemma_all_pred += lemma_pred
- lemma_all_gld += lemma_gld
- # POS Tags ...
- pos_gld, pos_pred, pos_match, pos_mistakes = eval_pos(s, g)
- pos_all_pred += pos_pred
- pos_all_gld += pos_gld
- pos_all_match += pos_match
- pos_all_err += len(pos_mistakes)
- pos_all_mistakes += pos_mistakes
-
- logger.info(f"A total of {n_sents} sentences were analyzed")
-
- # Lemmas ...
- logger.info(f"Lemma Matches = {lemma_all_match} || Errors = {lemma_all_err} || Symbol Chars = {len(lemma_all_symbols)}")
- logger.info(f"Lemma Accuracy = {(lemma_all_match*100/(lemma_all_match + lemma_all_err)):.2f}%\n")
- lemma_miss_df = pd.DataFrame(lemma_all_mistakes, columns =['Gold_Word', 'Gold_Lemma', 'Sys_Lemma']).value_counts()
- lemma_miss_df.to_csv(path_or_buf=f"outputs/LemmaErrors.{args.corpus_name}.{args.type_sys}.tsv", sep="\t")
- save_evaluated(Counter(lemma_all_pred), Counter(lemma_all_gld),
- f"outputs/Lemma-Catalogue.{args.corpus_name}.{args.type_sys}.txt", print_gold=False)
-
- # POS Tags ...
- logger.info(f"POS Matches = {pos_all_match} || Errors = {pos_all_err}")
- logger.info(f"POS Tagging Accuracy = {(pos_all_match*100/(pos_all_match + pos_all_err)):.2f}%\n")
- pos_miss_df = pd.DataFrame(pos_all_mistakes, columns =['Gold_Word', 'Gold_POS', 'Sys_POS']).value_counts()
- pos_miss_df.to_csv(path_or_buf=f"outputs/POS-Errors.{args.corpus_name}.{args.type_sys}.tsv", sep="\t")
- save_evaluated(Counter(pos_all_pred), Counter(pos_all_gld), f"outputs/POS-Catalogue.{args.corpus_name}.{args.type_sys}.txt")
-
- ordered_labels = sorted(set(pos_all_gld))
- p_labels, r_labels, f_labels, support = eval_f1(y_true=pos_all_gld, y_pred=pos_all_pred, labels=ordered_labels , average=None)
- scores_per_label = zip(ordered_labels, [x*100 for x in p_labels], [x*100 for x in r_labels], [x*100 for x in f_labels])
- logger.info("\n\n")
- logger.info(tabulate(scores_per_label, headers=["POS Tag","Precision", "Recall", "F1"], floatfmt=".2f"))
- p_labels, r_labels, f_labels, support = eval_f1(y_true=np.array(pos_all_gld), y_pred=np.array(pos_all_pred), average='macro', zero_division=0)
- logger.info(f"Total Prec = {p_labels*100}\tRec = {r_labels*100}\tF1 = {f_labels*100}")
-
-
\ No newline at end of file
diff --git a/systems/parse_spacy.py b/systems/parse_spacy.py
index 3c56233..90ca237 100644
--- a/systems/parse_spacy.py
+++ b/systems/parse_spacy.py
@@ -29,9 +29,6 @@
conll_lines.append("\t".join(content))
return "\n".join(conll_lines)
-
-# def freeling_lemma_lookup():
-# dicts_path = "/home/daza/Frameworks/FreeLing/data/de/dictionary/entries/"
def find_germalemma(word, pos, spacy_lemma):
simplify_pos = {"ADJA":"ADJ", "ADJD":"ADJ",
@@ -81,7 +78,6 @@
time python systems/parse_spacy.py --corpus_name DeReKo_a00 --comment_str "#" \
-i /export/netapp/kupietz/N-GRAMM-STUDIE/conllu/a00.conllu.gz \
-o /export/netapp/kupietz/N-GRAMM-STUDIE/conllu/0_SpaCyParsed/a00.spacy.gl.conllu
-
"""
parser = argparse.ArgumentParser()
diff --git a/systems/parse_turku.py b/systems/parse_turku.py
index 284dbd4..668a48c 100644
--- a/systems/parse_turku.py
+++ b/systems/parse_turku.py
@@ -6,6 +6,7 @@
import my_utils.file_utils as fu
import argparse
+
if __name__ == "__main__":
"""
@@ -21,6 +22,7 @@
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input_file", help="Input Corpus", required=True)
+ parser.add_argument("-o", "--output_file", help="Output Parsed Corpus", default=None)
parser.add_argument("-n", "--corpus_name", help="Corpus Name", default="Corpus")
parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLL09_Token")
parser.add_argument("-c", "--comment_str", help="CoNLL Format of comentaries inside the file", default="#")
@@ -29,6 +31,8 @@
file_has_next, chunk_ix = True, 0
CHUNK_SIZE = 10000
+ output_file = args.input_file if not args.output_file else args.output_file
+
# =====================================================================================
# LOGGING INFO ...
# =====================================================================================
@@ -48,7 +52,7 @@
raw_text, file_has_next, n_sents = fu.get_file_chunk(line_generator, chunk_size=CHUNK_SIZE, token_class=get_token_type(args.gld_token_type), comment_str=args.comment_str)
total_processed_sents += n_sents
if len(raw_text) > 0:
- fu.turku_parse_file(raw_text, args.input_file, chunk_ix)
+ fu.turku_parse_file(raw_text, output_file, chunk_ix)
now = time.time()
elapsed = (now - start)
logger.info(f"Time Elapsed: {elapsed}. Processed {total_processed_sents}. [{total_processed_sents/elapsed} Sents/sec]\n") # Toks/Sec???