Successfully evaluated several taggers
diff --git a/TIGER/evaluate.py b/TIGER/evaluate.py
new file mode 100644
index 0000000..0dffc20
--- /dev/null
+++ b/TIGER/evaluate.py
@@ -0,0 +1,137 @@
+from lib.CoNLL_Annotation import *
+from collections import Counter
+import pandas as pd
+import numpy as np
+from sklearn.metrics import precision_recall_fscore_support as eval_f1
+from tabulate import tabulate
+import logging, argparse, sys
+from datetime import datetime
+
+
+def eval_lemma(sys, gld):
+ match, err, symbol = 0, 0, []
+ mistakes = []
+ for i, gld_tok in enumerate(gld.tokens):
+ if gld_tok.lemma == sys.tokens[i].lemma:
+ match += 1
+ elif not sys.tokens[i].lemma.isalnum(): # This was added because Turku does not lemmatize symbols (it only copies them) => ERR ((',', '--', ','), 43642)
+ symbol.append(sys.tokens[i].lemma)
+ if sys.tokens[i].word == sys.tokens[i].lemma:
+ match += 1
+ else:
+ err += 1
+ else:
+ err += 1
+ mistakes.append((gld_tok.word, gld_tok.lemma, sys.tokens[i].lemma))
+ return match, err, symbol, mistakes
+
+
+def eval_pos(sys, gld):
+ match, mistakes = 0, []
+ y_gld, y_pred = [], []
+ for i, gld_tok in enumerate(gld.tokens):
+ y_gld.append(gld_tok.pos_tag)
+ y_pred.append(sys.tokens[i].pos_tag)
+ all_pos_labels.add(gld_tok.pos_tag)
+ if gld_tok.pos_tag == sys.tokens[i].pos_tag:
+ match += 1
+ else:
+ mistakes.append((gld_tok.word, gld_tok.pos_tag, sys.tokens[i].pos_tag))
+ return y_gld, y_pred, match, mistakes
+
+
+
+if __name__ == "__main__":
+ """
+ EVALUATIONS:
+ python TIGER/evaluate.py -t Turku\
+ --sys_file /home/daza/datasets/TIGER_conll/tiger_turku_parsed.conllu \
+ --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
+
+ python TIGER/evaluate.py -t SpaCy\
+ --sys_file /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu \
+ --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
+
+ python TIGER/evaluate.py -t RNNTagger\
+ --sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.RNNTagger.conll \
+ --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
+
+ python TIGER/evaluate.py -t TreeTagger\
+ --sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.TreeTagger.conll \
+ --gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
+ """
+
+ # =====================================================================================
+ # INPUT PARAMS
+ # =====================================================================================
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-s", "--sys_file", help="System output in CoNLL-U Format", required=True)
+ parser.add_argument("-g", "--gld_file", help="Gold Labels to evaluate in CoNLL-U Format", required=True)
+ parser.add_argument("-t", "--type_sys", help="Which system produced the outputs", default="system")
+ args = parser.parse_args()
+
+ # =====================================================================================
+ # LOGGING INFO ...
+ # =====================================================================================
+ logger = logging.getLogger(__name__)
+ console_hdlr = logging.StreamHandler(sys.stdout)
+ file_hdlr = logging.FileHandler(filename=f"logs/Eval_Tiger.{args.type_sys}.log")
+ logging.basicConfig(level=logging.INFO, handlers=[console_hdlr, file_hdlr])
+ now_is = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+ logger.info(f"\n\nEvaluating TIGER Corpus {now_is}")
+
+ # Read the Original TiGeR Annotations
+ gld_generator = read_conll_generator(args.gld_file, token_class=CoNLL09_Token)
+ # Read the Annotations Generated by the Automatic Parser [Turku, SpaCy, RNNTagger]
+ if args.type_sys == "RNNTagger":
+ sys_generator = read_conll_generator(args.sys_file, token_class=RNNTagger_Token)
+ elif args.type_sys == "TreeTagger":
+ sys_generator = read_conll_generator(args.sys_file, token_class=RNNTagger_Token, sent_sep="</S>")
+ else:
+ sys_generator = read_conll_generator(args.sys_file, token_class=CoNLLUP_Token)
+
+ lemma_all_match, lemma_all_err, lemma_all_mistakes = 0, 0, []
+ lemma_all_symbols = []
+ pos_all_match, pos_all_err, pos_all_mistakes = 0, 0, []
+ pos_all_pred, pos_all_gld = [], []
+ all_pos_labels = set()
+
+ for i, (s,g) in enumerate(zip(sys_generator, gld_generator)):
+ # print([x.word for x in s.tokens])
+ # print([x.word for x in g.tokens])
+ assert len(s.tokens) == len(g.tokens), f"Token Mismatch! S={len(s.tokens)} G={len(g.tokens)} IX={i+1}"
+ # Lemmas ...
+ lemma_match, lemma_err, lemma_sym, mistakes = eval_lemma(s,g)
+ lemma_all_match += lemma_match
+ lemma_all_err += lemma_err
+ lemma_all_mistakes += mistakes
+ lemma_all_symbols += lemma_sym
+ # POS Tags ...
+ pos_gld, pos_pred, pos_match, pos_mistakes = eval_pos(s, g)
+ pos_all_pred += pos_pred
+ pos_all_gld += pos_gld
+ pos_all_match += pos_match
+ pos_all_err += len(pos_mistakes)
+ pos_all_mistakes += pos_mistakes
+
+ # Lemmas ...
+ logger.info(f"Lemma Matches = {lemma_all_match} || Errors = {lemma_all_err} || Symbol Chars = {len(lemma_all_symbols)}")
+ logger.info(f"Lemma Accuracy = {(lemma_all_match*100/(lemma_all_match + lemma_all_err)):.2f}%\n")
+ lemma_miss_df = pd.DataFrame(lemma_all_mistakes, columns =['Gold_Word', 'Gold_Lemma', 'Sys_Lemma']).value_counts()
+ lemma_miss_df.to_csv(path_or_buf=f"outputs/LemmaErrors.{args.type_sys}.tsv", sep="\t")
+
+ # POS Tags ...
+ logger.info(f"POS Matches = {pos_all_match} || Errors = {pos_all_err}")
+ logger.info(f"POS Tagging Accuracy = {(pos_all_match*100/(pos_all_match + pos_all_err)):.2f}%\n")
+ pos_miss_df = pd.DataFrame(pos_all_mistakes, columns =['Gold_Word', 'Gold_POS', 'Sys_POS']).value_counts()
+ pos_miss_df.to_csv(path_or_buf=f"outputs/POS-Errors.{args.type_sys}.tsv", sep="\t")
+
+ ordered_labels = sorted(all_pos_labels)
+ p_labels, r_labels, f_labels, support = eval_f1(y_true=pos_all_gld, y_pred=pos_all_pred, labels=ordered_labels , average=None)
+ scores_per_label = zip(ordered_labels, [x*100 for x in p_labels], [x*100 for x in r_labels], [x*100 for x in f_labels])
+ logger.info("\n\n")
+ logger.info(tabulate(scores_per_label, headers=["POS Tag","Precision", "Recall", "F1"], floatfmt=".2f"))
+ p_labels, r_labels, f_labels, support = eval_f1(y_true=np.array(pos_all_gld), y_pred=np.array(pos_all_pred), average='micro', zero_division=0)
+ logger.info(f"Total Prec = {p_labels*100}\tRec = {r_labels*100}\tF1 = {f_labels*100}")
+
+
\ No newline at end of file
diff --git a/TIGER/parse_spacy.py b/TIGER/parse_spacy.py
new file mode 100644
index 0000000..de82917
--- /dev/null
+++ b/TIGER/parse_spacy.py
@@ -0,0 +1,87 @@
+import spacy
+from spacy.tokens import Doc
+import logging, sys, time
+from lib.CoNLL_Annotation import CoNLL09_Token
+import my_utils.file_utils as fu
+from germalemma import GermaLemma
+
+
+TIGER_CORPUS = "/home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09"
+OUTPUT_FILE = "/home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu"
+TEXT_OUTPUT = "/home/daza/datasets/TIGER_conll/tiger_all.txt"
+
+
+class WhitespaceTokenizer(object):
+ def __init__(self, vocab):
+ self.vocab = vocab
+
+ def __call__(self, text):
+ words = text.split(' ')
+ # All tokens 'own' a subsequent space character in this tokenizer
+ spaces = [True] * len(words)
+ return Doc(self.vocab, words=words, spaces=spaces)
+
+
+def get_conll_str(spacy_doc):
+ conll_lines = [] # We want: [ID, FORM, LEMMA, UPOS, XPOS, FEATS, HEAD, DEPREL, DEPS, MISC]
+ for ix, token in enumerate(spacy_doc):
+ content = (str(ix), token.text, find_germalemma(token.text, token.tag_, token.lemma_), token.pos_, token.tag_, "_", "_", "_", "_", "_")
+ conll_lines.append("\t".join(content))
+ return "\n".join(conll_lines)
+
+
+# def freeling_lemma_lookup():
+# dicts_path = "/home/daza/Frameworks/FreeLing/data/de/dictionary/entries/"
+
+def find_germalemma(word, pos, spacy_lemma):
+ simplify_pos = {"ADJA":"ADJ", "ADJD":"ADJ",
+ "NA":"N", "NE":"N", "NN":"N",
+ "ADV":"ADV", "PAV":"ADV", "PROAV":"ADV", "PAVREL":"ADV", "PWAV":"ADV", "PWAVREL":"ADV",
+ "VAFIN":"V", "VAIMP":"V", "VAINF":"V", "VAPP":"V", "VMFIN":"V", "VMINF":"V",
+ "VMPP":"V", "VVFIN":"V", "VVIMP":"V", "VVINF":"V", "VVIZU":"V","VVPP":"V"
+ }
+ # simplify_pos = {"VERB": "V", "ADV": "ADV", "ADJ": "ADJ", "NOUN":"N", "PROPN": "N"}
+ try:
+ return lemmatizer.find_lemma(word, simplify_pos.get(pos, "UNK"))
+ except:
+ return spacy_lemma
+
+
+if __name__ == "__main__":
+ file_has_next, chunk_ix = True, 0
+ CHUNK_SIZE = 10000
+
+ # =====================================================================================
+ # LOGGING INFO ...
+ # =====================================================================================
+ logger = logging.getLogger(__name__)
+ console_hdlr = logging.StreamHandler(sys.stdout)
+ file_hdlr = logging.FileHandler(filename=f"logs/Parse_Tiger.SpaCy.log")
+ logging.basicConfig(level=logging.INFO, handlers=[console_hdlr, file_hdlr])
+ logger.info(f"Chunking TIGER Corpus in chunks of {CHUNK_SIZE} Sentences")
+
+ # =====================================================================================
+ # POS TAG DOCUMENTS
+ # =====================================================================================
+ spacy_de = spacy.load("de_core_news_lg", disable=["ner", "parser"])
+ spacy_de.tokenizer = WhitespaceTokenizer(spacy_de.vocab) # We won't re-tokenize to respect how the source CoNLL are tokenized!
+ write_out = open(OUTPUT_FILE, "w")
+ lemmatizer = GermaLemma()
+ write_plain = open(TEXT_OUTPUT, "w")
+
+ start = time.time()
+ total_processed_sents = 0
+ line_generator = fu.file_generator(TIGER_CORPUS)
+ while file_has_next:
+ sents, gld, file_has_next = fu.get_file_text_chunk(line_generator, chunk_size=CHUNK_SIZE, token_class=CoNLL09_Token)
+ total_processed_sents += len(sents)
+ logger.info(f"Already processed {total_processed_sents} sentences...")
+ for doc in spacy_de.pipe(sents, batch_size=1000, n_process=10):
+ conll_str = get_conll_str(doc)
+ write_out.write(conll_str)
+ write_out.write("\n\n")
+ write_plain.write(" ".join([x.text for x in doc])+"\n") # OPTIONAL: This can be commented for efficiency...
+
+ end = time.time()
+ logger.info(f"Processing File {TIGER_CORPUS} took {(end - start)} seconds!")
+
\ No newline at end of file
diff --git a/TIGER/tiger_parse_turku.py b/TIGER/parse_turku.py
similarity index 79%
rename from TIGER/tiger_parse_turku.py
rename to TIGER/parse_turku.py
index 6ccb64a..140ea0c 100644
--- a/TIGER/tiger_parse_turku.py
+++ b/TIGER/parse_turku.py
@@ -2,8 +2,8 @@
import subprocess, json, time
import requests, glob, logging
import os.path, sys
-from CoNLL_Annotation import CoNLL09_Token
-from my_utils import *
+from lib.CoNLL_Annotation import CoNLL09_Token
+import my_utils.file_utils as fu
TIGER_CORPUS = "/home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09"
@@ -18,7 +18,7 @@
# =====================================================================================
logger = logging.getLogger(__name__)
console_hdlr = logging.StreamHandler(sys.stdout)
- file_hdlr = logging.FileHandler(filename=f"ParseTests.log")
+ file_hdlr = logging.FileHandler(filename=f"logs/Parse_Tiger_Turku.log")
logging.basicConfig(level=logging.INFO, handlers=[console_hdlr, file_hdlr])
logger.info(f"Chunking TIGER Corpus in chunks of {CHUNK_SIZE} Sentences")
@@ -27,12 +27,12 @@
# =====================================================================================
start = time.time()
total_processed_sents = 0
- line_generator = file_generator(TIGER_CORPUS)
+ line_generator = fu.file_generator(TIGER_CORPUS)
while file_has_next:
- raw_text, file_has_next, n_sents = get_file_chunk(line_generator, chunk_size=CHUNK_SIZE, token_class=CoNLL09_Token)
+ raw_text, file_has_next, n_sents = fu.get_file_chunk(line_generator, chunk_size=CHUNK_SIZE, token_class=CoNLL09_Token)
total_processed_sents += n_sents
if len(raw_text) > 0:
- turku_parse_file(raw_text, TIGER_CORPUS, chunk_ix)
+ fu.turku_parse_file(raw_text, TIGER_CORPUS, chunk_ix)
now = time.time()
elapsed = (now - start)
logger.info(f"Time Elapsed: {elapsed}. Processed {total_processed_sents}. [{total_processed_sents/elapsed} Sents/sec]\n") # Toks/Sec???
diff --git a/TIGER/tiger_evaluate.py b/TIGER/tiger_evaluate.py
deleted file mode 100644
index e89e277..0000000
--- a/TIGER/tiger_evaluate.py
+++ /dev/null
@@ -1,96 +0,0 @@
-from CoNLL_Annotation import *
-from collections import Counter
-import pandas as pd
-import numpy as np
-from sklearn.metrics import precision_recall_fscore_support as eval_f1
-from tabulate import tabulate
-
-
-def eval_lemma(sys, gld):
- match, err, symbol = 0, 0, []
- mistakes = []
- for i, gld_tok in enumerate(gld.tokens):
- if gld_tok.lemma == sys.tokens[i].lemma:
- match += 1
- elif not sys.tokens[i].lemma.isalnum(): # This was added because Turku does not lemmatize symbols (it only copies them) => ERR ((',', '--', ','), 43642)
- symbol.append(sys.tokens[i].lemma)
- if sys.tokens[i].word == sys.tokens[i].lemma:
- match += 1
- else:
- err += 1
- else:
- err += 1
- mistakes.append((gld_tok.word, gld_tok.lemma, sys.tokens[i].lemma))
- return match, err, symbol, mistakes
-
-
-def eval_pos(sys, gld):
- match, mistakes = 0, []
- y_gld, y_pred = [], []
- for i, gld_tok in enumerate(gld.tokens):
- y_gld.append(gld_tok.pos_tag)
- y_pred.append(sys.tokens[i].pos_tag)
- all_pos_labels.add(gld_tok.pos_tag)
- all_pos_labels.add(sys.tokens[i].pos_tag)
- if gld_tok.pos_tag == sys.tokens[i].pos_tag:
- match += 1
- else:
- mistakes.append((gld_tok.word, gld_tok.pos_tag, sys.tokens[i].pos_tag))
- return y_gld, y_pred, match, mistakes
-
-
-
-if __name__ == "__main__":
-
- # Read the Original TiGeR Annotations
- gld_filename = "/home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09"
- gld_generator = read_conll_generator(gld_filename, token_class=CoNLL09_Token)
-
- # Read the Annotations Generated by the Automatic Parser [Turku]
- sys_filename = "/home/daza/datasets/TIGER_conll/tiger_turku_parsed.conllu"
- sys_generator = read_conll_generator(sys_filename, token_class=CoNLLUP_Token)
-
- lemma_all_match, lemma_all_err, lemma_all_mistakes = 0, 0, []
- lemma_all_symbols = []
- pos_all_match, pos_all_err, pos_all_mistakes = 0, 0, []
- pos_all_pred, pos_all_gld = [], []
- all_pos_labels = set()
-
- for i, (s,g) in enumerate(zip(sys_generator, gld_generator)):
- assert len(s.tokens) == len(g.tokens), "Token Mismatch!"
- # Lemmas ...
- lemma_match, lemma_err, lemma_sym, mistakes = eval_lemma(s,g)
- lemma_all_match += lemma_match
- lemma_all_err += lemma_err
- lemma_all_mistakes += mistakes
- lemma_all_symbols += lemma_sym
- # POS Tags ...
- pos_gld, pos_pred, pos_match, pos_mistakes = eval_pos(s, g)
- pos_all_pred += pos_pred
- pos_all_gld += pos_gld
- pos_all_match += pos_match
- pos_all_err += len(pos_mistakes)
- pos_all_mistakes += pos_mistakes
-
- # Lemmas ...
- print(f"Lemma Matches = {lemma_all_match} || Errors = {lemma_all_err} || Symbol Chars = {len(lemma_all_symbols)}")
- print(f"Lemma Accuracy = {(lemma_all_match*100/(lemma_all_match + lemma_all_err)):.2f}%\n")
- lemma_miss_df = pd.DataFrame(lemma_all_mistakes, columns =['Gold_Word', 'Gold_Lemma', 'Sys_Lemma']).value_counts()
- lemma_miss_df.to_csv(path_or_buf="LemmaErrors.tsv", sep="\t")
-
- # POS Tags ...
- print(f"POS Matches = {pos_all_match} || Errors = {pos_all_err}")
- print(f"POS Tagging Accuracy = {(pos_all_match*100/(pos_all_match + pos_all_err)):.2f}%\n")
- pos_miss_df = pd.DataFrame(pos_all_mistakes, columns =['Gold_Word', 'Gold_POS', 'Sys_POS']).value_counts()
- pos_miss_df.to_csv(path_or_buf="POS-Errors.tsv", sep="\t")
-
- ordered_labels = sorted(all_pos_labels)
- p_labels, r_labels, f_labels, support = eval_f1(y_true=pos_all_gld, y_pred=pos_all_pred, labels=ordered_labels , average=None)
- scores_per_label = zip(ordered_labels, [x*100 for x in p_labels], [x*100 for x in r_labels], [x*100 for x in f_labels])
- print("\n\n")
- print(tabulate(scores_per_label, headers=["POS Tag","Precision", "Recall", "F1"], floatfmt=".2f"))
- print("\n Total Prec, Rec, and F1 Score: ")
- p_labels, r_labels, f_labels, support = eval_f1(y_true=np.array(pos_all_gld), y_pred=np.array(pos_all_pred), average='micro', zero_division=0)
- print(p_labels*100, r_labels*100, f_labels*100)
-
-
\ No newline at end of file
diff --git a/TIGER/tiger_parse_spacy.py b/TIGER/tiger_parse_spacy.py
deleted file mode 100644
index e69de29..0000000
--- a/TIGER/tiger_parse_spacy.py
+++ /dev/null