upgrading repo to latest version
diff --git a/systems/Run_Tree-RNN_Taggers.txt b/systems/Run_Tree-RNN_Taggers.txt
index e0ea45f..98a9dd2 100644
--- a/systems/Run_Tree-RNN_Taggers.txt
+++ b/systems/Run_Tree-RNN_Taggers.txt
@@ -1,7 +1,17 @@
+# To Preprocess Files (conllu ---> .sep.tok ^ .tok)
+
+python my_utils/conll_to_tok.py \
+-s /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu \
+-ss "</S>" \
+--token_type CoNLLUP_Token
# TreeTagger:
-time cmd/tree-tagger-german-notokenize /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu.sep.tok /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.treetagger.parsed.conll
+time cmd/tree-tagger-german-notokenize /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu.sep.tok > /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.treetagger.parsed.conll
+
+time cmd/tree-tagger-german-notokenize /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.OldOrth.test.conll.sep.tok > /home/daza/datasets/TIGER_conll/sys_outputs/Tiger.OldOrth.test.TreeTagger.conll
# RNN Tagger:
-time cmd/rnn-tagger-german-notokenize.sh /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu.tok > /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.RNNtagger.parsed.conll
\ No newline at end of file
+time cmd/rnn-tagger-german-notokenize.sh /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu.tok > /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.RNNtagger.parsed.conll
+
+time cmd/rnn-tagger-german-notokenize.sh /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll.tok > /home/daza/datasets/TIGER_conll/sys_outputs/Tiger.NewOrth.test.RNNTagger.conll
diff --git a/systems/evaluate.py b/systems/evaluate.py
index 11ffd51..839cd37 100644
--- a/systems/evaluate.py
+++ b/systems/evaluate.py
@@ -44,8 +44,8 @@
match, err, symbol = 0, 0, []
y_gld, y_pred, mistakes = [], [], []
for i, gld_tok in enumerate(gld.tokens):
- # sys_lemma = tree_tagger_fixes.get(sys.tokens[i].lemma, sys.tokens[i].lemma) # Omit TreeTagger "errors" because of article lemma disagreement
sys_lemma = sys.tokens[i].lemma
+ # sys_lemma = tree_tagger_fixes.get(sys.tokens[i].lemma, sys.tokens[i].lemma) # Omit TreeTagger "errors" because of article lemma disagreement
y_gld.append(gld_tok.pos_tag)
y_pred.append(sys_lemma)
if gld_tok.lemma == sys_lemma:
@@ -87,41 +87,57 @@
********** TIGER CORPUS ALL ************
- python systems/evaluate.py -t Turku --corpus_name Tiger\
+ python systems/evaluate.py -t Turku --corpus_name Tiger --gld_token_type CoNLL09_Token \
--sys_file /home/daza/datasets/TIGER_conll/tiger_turku_parsed.conllu \
--gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
- python systems/evaluate.py -t SpaCy --corpus_name Tiger\
+ python systems/evaluate.py -t SpaCy --corpus_name Tiger --gld_token_type CoNLL09_Token \
--sys_file /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu \
--gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
- python systems/evaluate.py -t RNNTagger --corpus_name Tiger\
+ python systems/evaluate.py -t RNNTagger --corpus_name Tiger --gld_token_type CoNLL09_Token \
--sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.RNNTagger.conll \
--gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
- python systems/evaluate.py -t TreeTagger --corpus_name Tiger\
+ python systems/evaluate.py -t TreeTagger --corpus_name Tiger --gld_token_type CoNLL09_Token \
--sys_file /home/daza/datasets/TIGER_conll/tiger_all.parsed.TreeTagger.conll \
--gld_file /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
+
+ ********** TIGER CORPUS TEST ************
+
+ python systems/evaluate.py -t SpaCy --corpus_name TigerTestOld \
+ --sys_file /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.test.conllu \
+ --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.OldOrth.test.conll
+
+ python systems/evaluate.py -t SpaCy --corpus_name TigerTestNew \
+ --sys_file /home/daza/datasets/TIGER_conll/Tiger.NewOrth.test.spacy_parsed.conllu\
+ --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll
+
+
+ python systems/evaluate.py -t Turku --corpus_name TigerTestNew \
+ --sys_file /home/daza/datasets/TIGER_conll/sys_outputs/Tiger.NewOrth.test.turku_parsed.conllu \
+ --gld_file /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll
+
********** UNIVERSAL DEPENDENCIES TEST-SET ************
- python systems/evaluate.py -t Turku --gld_token_type CoNLLUP_Token --corpus_name DE_GSD\
+ python systems/evaluate.py -t Turku --corpus_name DE_GSD \
--sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu.parsed.0.conllu \
--gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
- python systems/evaluate.py -t SpaCyGL --gld_token_type CoNLLUP_Token --corpus_name DE_GSD\
+ python systems/evaluate.py -t SpaCyGL --corpus_name DE_GSD \
--sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.germalemma.conllu \
--gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
- python systems/evaluate.py -t SpaCy --gld_token_type CoNLLUP_Token --corpus_name DE_GSD\
+ python systems/evaluate.py -t SpaCy --corpus_name DE_GSD \
--sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.conllu \
--gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
- python systems/evaluate.py -t RNNTagger --gld_token_type CoNLLUP_Token --corpus_name DE_GSD\
+ python systems/evaluate.py -t RNNTagger --corpus_name DE_GSD \
--sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.RNNtagger.parsed.conll \
--gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
- python systems/evaluate.py -t TreeTagger --gld_token_type CoNLLUP_Token --corpus_name DE_GSD\
+ python systems/evaluate.py -t TreeTagger --corpus_name DE_GSD \
--sys_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.treetagger.parsed.conll \
--gld_file /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
@@ -133,9 +149,9 @@
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--sys_file", help="System output in CoNLL-U Format", required=True)
parser.add_argument("-g", "--gld_file", help="Gold Labels to evaluate in CoNLL-U Format", required=True)
- parser.add_argument("-t", "--type_sys", help="Which system produced the outputs", default="system")
parser.add_argument("-c", "--corpus_name", help="Corpus Name for Gold Labels", required=True)
- parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLL09_Token")
+ parser.add_argument("-t", "--type_sys", help="Which system produced the outputs", default="system")
+ parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLLUP_Token")
parser.add_argument("-cs", "--comment_str", help="CoNLL Format of comentaries inside the file", default="#")
args = parser.parse_args()
diff --git a/systems/parse_spacy.py b/systems/parse_spacy.py
index 1a32b67..3c56233 100644
--- a/systems/parse_spacy.py
+++ b/systems/parse_spacy.py
@@ -51,26 +51,34 @@
"""
EXAMPLE:
--- TIGER Classic Orthography ---
- python systems/parse_spacy.py --corpus_name Tiger \
+ python systems/parse_spacy.py --corpus_name Tiger --gld_token_type CoNLL09_Token \
-i /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09 \
-o /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu \
-t /home/daza/datasets/TIGER_conll/tiger_all.txt
+
+ python systems/parse_spacy.py --corpus_name TigerOld_test \
+ -i /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.OldOrth.test.conll \
+ -o /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.test.conllu
--- TIGER New Orthography ---
- python systems/parse_spacy.py --corpus_name TigerNew --gld_token_type CoNLLUP_Token \
+ python systems/parse_spacy.py --corpus_name TigerNew \
-i /home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.conll \
-o /home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.spacy_parsed.conllu \
-t /home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.txt
+
+ python systems/parse_spacy.py --corpus_name TigerNew_test \
+ -i /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll \
+ -o /home/daza/datasets/TIGER_conll/Tiger.NewOrth.test.spacy_parsed.conllu
--- German GSD Universal Deps ---
- python systems/parse_spacy.py --corpus_name DE_GSD --gld_token_type CoNLLUP_Token \
+ python systems/parse_spacy.py --corpus_name DE_GSD \
-i /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu \
-o /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.germalemma.conllu \
-t /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.txt
--- Real Data TEST ---
- time python systems/parse_spacy.py --corpus_name DeReKo_a00 --gld_token_type CoNLLUP_Token --comment_str "#" \
+ time python systems/parse_spacy.py --corpus_name DeReKo_a00 --comment_str "#" \
-i /export/netapp/kupietz/N-GRAMM-STUDIE/conllu/a00.conllu.gz \
-o /export/netapp/kupietz/N-GRAMM-STUDIE/conllu/0_SpaCyParsed/a00.spacy.gl.conllu
@@ -81,15 +89,16 @@
parser.add_argument("-n", "--corpus_name", help="Corpus Name", default="Corpus")
parser.add_argument("-o", "--output_file", help="File where the Predictions will be saved", required=True)
parser.add_argument("-t", "--text_file", help="Output Plain Text File", default=None)
- parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLL09_Token")
+ parser.add_argument("-sm", "--spacy_model", help="Spacy model containing the pipeline to tag", default="de_core_news_lg")
+ parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLLUP_Token")
parser.add_argument("-ugl", "--use_germalemma", help="Use Germalemma lemmatizer on top of SpaCy", default="True")
parser.add_argument("-c", "--comment_str", help="CoNLL Format of comentaries inside the file", default="#")
args = parser.parse_args()
file_has_next, chunk_ix = True, 0
- CHUNK_SIZE = 100000
- SPACY_BATCH = 10000
- SPACY_PROC = 50
+ CHUNK_SIZE = 20000
+ SPACY_BATCH = 2000
+ SPACY_PROC = 10
# =====================================================================================
# LOGGING INFO ...
@@ -103,7 +112,7 @@
# =====================================================================================
# POS TAG DOCUMENTS
# =====================================================================================
- spacy_de = spacy.load("de_core_news_lg", disable=["ner", "parser"])
+ spacy_de = spacy.load(args.spacy_model, disable=["ner", "parser"])
spacy_de.tokenizer = WhitespaceTokenizer(spacy_de.vocab) # We won't re-tokenize to respect how the source CoNLL are tokenized!
write_out = open(args.output_file, "w")
lemmatizer = GermaLemma()
diff --git a/systems/parse_spacy3.py b/systems/parse_spacy3.py
new file mode 100644
index 0000000..14e5a9f
--- /dev/null
+++ b/systems/parse_spacy3.py
@@ -0,0 +1,123 @@
+import argparse, os
+import spacy
+from spacy.language import Language
+from spacy.tokens import Doc
+import logging, sys, time
+from lib.CoNLL_Annotation import get_token_type
+import my_utils.file_utils as fu
+from germalemma import GermaLemma
+
+
+@Language.factory("my_component")
+class WhitespaceTokenizer(object):
+ def __init__(self, nlp, name):
+ self.vocab = nlp.vocab
+
+ def __call__(self, text):
+ words = text.split(' ')
+ # All tokens 'own' a subsequent space character in this tokenizer
+ spaces = [True] * len(words)
+ return Doc(self.vocab, words=words, spaces=spaces)
+
+
+def get_conll_str(anno_obj, spacy_doc, use_germalemma):
+ # First lines are comments. (metadata)
+ conll_lines = anno_obj.metadata # Then we want: [ID, FORM, LEMMA, UPOS, XPOS, FEATS, HEAD, DEPREL, DEPS, MISC]
+ for ix, token in enumerate(spacy_doc):
+ if use_germalemma == "True":
+ content = (str(ix), token.text, find_germalemma(token.text, token.tag_, token.lemma_), token.pos_, token.tag_, "_", "_", "_", "_", "_")
+ else:
+ content = (str(ix), token.text, token.lemma_, token.pos_, token.tag_, "_", "_", "_", "_", "_") # Pure SpaCy!
+ conll_lines.append("\t".join(content))
+ return "\n".join(conll_lines)
+
+
+# def freeling_lemma_lookup():
+# dicts_path = "/home/daza/Frameworks/FreeLing/data/de/dictionary/entries/"
+
+def find_germalemma(word, pos, spacy_lemma):
+ simplify_pos = {"ADJA":"ADJ", "ADJD":"ADJ",
+ "NA":"N", "NE":"N", "NN":"N",
+ "ADV":"ADV", "PAV":"ADV", "PROAV":"ADV", "PAVREL":"ADV", "PWAV":"ADV", "PWAVREL":"ADV",
+ "VAFIN":"V", "VAIMP":"V", "VAINF":"V", "VAPP":"V", "VMFIN":"V", "VMINF":"V",
+ "VMPP":"V", "VVFIN":"V", "VVIMP":"V", "VVINF":"V", "VVIZU":"V","VVPP":"V"
+ }
+ # simplify_pos = {"VERB": "V", "ADV": "ADV", "ADJ": "ADJ", "NOUN":"N", "PROPN": "N"}
+ try:
+ return lemmatizer.find_lemma(word, simplify_pos.get(pos, "UNK"))
+ except:
+ return spacy_lemma
+
+
+if __name__ == "__main__":
+ """
+ EXAMPLE:
+ --- TIGER Classic Orthography ---
+ python systems/parse_spacy3.py --corpus_name TigerTestNew \
+ -i /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.NewOrth.test.conll \
+ -o /home/daza/datasets/TIGER_conll/tiger_spacy3_parsed.conllu
+ """
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-i", "--input_file", help="Input Corpus", required=True)
+ parser.add_argument("-n", "--corpus_name", help="Corpus Name", default="Corpus")
+ parser.add_argument("-o", "--output_file", help="File where the Predictions will be saved", required=True)
+ parser.add_argument("-sm", "--spacy_model", help="Spacy model containing the pipeline to tag", default="de_core_news_sm")
+ parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLLUP_Token")
+ parser.add_argument("-ugl", "--use_germalemma", help="Use Germalemma lemmatizer on top of SpaCy", default="True")
+ parser.add_argument("-c", "--comment_str", help="CoNLL Format of comentaries inside the file", default="#")
+ args = parser.parse_args()
+
+ file_has_next, chunk_ix = True, 0
+ CHUNK_SIZE = 1000
+ SPACY_BATCH = 100
+ SPACY_PROC = 4
+
+ # =====================================================================================
+ # LOGGING INFO ...
+ # =====================================================================================
+ logger = logging.getLogger(__name__)
+ console_hdlr = logging.StreamHandler(sys.stdout)
+ file_hdlr = logging.FileHandler(filename=f"logs/Parse_{args.corpus_name}.SpaCy.log")
+ logging.basicConfig(level=logging.INFO, handlers=[console_hdlr, file_hdlr])
+ logger.info(f"Chunking {args.corpus_name} Corpus in chunks of {CHUNK_SIZE} Sentences")
+
+ # =====================================================================================
+ # POS TAG DOCUMENTS
+ # =====================================================================================
+
+ if os.path.exists(args.spacy_model):
+ pass # Load Custom Trained model
+ else:
+ # try:
+ spacy_de = spacy.load(args.spacy_model, disable=["ner", "parser"])
+ spacy_de.tokenizer = WhitespaceTokenizer(spacy_de, "keep_original_tokens") # We won't re-tokenize to respect how the source CoNLL are tokenized!
+ # except:
+ # print(f"Check if model {args.spacy_model} is a valid SpaCy Pipeline or if the Path containing the trained model exists!")
+ # exit()
+
+ write_out = open(args.output_file, "w")
+ lemmatizer = GermaLemma()
+
+ if ".gz" == args.input_file[-3:]:
+ in_file = fu.expand_file(args.input_file)
+ else:
+ in_file = args.input_file
+
+ start = time.time()
+ total_processed_sents = 0
+ line_generator = fu.file_generator(in_file)
+ while file_has_next:
+ annos, file_has_next = fu.get_file_annos_chunk(line_generator, chunk_size=CHUNK_SIZE, token_class=get_token_type(args.gld_token_type), comment_str=args.comment_str)
+ if len(annos) == 0: break
+ total_processed_sents += len(annos)
+ logger.info(f"Already processed {total_processed_sents} sentences...")
+ sents = [a.get_sentence() for a in annos]
+ for ix, doc in enumerate(spacy_de.pipe(sents, batch_size=SPACY_BATCH, n_process=SPACY_PROC)):
+ conll_str = get_conll_str(annos[ix], doc, use_germalemma=args.use_germalemma)
+ write_out.write(conll_str)
+ write_out.write("\n\n")
+
+ end = time.time()
+ logger.info(f"Processing {args.corpus_name} took {(end - start)} seconds!")
+
\ No newline at end of file
diff --git a/systems/parse_turku.py b/systems/parse_turku.py
index 7694504..284dbd4 100644
--- a/systems/parse_turku.py
+++ b/systems/parse_turku.py
@@ -6,11 +6,6 @@
import my_utils.file_utils as fu
import argparse
-
-TIGER_CORPUS = "/home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09"
-DE_GSD_CORPUS = "/home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu"
-
-
if __name__ == "__main__":
"""
@@ -18,6 +13,9 @@
python systems/parse_turku.py --corpus_name DE_GSD --gld_token_type CoNLLUP_Token \
-i /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu
+
+ python systems/parse_turku.py --corpus_name TigerTestOld --gld_token_type CoNLLUP_Token \
+ -i /home/daza/datasets/TIGER_conll/data_splits/test/Tiger.OldOrth.test.conll
"""