Added training examples for SpaCy3
diff --git a/systems/eval_old_vs_new_tiger.py b/systems/eval_old_vs_new_tiger.py
new file mode 100644
index 0000000..38dc597
--- /dev/null
+++ b/systems/eval_old_vs_new_tiger.py
@@ -0,0 +1,35 @@
+import my_utils.file_utils as fu
+from lib.CoNLL_Annotation import read_conll, CoNLLUP_Token
+from collections import Counter
+from germalemma import GermaLemma
+
+SPACY_NEW = "/home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.spacy_parsed.conllu"
+CASES = "/home/daza/datasets/TIGER_conll/NewOrthProblems_Indices.train.txt"
+
+orth_dict = fu.file_to_dict("/vol/netapp/daza/datasets/TIGER_conll/TigerOrthMapping.train.json")
+new_to_old = {v:k for k,v in orth_dict.items()}
+
+
+if __name__ == "__main__":
+ line_generator = fu.file_generator(SPACY_NEW)
+ conll_sents, _ = read_conll(line_generator, chunk_size=60000, token_class=CoNLLUP_Token, comment_str="#")
+ special_cases = [int(line) for line in open(CASES).read().splitlines()]
+ checked_cases = []
+
+ lemmatizer = GermaLemma()
+
+ for ix, sent in enumerate(conll_sents):
+ if ix in special_cases:
+ for tok in sent.tokens:
+ old_word_change = new_to_old.get(tok.word)
+ if old_word_change:
+ try:
+ old_lemma = lemmatizer.find_lemma(old_word_change, tok.pos_tag)
+ except:
+ old_lemma = f"UNK_{tok.pos_tag}"
+ checked_cases.append((old_word_change, tok.word, old_lemma, tok.lemma))
+
+ print(f"Cases checked: {len(checked_cases)}")
+ case_count = Counter(checked_cases).most_common()
+ fu.counter_to_file(case_count, "/home/daza/datasets/TIGER_conll/TigerLemmas_Old_New.tsv")
+
\ No newline at end of file
diff --git a/systems/parse_spacy.py b/systems/parse_spacy.py
index 75a1eb5..1a32b67 100644
--- a/systems/parse_spacy.py
+++ b/systems/parse_spacy.py
@@ -18,8 +18,9 @@
return Doc(self.vocab, words=words, spaces=spaces)
-def get_conll_str(spacy_doc, use_germalemma):
- conll_lines = [] # We want: [ID, FORM, LEMMA, UPOS, XPOS, FEATS, HEAD, DEPREL, DEPS, MISC]
+def get_conll_str(anno_obj, spacy_doc, use_germalemma):
+ # First lines are comments. (metadata)
+ conll_lines = anno_obj.metadata # Then we want: [ID, FORM, LEMMA, UPOS, XPOS, FEATS, HEAD, DEPREL, DEPS, MISC]
for ix, token in enumerate(spacy_doc):
if use_germalemma == "True":
content = (str(ix), token.text, find_germalemma(token.text, token.tag_, token.lemma_), token.pos_, token.tag_, "_", "_", "_", "_", "_")
@@ -49,15 +50,30 @@
if __name__ == "__main__":
"""
EXAMPLE:
+ --- TIGER Classic Orthography ---
python systems/parse_spacy.py --corpus_name Tiger \
-i /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09 \
-o /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu \
-t /home/daza/datasets/TIGER_conll/tiger_all.txt
-
+
+ --- TIGER New Orthography ---
+ python systems/parse_spacy.py --corpus_name TigerNew --gld_token_type CoNLLUP_Token \
+ -i /home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.conll \
+ -o /home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.spacy_parsed.conllu \
+ -t /home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.txt
+
+ --- German GSD Universal Deps ---
python systems/parse_spacy.py --corpus_name DE_GSD --gld_token_type CoNLLUP_Token \
-i /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu \
-o /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.germalemma.conllu \
- -t/home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.txt
+ -t /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.txt
+
+
+ --- Real Data TEST ---
+ time python systems/parse_spacy.py --corpus_name DeReKo_a00 --gld_token_type CoNLLUP_Token --comment_str "#" \
+ -i /export/netapp/kupietz/N-GRAMM-STUDIE/conllu/a00.conllu.gz \
+ -o /export/netapp/kupietz/N-GRAMM-STUDIE/conllu/0_SpaCyParsed/a00.spacy.gl.conllu
+
"""
parser = argparse.ArgumentParser()
@@ -71,7 +87,9 @@
args = parser.parse_args()
file_has_next, chunk_ix = True, 0
- CHUNK_SIZE = 10000
+ CHUNK_SIZE = 100000
+ SPACY_BATCH = 10000
+ SPACY_PROC = 50
# =====================================================================================
# LOGGING INFO ...
@@ -91,16 +109,22 @@
lemmatizer = GermaLemma()
if args.text_file: write_plain = open(args.text_file, "w")
+ if ".gz" == args.input_file[-3:]:
+ in_file = fu.expand_file(args.input_file)
+ else:
+ in_file = args.input_file
+
start = time.time()
total_processed_sents = 0
- line_generator = fu.file_generator(args.input_file)
+ line_generator = fu.file_generator(in_file)
while file_has_next:
- sents, gld, file_has_next = fu.get_file_text_chunk(line_generator, chunk_size=CHUNK_SIZE, token_class=get_token_type(args.gld_token_type), comment_str=args.comment_str)
- if len(sents) == 0: break
- total_processed_sents += len(sents)
+ annos, file_has_next = fu.get_file_annos_chunk(line_generator, chunk_size=CHUNK_SIZE, token_class=get_token_type(args.gld_token_type), comment_str=args.comment_str)
+ if len(annos) == 0: break
+ total_processed_sents += len(annos)
logger.info(f"Already processed {total_processed_sents} sentences...")
- for doc in spacy_de.pipe(sents, batch_size=1000, n_process=10):
- conll_str = get_conll_str(doc, use_germalemma=args.use_germalemma)
+ sents = [a.get_sentence() for a in annos]
+ for ix, doc in enumerate(spacy_de.pipe(sents, batch_size=SPACY_BATCH, n_process=SPACY_PROC)):
+ conll_str = get_conll_str(annos[ix], doc, use_germalemma=args.use_germalemma)
write_out.write(conll_str)
write_out.write("\n\n")
if args.text_file: