Added training examples for SpaCy3
diff --git a/systems/parse_spacy.py b/systems/parse_spacy.py
index 75a1eb5..1a32b67 100644
--- a/systems/parse_spacy.py
+++ b/systems/parse_spacy.py
@@ -18,8 +18,9 @@
return Doc(self.vocab, words=words, spaces=spaces)
-def get_conll_str(spacy_doc, use_germalemma):
- conll_lines = [] # We want: [ID, FORM, LEMMA, UPOS, XPOS, FEATS, HEAD, DEPREL, DEPS, MISC]
+def get_conll_str(anno_obj, spacy_doc, use_germalemma):
+ # First lines are comments. (metadata)
+ conll_lines = anno_obj.metadata # Then we want: [ID, FORM, LEMMA, UPOS, XPOS, FEATS, HEAD, DEPREL, DEPS, MISC]
for ix, token in enumerate(spacy_doc):
if use_germalemma == "True":
content = (str(ix), token.text, find_germalemma(token.text, token.tag_, token.lemma_), token.pos_, token.tag_, "_", "_", "_", "_", "_")
@@ -49,15 +50,30 @@
if __name__ == "__main__":
"""
EXAMPLE:
+ --- TIGER Classic Orthography ---
python systems/parse_spacy.py --corpus_name Tiger \
-i /home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09 \
-o /home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu \
-t /home/daza/datasets/TIGER_conll/tiger_all.txt
-
+
+ --- TIGER New Orthography ---
+ python systems/parse_spacy.py --corpus_name TigerNew --gld_token_type CoNLLUP_Token \
+ -i /home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.conll \
+ -o /home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.spacy_parsed.conllu \
+ -t /home/daza/datasets/TIGER_conll/Tiger.NewOrth.train.txt
+
+ --- German GSD Universal Deps ---
python systems/parse_spacy.py --corpus_name DE_GSD --gld_token_type CoNLLUP_Token \
-i /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.conllu \
-o /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.parsed.germalemma.conllu \
- -t/home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.txt
+ -t /home/daza/datasets/ud-treebanks-v2.2/UD_German-GSD/de_gsd-ud-test.txt
+
+
+ --- Real Data TEST ---
+ time python systems/parse_spacy.py --corpus_name DeReKo_a00 --gld_token_type CoNLLUP_Token --comment_str "#" \
+ -i /export/netapp/kupietz/N-GRAMM-STUDIE/conllu/a00.conllu.gz \
+ -o /export/netapp/kupietz/N-GRAMM-STUDIE/conllu/0_SpaCyParsed/a00.spacy.gl.conllu
+
"""
parser = argparse.ArgumentParser()
@@ -71,7 +87,9 @@
args = parser.parse_args()
file_has_next, chunk_ix = True, 0
- CHUNK_SIZE = 10000
+ CHUNK_SIZE = 100000
+ SPACY_BATCH = 10000
+ SPACY_PROC = 50
# =====================================================================================
# LOGGING INFO ...
@@ -91,16 +109,22 @@
lemmatizer = GermaLemma()
if args.text_file: write_plain = open(args.text_file, "w")
+ if ".gz" == args.input_file[-3:]:
+ in_file = fu.expand_file(args.input_file)
+ else:
+ in_file = args.input_file
+
start = time.time()
total_processed_sents = 0
- line_generator = fu.file_generator(args.input_file)
+ line_generator = fu.file_generator(in_file)
while file_has_next:
- sents, gld, file_has_next = fu.get_file_text_chunk(line_generator, chunk_size=CHUNK_SIZE, token_class=get_token_type(args.gld_token_type), comment_str=args.comment_str)
- if len(sents) == 0: break
- total_processed_sents += len(sents)
+ annos, file_has_next = fu.get_file_annos_chunk(line_generator, chunk_size=CHUNK_SIZE, token_class=get_token_type(args.gld_token_type), comment_str=args.comment_str)
+ if len(annos) == 0: break
+ total_processed_sents += len(annos)
logger.info(f"Already processed {total_processed_sents} sentences...")
- for doc in spacy_de.pipe(sents, batch_size=1000, n_process=10):
- conll_str = get_conll_str(doc, use_germalemma=args.use_germalemma)
+ sents = [a.get_sentence() for a in annos]
+ for ix, doc in enumerate(spacy_de.pipe(sents, batch_size=SPACY_BATCH, n_process=SPACY_PROC)):
+ conll_str = get_conll_str(annos[ix], doc, use_germalemma=args.use_germalemma)
write_out.write(conll_str)
write_out.write("\n\n")
if args.text_file: