Added SpaCy parser compatible with linux pipe
Change-Id: Ia20b48faec6dffc038f984756e43e4179e5df415
diff --git a/.gitignore b/.gitignore
index 0df714e..77f8900 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,6 @@
-<<<<<<< HEAD
outputs/
+logs/
-=======
->>>>>>> 54e072e61c24a3ce12a7f47e17e9d8d0d1583236
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
@@ -117,10 +115,7 @@
.venv
env/
venv/
-<<<<<<< HEAD
venv-*
-=======
->>>>>>> 54e072e61c24a3ce12a7f47e17e9d8d0d1583236
ENV/
env.bak/
venv.bak/
diff --git a/systems/parse_spacy_pipe.py b/systems/parse_spacy_pipe.py
new file mode 100644
index 0000000..b39e2f2
--- /dev/null
+++ b/systems/parse_spacy_pipe.py
@@ -0,0 +1,100 @@
+from sys import stdin
+import argparse
+import spacy
+from spacy.tokens import Doc
+import logging, sys, time
+from lib.CoNLL_Annotation import get_token_type
+import my_utils.file_utils as fu
+from germalemma import GermaLemma
+
+
+class WhitespaceTokenizer(object):
+ def __init__(self, vocab):
+ self.vocab = vocab
+
+ def __call__(self, text):
+ words = text.split(' ')
+ # All tokens 'own' a subsequent space character in this tokenizer
+ spaces = [True] * len(words)
+ return Doc(self.vocab, words=words, spaces=spaces)
+
+
+def get_conll_str(anno_obj, spacy_doc, use_germalemma):
+ # First lines are comments. (metadata)
+ conll_lines = anno_obj.metadata # Then we want: [ID, FORM, LEMMA, UPOS, XPOS, FEATS, HEAD, DEPREL, DEPS, MISC]
+ for ix, token in enumerate(spacy_doc):
+ if use_germalemma == "True":
+ content = (str(ix), token.text, find_germalemma(token.text, token.tag_, token.lemma_), token.pos_, token.tag_, "_", "_", "_", "_", "_")
+ else:
+ content = (str(ix), token.text, token.lemma_, token.pos_, token.tag_, "_", "_", "_", "_", "_") # Pure SpaCy!
+ conll_lines.append("\t".join(content))
+ return "\n".join(conll_lines)
+
+
+def find_germalemma(word, pos, spacy_lemma):
+ simplify_pos = {"ADJA":"ADJ", "ADJD":"ADJ",
+ "NA":"N", "NE":"N", "NN":"N",
+ "ADV":"ADV", "PAV":"ADV", "PROAV":"ADV", "PAVREL":"ADV", "PWAV":"ADV", "PWAVREL":"ADV",
+ "VAFIN":"V", "VAIMP":"V", "VAINF":"V", "VAPP":"V", "VMFIN":"V", "VMINF":"V",
+ "VMPP":"V", "VVFIN":"V", "VVIMP":"V", "VVINF":"V", "VVIZU":"V","VVPP":"V"
+ }
+ # simplify_pos = {"VERB": "V", "ADV": "ADV", "ADJ": "ADJ", "NOUN":"N", "PROPN": "N"}
+ try:
+ return lemmatizer.find_lemma(word, simplify_pos.get(pos, "UNK"))
+ except:
+ return spacy_lemma
+
+
+if __name__ == "__main__":
+ """
+ --- Example Real Data TEST ---
+
+ cat /export/netapp/kupietz/N-GRAMM-STUDIE/conllu/zca18.conllu | python systems/parse_spacy_pipe.py \
+ --corpus_name DeReKo_zca18 --comment_str "#" > output_zca18.conll
+ """
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-n", "--corpus_name", help="Corpus Name", default="Corpus")
+ parser.add_argument("-sm", "--spacy_model", help="Spacy model containing the pipeline to tag", default="de_core_news_lg")
+ parser.add_argument("-gtt", "--gld_token_type", help="CoNLL Format of the Gold Data", default="CoNLLUP_Token")
+ parser.add_argument("-ugl", "--use_germalemma", help="Use Germalemma lemmatizer on top of SpaCy", default="True")
+ parser.add_argument("-c", "--comment_str", help="CoNLL Format of comentaries inside the file", default="#")
+ args = parser.parse_args()
+
+ file_has_next, chunk_ix = True, 0
+ CHUNK_SIZE = 20000
+ SPACY_BATCH = 2000
+ SPACY_PROC = 10
+
+ # =====================================================================================
+ # LOGGING INFO ...
+ # =====================================================================================
+ logger = logging.getLogger(__name__)
+ console_hdlr = logging.StreamHandler(sys.stderr)
+ file_hdlr = logging.FileHandler(filename=f"logs/Parse_{args.corpus_name}.SpaCy.log")
+ logging.basicConfig(level=logging.INFO, handlers=[console_hdlr, file_hdlr])
+ logger.info(f"Chunking {args.corpus_name} Corpus in chunks of {CHUNK_SIZE} Sentences")
+
+ # =====================================================================================
+ # POS TAG DOCUMENTS
+ # =====================================================================================
+ spacy_de = spacy.load(args.spacy_model, disable=["ner", "parser"])
+ spacy_de.tokenizer = WhitespaceTokenizer(spacy_de.vocab) # We won't re-tokenize to respect how the source CoNLL are tokenized!
+ lemmatizer = GermaLemma()
+
+ start = time.time()
+ total_processed_sents = 0
+
+ while file_has_next:
+ annos, file_has_next = fu.get_file_annos_chunk(stdin, chunk_size=CHUNK_SIZE, token_class=get_token_type(args.gld_token_type), comment_str=args.comment_str)
+ if len(annos) == 0: break
+ total_processed_sents += len(annos)
+ logger.info(f"Already processed {total_processed_sents} sentences...")
+ sents = [a.get_sentence() for a in annos]
+ for ix, doc in enumerate(spacy_de.pipe(sents, batch_size=SPACY_BATCH, n_process=SPACY_PROC)):
+ conll_str = get_conll_str(annos[ix], doc, use_germalemma=args.use_germalemma)
+ print(conll_str+ "\n")
+
+ end = time.time()
+ logger.info(f"Processing {args.corpus_name} took {(end - start)} seconds!")
+
\ No newline at end of file