| daza | ff42f63 | 2020-10-08 14:46:32 +0200 | [diff] [blame^] | 1 | import spacy |
| 2 | from spacy.tokens import Doc |
| 3 | import logging, sys, time |
| 4 | from lib.CoNLL_Annotation import CoNLL09_Token |
| 5 | import my_utils.file_utils as fu |
| 6 | from germalemma import GermaLemma |
| 7 | |
| 8 | |
| 9 | TIGER_CORPUS = "/home/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09" |
| 10 | OUTPUT_FILE = "/home/daza/datasets/TIGER_conll/tiger_spacy_parsed.conllu" |
| 11 | TEXT_OUTPUT = "/home/daza/datasets/TIGER_conll/tiger_all.txt" |
| 12 | |
| 13 | |
| 14 | class WhitespaceTokenizer(object): |
| 15 | def __init__(self, vocab): |
| 16 | self.vocab = vocab |
| 17 | |
| 18 | def __call__(self, text): |
| 19 | words = text.split(' ') |
| 20 | # All tokens 'own' a subsequent space character in this tokenizer |
| 21 | spaces = [True] * len(words) |
| 22 | return Doc(self.vocab, words=words, spaces=spaces) |
| 23 | |
| 24 | |
| 25 | def get_conll_str(spacy_doc): |
| 26 | conll_lines = [] # We want: [ID, FORM, LEMMA, UPOS, XPOS, FEATS, HEAD, DEPREL, DEPS, MISC] |
| 27 | for ix, token in enumerate(spacy_doc): |
| 28 | content = (str(ix), token.text, find_germalemma(token.text, token.tag_, token.lemma_), token.pos_, token.tag_, "_", "_", "_", "_", "_") |
| 29 | conll_lines.append("\t".join(content)) |
| 30 | return "\n".join(conll_lines) |
| 31 | |
| 32 | |
| 33 | # def freeling_lemma_lookup(): |
| 34 | # dicts_path = "/home/daza/Frameworks/FreeLing/data/de/dictionary/entries/" |
| 35 | |
| 36 | def find_germalemma(word, pos, spacy_lemma): |
| 37 | simplify_pos = {"ADJA":"ADJ", "ADJD":"ADJ", |
| 38 | "NA":"N", "NE":"N", "NN":"N", |
| 39 | "ADV":"ADV", "PAV":"ADV", "PROAV":"ADV", "PAVREL":"ADV", "PWAV":"ADV", "PWAVREL":"ADV", |
| 40 | "VAFIN":"V", "VAIMP":"V", "VAINF":"V", "VAPP":"V", "VMFIN":"V", "VMINF":"V", |
| 41 | "VMPP":"V", "VVFIN":"V", "VVIMP":"V", "VVINF":"V", "VVIZU":"V","VVPP":"V" |
| 42 | } |
| 43 | # simplify_pos = {"VERB": "V", "ADV": "ADV", "ADJ": "ADJ", "NOUN":"N", "PROPN": "N"} |
| 44 | try: |
| 45 | return lemmatizer.find_lemma(word, simplify_pos.get(pos, "UNK")) |
| 46 | except: |
| 47 | return spacy_lemma |
| 48 | |
| 49 | |
| 50 | if __name__ == "__main__": |
| 51 | file_has_next, chunk_ix = True, 0 |
| 52 | CHUNK_SIZE = 10000 |
| 53 | |
| 54 | # ===================================================================================== |
| 55 | # LOGGING INFO ... |
| 56 | # ===================================================================================== |
| 57 | logger = logging.getLogger(__name__) |
| 58 | console_hdlr = logging.StreamHandler(sys.stdout) |
| 59 | file_hdlr = logging.FileHandler(filename=f"logs/Parse_Tiger.SpaCy.log") |
| 60 | logging.basicConfig(level=logging.INFO, handlers=[console_hdlr, file_hdlr]) |
| 61 | logger.info(f"Chunking TIGER Corpus in chunks of {CHUNK_SIZE} Sentences") |
| 62 | |
| 63 | # ===================================================================================== |
| 64 | # POS TAG DOCUMENTS |
| 65 | # ===================================================================================== |
| 66 | spacy_de = spacy.load("de_core_news_lg", disable=["ner", "parser"]) |
| 67 | spacy_de.tokenizer = WhitespaceTokenizer(spacy_de.vocab) # We won't re-tokenize to respect how the source CoNLL are tokenized! |
| 68 | write_out = open(OUTPUT_FILE, "w") |
| 69 | lemmatizer = GermaLemma() |
| 70 | write_plain = open(TEXT_OUTPUT, "w") |
| 71 | |
| 72 | start = time.time() |
| 73 | total_processed_sents = 0 |
| 74 | line_generator = fu.file_generator(TIGER_CORPUS) |
| 75 | while file_has_next: |
| 76 | sents, gld, file_has_next = fu.get_file_text_chunk(line_generator, chunk_size=CHUNK_SIZE, token_class=CoNLL09_Token) |
| 77 | total_processed_sents += len(sents) |
| 78 | logger.info(f"Already processed {total_processed_sents} sentences...") |
| 79 | for doc in spacy_de.pipe(sents, batch_size=1000, n_process=10): |
| 80 | conll_str = get_conll_str(doc) |
| 81 | write_out.write(conll_str) |
| 82 | write_out.write("\n\n") |
| 83 | write_plain.write(" ".join([x.text for x in doc])+"\n") # OPTIONAL: This can be commented for efficiency... |
| 84 | |
| 85 | end = time.time() |
| 86 | logger.info(f"Processing File {TIGER_CORPUS} took {(end - start)} seconds!") |
| 87 | |