Successfully evaluated several taggers
diff --git a/my_utils/conll_to_tok.py b/my_utils/conll_to_tok.py
new file mode 100644
index 0000000..9d77543
--- /dev/null
+++ b/my_utils/conll_to_tok.py
@@ -0,0 +1,26 @@
+import argparse
+from lib.CoNLL_Annotation import read_conll_generator, CoNLL09_Token
+
+# TODO: Parallelize this for HUGE Files: All sentences can be processed independently
+
+if __name__ == "__main__":
+ """
+ EXAMPLE:
+ For TreeTagger:
+ python my_utils/conll_to_tok.py -s /vol/netapp/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09 -ss "</S>"
+
+ For RNNTagger
+ python my_utils/conll_to_tok.py -s /vol/netapp/daza/datasets/TIGER_conll/tiger_release_aug07.corrected.16012013.conll09
+ """
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-s", "--src_file", help="CoNLLU File to Convert into the .tok input for RNNTagger/TreeTagger", required=True)
+ parser.add_argument("-ss", "--sent_sep", help="Special separator to distinguish sentence boundaries", default="")
+ args = parser.parse_args()
+
+ output_file = open(f"{args.src_file}.tok","w")
+
+ for conll_obj in read_conll_generator(args.src_file, token_class=CoNLL09_Token):
+ for tok in conll_obj.tokens:
+ output_file.write(tok.word+"\n")
+ output_file.write(args.sent_sep+"\n")
\ No newline at end of file