Add cutter, more spacys and Stanford Tokenizer only
Change-Id: I6ae4c014298d9c07e62850d39408b77cb145a9cd
diff --git a/cutter/cutter.py b/cutter/cutter.py
new file mode 100644
index 0000000..a865a49
--- /dev/null
+++ b/cutter/cutter.py
@@ -0,0 +1,22 @@
+import Cutter
+import sys
+
+sys.setrecursionlimit(100000)
+
+cutter = Cutter.Cutter(profile='de')
+
+sent = sys.argv[1]
+
+file = open(sys.argv[2], 'r')
+
+text = file.read()
+
+file.close()
+
+for token in cutter.cut(text):
+ if token[0]:
+ print(token[0])
+
+ if sent == "sent":
+ if token[1].startswith("+EOS"):
+ print("</eos>")