commit d53ab4b72902bb0f78a8068a471f4b3debd94a64
author Akron <nils@diewald-online.de> Wed Feb 24 09:56:12 2021 +0100
committer Akron <nils@diewald-online.de> Wed Feb 24 11:58:52 2021 +0100
tree 95acfa11ae7568932965c2602515086a400dddf5
parent 3556c75666d0000d22aff19715b9278a0eae8cb4

Sentencize only if ext_tok is available

Change-Id: I0b1b1ce3e724a546caeab7524b6cf353aeaf0dda

script/tei2korapxml

diff --git a/script/tei2korapxml b/script/tei2korapxml
index 561c824..70289f4 100755
--- a/script/tei2korapxml
+++ b/script/tei2korapxml
@@ -299,6 +299,10 @@
             $zipper->new_stream("$dir/$base_dir/${tokens_file}.xml"),
             $text_id_esc
           );
+
+          if ($use_tokenizer_sentence_splits) {
+            $ext_tok->sentencize_from_previous_input($structures);
+          };
         };
 
         # Tokenize with internal tokenizer
@@ -319,10 +323,6 @@
           $cons_tok->reset;
         };
 
-        if ($use_tokenizer_sentence_splits) {
-          $ext_tok->sentencize_from_previous_input($structures);
-        };
-
         # ~ write structures ~
         if (!$structures->empty) {
           $structures->to_zip(