commit 1e882fb0d540f28f88576c61d200ad60ddc7bc88
author Marc Kupietz <kupietz@ids-mannheim.de> Wed Sep 09 00:05:46 2020 +0200
committer Marc Kupietz <kupietz@ids-mannheim.de> Thu Sep 24 20:08:27 2020 +0200
tree 0cc249aa8d697d8152c983a7794e62661aa077c5
parent 598d1a70953b6cd33c3952b39c7778a8ce579586

Add -tk option to use the standard KoAP tokenizer

Change-Id: I992fe37463926c8ecbca933fbb709f8640d6fb93

t/cmd/tokenizer.pl

diff --git a/t/cmd/tokenizer.pl b/t/cmd/tokenizer.pl
index bf948e8..4e8b2d0 100644
--- a/t/cmd/tokenizer.pl
+++ b/t/cmd/tokenizer.pl
@@ -2,6 +2,7 @@
 use strict;
 use warnings;
 use FindBin;
+use Encode;
 BEGIN {
   unshift @INC, "$FindBin::Bin/../../lib";
 };
@@ -14,7 +15,7 @@
 
 # Read lines from input and return boundaries
 while (!eof(STDIN)) {
-  my $line = <>;
+  my $line = decode_utf8(<>);
   for my $text (split(/\n?\x{04}\n?/, $line)) {
     $tok->tokenize($text);
     print join(' ', $tok->boundaries), "\n";