Improve emoji detection and support
Change-Id: Ie245920a51d64e934522a26eb378d350d69f5a98
diff --git a/t/real/ndy.t b/t/real/ndy.t
index e6ce10a..c0f5dc7 100644
--- a/t/real/ndy.t
+++ b/t/real/ndy.t
@@ -66,5 +66,22 @@
ok(!$meta->{A_doc_editor}, 'Correct doc editor');
+# Tokenization
+use_ok('KorAP::XML::Tokenizer');
+
+my ($token_base_foundry, $token_base_layer) = (qw/Base Tokens/);
+
+# Get tokenization
+my $tokens = KorAP::XML::Tokenizer->new(
+ path => $doc->path,
+ doc => $doc,
+ foundry => $token_base_foundry,
+ layer => $token_base_layer,
+ name => 'tokens'
+);
+
+ok($tokens, 'Token Object is fine');
+ok($tokens->parse, 'Token parsing is fine');
+
done_testing;
__END__