Gitiles
Code Review
Sign In
korap.ids-mannheim.de
/
KorAP
/
Tokenizer-Evaluation
/
9c944a4a592acbd230d567d0fcb95efea3498837
/
.
/
benchmarks
/
cleanup
/
tokenize_nn.pl
blob: 3124c6aed3b3823d3cd6df0ba32a4ecec9d8964e [
file
] [
log
] [
blame
]
#!/usr/bin/env perl
use
strict
;
use
warnings
;
my
$c
=
''
;
foreach
(<>)
{
$c
.=
$_
;
};
$c
=~
s
/^
\n
+/
/s;
foreach my $c (split(/
\n\n
/,
$c
))
{
$c
=~
s
/[
\s\n\t
]+//
g
;
print
$c
,
"\n"
;
};