clean up intern tokenization
- moved intern tokenization out of retr_info()
- removed $offset parameter (not anymore necessary)
Change-Id: I063efdee193ab41c2705971a95341573884fd3a2
diff --git a/xt/benchmark.pl b/xt/benchmark.pl
index c1657a6..9d0489a 100644
--- a/xt/benchmark.pl
+++ b/xt/benchmark.pl
@@ -106,14 +106,14 @@
Dumbbench::Instance::PerlSub->new(
name => 'Tokenizer-conservative',
code => sub {
- $result = $cons_tok->reset->tokenize($t_data, 0);
+ $result = $cons_tok->reset->tokenize($t_data);
$result = 0;
}
),
Dumbbench::Instance::PerlSub->new(
name => 'Tokenizer-aggressive',
code => sub {
- $result = $aggr_tok->reset->tokenize($t_data, 0);
+ $result = $aggr_tok->reset->tokenize($t_data);
$result = 0;
}
),