Test matches with new search engine
Change-Id: I2d29d4e492b7a0cc34f4a3d5ba3126720bc11e66
diff --git a/kalamar.conf b/kalamar.conf
index 700576a..e194d0a 100644
--- a/kalamar.conf
+++ b/kalamar.conf
@@ -24,6 +24,8 @@
# See Mojolicious::Plugin::Localize
# - TagHelpers-ContentBlock
# See Mojolicious::Plugin::TagHelpers::ContentBlock
+# - CHI
+# See Mojolicious::Plugin::CHI
# The default Kustvakt api endpoint
my $api = 'http://localhost:9999/api/';
diff --git a/lib/Kalamar/Controller/Search2.pm b/lib/Kalamar/Controller/Search2.pm
index e018fec..7440444 100644
--- a/lib/Kalamar/Controller/Search2.pm
+++ b/lib/Kalamar/Controller/Search2.pm
@@ -3,6 +3,7 @@
use Data::Dumper;
use Mojo::Collection 'c';
use Mojo::ByteStream 'b';
+use POSIX 'ceil';
# This should be implemented as a helper
has api => '/api/';
@@ -12,6 +13,9 @@
has items_per_page => 25;
+# TODO:
+# Support server timing API
+
# Catch connection errors
sub _catch_http_errors {
my $tx = shift;
@@ -28,15 +32,7 @@
# Catch koral errors
sub _catch_koral_errors {
- my $res = shift;
-
- my $json = $res->json;
-
- unless ($json) {
- return Mojo::Promise->new->reject([
- [undef, 'JSON response is invalid']
- ]);
- };
+ my $json = shift;
# Get errors
my $err = $json->{errors};
@@ -82,24 +78,6 @@
};
-# Notify the user in case of errors
-#sub _notify_on_warnings {
-# my ($self, $json) = @_;
-#
-# if ($json->{warnings}) {
-#
-# # TODO: Check for ref!
-# foreach (@{$json->{warnings}}) {
-# $self->notify(
-# warn =>
-# ($_->[0] ? $_->[0] . ': ' : '') .
-# $_->[1]
-# );
-# };
-# };
-#}
-
-
# Process response and set stash values
sub _process_matches {
my ($self, $json) = @_;
@@ -116,8 +94,8 @@
# if ($benchmark && $benchmark =~ s/\s+(m)?s$//) {
# $benchmark = sprintf("%.2f", $benchmark) . ($1 ? $1 : '') . 's';
# };
- # Set benchmark
- # $index->benchmark($benchmark);
+ # # Set benchmark
+ # $self->stash(benchmark => $benchmark);
# Set time exceeded
if ($meta->{timeExceeded} && $meta->{timeExceeded} eq Mojo::JSON::true) {
@@ -199,13 +177,14 @@
$v->optional('q', 'trim');
$v->optional('ql')->in(qw/poliqarp cosmas2 annis cql fcsql/);
- $v->optional('collection'); # Legacy
+ $v->optional('collection', 'trim'); # Legacy
$v->optional('cq', 'trim');
# $v->optional('action'); # action 'inspect' is no longer valid
$v->optional('snippet');
$v->optional('cutoff')->in(qw/true false/);
$v->optional('count')->num(1, undef);
- $v->optional('p');
+ $v->optional('p', 'trim')->num(1, undef); # Start page
+ $v->optional('o', 'trim')->num(1, undef); # Offset
$v->optional('context');
# Get query
@@ -218,13 +197,41 @@
my %query = ();
$query{q} = $v->param('q');
- $query{ql} = $v->param('ql');
- $query{page} = $v->param('p') if $v->param('p');
+ $query{ql} = $v->param('ql') // 'poliqarp';
+ $query{p} = $v->param('p') // 1; # Start page
$query{count} = $v->param('count') // $c->items_per_page;
$query{cq} = $v->param('cq') // $v->param('collection');
$query{cutoff} = $v->param('cutoff');
$query{context} = $v->param('context') // '40-t,40-t'; # 'base/s:p'/'paragraph'
+ my $items_per_page = $c->items_per_page;
+
+ # Set count
+ if ($query{count} && $query{count} <= $c->items_per_page ) {
+ $items_per_page = delete $query{count};
+ };
+
+ $c->stash(items_per_page => $items_per_page);
+
+ # Set offset
+ # From Mojolicious::Plugin::Search::Index
+ $query{o} = $v->param('o') || ((($query{p} // 1) - 1) * ($items_per_page || 1));
+
+
+ # already set by stash - or use plugin param
+ # else {
+ # $items_per_page = $c->stash('search.count') // $plugin->items_per_page
+ # };
+
+ # Set start page based on param
+ #if ($query{p}) {
+ # $index->start_page(delete $param{start_page});
+ #}
+ ## already set by stash
+ #elsif ($c->stash('search.start_page')) {
+ # $index->start_page($c->stash('search.start_page'));
+ #};
+
# Create remote request URL
my $url = Mojo::URL->new($c->api);
@@ -232,10 +239,14 @@
$url->query(\%query);
# Check if total results is cached
- my $total_results;
- if (!$c->no_cache && 0) {
+ my $total_results = -1;
+ unless ($c->no_cache) {
+
+ # Get total results value
$total_results = $c->chi->get('total-' . $user . '-' . $url->to_string);
- $c->stash(total_results => $total_results);
+
+ # Set stash if cache exists
+ $c->stash(total_results => $total_results) if $total_results;
$c->app->log->debug('Get total result from cache');
# Set cutoff unless already set
@@ -276,7 +287,7 @@
my $url_string = $url->to_string;
# Set api request for debugging
- # $c->stash('api_request' => $url_string);
+ $c->stash('api_request' => $url_string);
# Debugging
$c->app->log->debug('Search for ' . $url_string);
@@ -300,36 +311,67 @@
# Wrap a user agent method with a promise
$promise = $c->user->auth_request_p(get => $url)
+ # TODO: Better use a single then
->then(\&_catch_http_errors)
+ ->then(
+ sub {
+ my $json = shift->json;
+
+ unless ($json) {
+ return Mojo::Promise->new->reject([
+ [undef, 'JSON response is invalid']
+ ]);
+ };
+
+ $c->stash('api_response' => $json);
+ return $json;
+ })
->then(\&_catch_koral_errors)
;
};
+ # Wait for rendering
$c->render_later;
- # Prepare warnings
+ # Process response
$promise->then(
sub {
my $json = shift;
- $c->_notify_on_warnings($json->{warnings}) if $json->{warnings};
- return $json
- }
- # Process response
- )->then(
- sub {
- my $json = shift;
+ # Prepare warnings
+ $c->_notify_on_warnings($json->{warnings}) if $json->{warnings};
# Cache total results
- unless ($c->stash('total_results') && $json->{meta}->{totalResults}) {
+ # The stash is set in case the total results value is from the cache,
+ # so in that case, it does not need to be cached again
+ my $total_results = $c->stash('total_results');
+ if (!$total_results) {
- # Remove cutoff requirement again
- $url->query([cutoff => 'true']);
+ # There are results to remember
+ if ($json->{meta}->{totalResults} >= 0) {
- # Set cache
- $c->chi->set(
- 'total-' . $user . '-' . $url->to_string => $json->{meta}->{totalResults}
- )
+ # Remove cutoff requirement again
+ $url->query([cutoff => 'true']);
+
+ $total_results = $json->{meta}->{totalResults};
+ $c->stash(total_results => $total_results);
+
+ # Set cache
+ $c->chi->set(
+ 'total-' . $user . '-' . $url->to_string => $total_results
+ );
+ };
+ }
+ else {
+ $c->stash(total_results => -1);
+ }
+
+ $c->stash(total_pages => 0);
+
+ # Set total pages
+ # From Mojolicious::Plugin::Search::Index
+ if ($total_results > 0) {
+ $c->stash(total_pages => ceil($total_results / ($c->stash('items_per_page') || 1)));
};
# Cache result
@@ -351,19 +393,11 @@
# Choose the snippet based on the parameter
my $template = scalar $v->param('snippet') ? 'snippet' : 'search2';
- $c->app->log->debug('Render template ...');
-
return $c->render(
template => $template,
q => $query,
- ql => scalar $v->param('ql') // 'poliqarp',
- results => c(),
- start_page => 1,
- total_pages => 20,
- # total_results => 40,
- time_exceeded => 0,
- benchmark => 'Long ...',
- api_response => ''
+ ql => $query{ql},
+ start_page => $query{p},
);
}
)->wait;
diff --git a/t/fixtures/response_baum.json b/t/fixtures/response_baum.json
index 3ad2ae3..a1727e3 100644
--- a/t/fixtures/response_baum.json
+++ b/t/fixtures/response_baum.json
@@ -57,7 +57,7 @@
{
"field" : "tokens",
"pubPlace" : "München",
- "textSigle" : "GOE/AGI/00000",
+ "textSigle" : "GOE/AGI/00001",
"docSigle" : "GOE/AGI",
"corpusSigle" : "GOE",
"title" : "Italienische Reise",
diff --git a/t/query.t b/t/query.t
index 75ed9b8..bf82ba2 100644
--- a/t/query.t
+++ b/t/query.t
@@ -31,9 +31,40 @@
->text_is('title', 'KorAP: Find »baum« with Poliqarp')
->element_exists('meta[name="DC.title"][content="KorAP: Find »baum« with Poliqarp"]')
->element_exists('body[itemscope][itemtype="http://schema.org/SearchResultsPage"]')
+
+ # Total results
->text_is('#total-results', 51)
+
+ # Total pages
+ ->element_count_is('#pagination a', 5)
+
+ # api_response
+ ->content_like(qr/\"authorized\":null/)
+ ->content_like(qr/\"pubDate\",\"subTitle\",\"author\"/)
+
+ ->element_exists('li[data-text-sigle=GOE/AGI/00000]')
+ ->element_exists('li:nth-of-type(1) div.flop')
+ ->element_exists('li[data-text-sigle=GOE/AGI/00001]')
+ ->element_exists('li:nth-of-type(2) div.flip')
+
+ # Match1
+ ->element_exists('li:nth-of-type(1)' .
+ '[data-match-id="p2030-2031"]' .
+ '[data-text-sigle="GOE/AGI/00000"]' .
+ '[id="GOE/AGI/00000#p2030-2031"]' .
+ '[data-available-info^="base/s=spans"]' .
+ '[data-info^="{"]')
+ ->text_is('li:nth-of-type(1) div.meta', 'GOE/AGI/00000')
+ ->element_exists('li:nth-of-type(1) div.match-main div.match-wrap div.snippet')
+ ->element_exists('li:nth-of-type(1) div.snippet.startMore.endMore')
+ ->text_like('li:nth-of-type(1) div.snippet span.context-left',qr!sie etwas bedeuten!)
+ ->text_like('li:nth-of-type(1) div.snippet span.context-left',qr!sie etwas bedeuten!)
+ ->text_is('li:nth-of-type(1) div.snippet span.match mark','Baum')
+ ->text_like('li:nth-of-type(1) div.snippet span.context-right',qr!es war!)
+ ->text_is('li:nth-of-type(1) p.ref strong', 'Italienische Reise')
+ ->text_like('li:nth-of-type(1) p.ref', qr!by Goethe, Johann Wolfgang!)
+ ->text_is('li:nth-of-type(1) p.ref time[datetime=1982]', 1982)
+ ->text_is('li:nth-of-type(1) p.ref span.sigle', '[GOE/AGI/00000]')
;
-
-
done_testing;
diff --git a/templates/match.html.ep b/templates/match.html.ep
index dbc6e4f..038a730 100644
--- a/templates/match.html.ep
+++ b/templates/match.html.ep
@@ -4,15 +4,13 @@
% delete @match_data{qw/snippet startMore endMore field/};
% my $text_sigle = $match->{textSigle} // join('/', $match->{corpusID}, $match->{docID}, $match->{textID});
% my $id = $text_sigle . '#' . $match->{matchID};
+% # Legacy:
+% my $layer_infos = $match->{layerInfos} // $match->{layerInfo} // 'cnx/c=spans corenlp/ne=tokens corenlp/p=tokens mate/l=tokens mate/m=tokens mate/p=tokens opennlp/p=tokens tt/l=tokens tt/p=tokens xip/c=spans';
<li data-match-id="<%= $match->{matchID} %>"
data-text-sigle="<%= $text_sigle %>"
-
- %# TODO: This needs to be retrieved per match
- data-available-info="<%= $match->{layerInfos} // $match->{layerInfo} // 'cnx/c=spans corenlp/ne=tokens corenlp/p=tokens mate/l=tokens mate/m=tokens mate/p=tokens opennlp/p=tokens tt/l=tokens tt/p=tokens xip/c=spans' %>"
+ data-available-info="<%= $layer_infos %>"
data-info="<%== b(encode_json(\%match_data))->decode->xml_escape %>"
- id="<%= $id %>"\
- <% if (current_route eq 'match') { %> class="active"<% } =%>>
-
+ id="<%= $id %>"<% if (current_route eq 'match') { %> class="active"<% } =%>>
%# This should be done using JavaScript
% my ($show_sigle, $flip) = ('', stash('flip') // 'flip');
% if ($text_sigle ne (stash('last_sigle') // '')) {
@@ -20,8 +18,7 @@
% stash(last_sigle => $text_sigle);
% $flip = $flip eq 'flip' ? 'flop' : 'flip';
% stash(flip => $flip);
-% }
-
+% }
<div class="meta <%= $flip %>"><%= $show_sigle %></div>
<div class="match-main">
<div class="match-wrap">