Nils Diewald | 7148c6f | 2015-05-04 15:07:53 +0000 | [diff] [blame] | 1 | package Kalamar::API; |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 2 | use Mojo::Base 'Mojolicious::Plugin'; |
Nils Diewald | 791b590 | 2014-12-04 04:47:24 +0000 | [diff] [blame] | 3 | use Scalar::Util qw/blessed weaken/; |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 4 | use strict; |
| 5 | use warnings; |
| 6 | |
| 7 | # KorAP Search engine for Mojolicious::Plugin::Search |
| 8 | |
Nils Diewald | 7148c6f | 2015-05-04 15:07:53 +0000 | [diff] [blame] | 9 | # TODO: Add fixtures |
| 10 | # TODO: Support search in corpus and virtualcollection |
| 11 | # TODO: Support caching everywhere! |
| 12 | # TODO: Correct use of stash info everywhere! |
Akron | f809da2 | 2015-06-18 22:08:19 +0200 | [diff] [blame] | 13 | # TODO: Alot is now underneath "meta" |
| 14 | |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 15 | |
| 16 | # Register the plugin |
| 17 | sub register { |
| 18 | my ($plugin, $mojo, $index_class, $param) = @_; |
| 19 | $param ||= {}; |
| 20 | |
| 21 | # Add attributes to the index class |
| 22 | $index_class->attr(api => $param->{api}); |
| 23 | $index_class->attr([qw/cutoff |
| 24 | query_language |
| 25 | time_exceeded |
| 26 | api_request |
Akron | bc213c0 | 2017-04-20 16:45:55 +0200 | [diff] [blame] | 27 | authorized |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 28 | _api_cache |
| 29 | api_response |
| 30 | benchmark |
Akron | 9cc3eaf | 2015-06-10 22:15:52 +0200 | [diff] [blame] | 31 | query_jsonld |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 32 | collection |
Akron | 9cc3eaf | 2015-06-10 22:15:52 +0200 | [diff] [blame] | 33 | collection_jsonld/]); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 34 | $index_class->attr(no_cache => 0); |
Akron | 515851a | 2017-05-02 12:53:17 +0200 | [diff] [blame] | 35 | $index_class->attr(status => 200); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 36 | }; |
| 37 | |
| 38 | |
| 39 | # Search the index |
| 40 | sub search { |
| 41 | my $self = shift; |
| 42 | my $index = shift; |
| 43 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 44 | # Get controller |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 45 | my $c = $index->controller; |
| 46 | |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 47 | # If there is a callback, do async |
| 48 | my $cb = pop if ref $_[-1] && ref $_[-1] eq 'CODE'; |
| 49 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 50 | # No query defined |
| 51 | unless ($index->query) { |
| 52 | return $cb->($index) if $cb; |
| 53 | return; |
| 54 | }; |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 55 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 56 | # Get query url |
| 57 | my $url = _query_url($index, @_); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 58 | |
| 59 | # Cache based on URL |
| 60 | $index->_api_cache('total-' . $url->to_string); |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 61 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 62 | my %param = @_; |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 63 | |
| 64 | # Set context based on parameter |
Akron | 30ee514 | 2015-06-26 01:50:14 +0200 | [diff] [blame] | 65 | # base/s:p |
Akron | b7bae28 | 2016-02-10 01:17:25 +0100 | [diff] [blame] | 66 | $url->query({ context => $param{'context'} // '40-t,40-t' }); |
| 67 | # 'base/s:p'/'paragraph' |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 68 | |
| 69 | # Set path to search |
| 70 | $url->path('search'); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 71 | |
| 72 | # Check cache for total results |
| 73 | my $total_results; |
| 74 | |
Akron | 1b0c265 | 2017-04-27 15:28:49 +0200 | [diff] [blame] | 75 | # In case the user is not known, it is assumed, the user is not logged in |
| 76 | my $user = $c->stash('user') // 'not_logged_in'; |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 77 | |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 78 | if (!$index->no_cache && |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 79 | defined ($total_results = $c->chi->get($user . $index->_api_cache))) { |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 80 | |
| 81 | # Set total results from cache |
| 82 | $index->total_results($total_results); |
| 83 | $c->app->log->debug('Get total result from cache'); |
| 84 | |
| 85 | # Set cutoff unless already set |
| 86 | $url->query({cutoff => 'true'}) unless defined $index->cutoff; |
| 87 | }; |
| 88 | |
| 89 | # Set api request for debugging |
| 90 | $index->api_request($url->to_string); |
| 91 | |
| 92 | # Create new user agent and set timeout to 2 minutes |
Akron | bc213c0 | 2017-04-20 16:45:55 +0200 | [diff] [blame] | 93 | #my $ua = $c->user->ua; |
| 94 | #$tx = $plugin->ua->start($tx); |
| 95 | |
| 96 | #$ua->inactivity_timeout(120); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 97 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 98 | # Debugging |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 99 | $c->app->log->debug('Search for ' . $index->api_request); |
| 100 | |
| 101 | # Search non-blocking |
| 102 | if ($cb) { |
Akron | bc213c0 | 2017-04-20 16:45:55 +0200 | [diff] [blame] | 103 | $c->user->auth_request( |
| 104 | get => $url => sub { |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 105 | my $tx = pop; |
Akron | bc213c0 | 2017-04-20 16:45:55 +0200 | [diff] [blame] | 106 | |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 107 | $self->_process_response('matches', $index, $tx); |
| 108 | weaken $index; |
| 109 | return $cb->($index); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 110 | }); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 111 | } |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 112 | |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 113 | # Search blocking |
| 114 | else { |
Akron | eb25016 | 2017-04-28 17:26:49 +0200 | [diff] [blame] | 115 | my $tx = $c->user->auth_request(get => $url); |
Nils Diewald | 034ea70 | 2015-01-16 19:41:52 +0000 | [diff] [blame] | 116 | $self->_process_response('matches', $index, $tx); |
| 117 | return $index; |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 118 | }; |
| 119 | }; |
| 120 | |
| 121 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 122 | # Trace query serialization |
| 123 | sub trace { |
| 124 | my $self = shift; |
| 125 | my $index = shift; |
| 126 | |
| 127 | # Get controller |
| 128 | my $c = $index->controller; |
| 129 | |
| 130 | # If there is a callback, do async |
| 131 | my $cb = pop if ref $_[-1] && ref $_[-1] eq 'CODE'; |
| 132 | |
| 133 | my %param = @_; |
| 134 | |
| 135 | # No query defined |
| 136 | unless ($index->query(delete $param{query})) { |
| 137 | return $cb->($index) if $cb; |
| 138 | return; |
| 139 | }; |
| 140 | |
| 141 | # Get query url |
| 142 | my $url = _query_url($index, @_); |
| 143 | |
| 144 | $url->path('search'); |
| 145 | |
| 146 | # Create new user agent and set timeout to 30 seconds |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 147 | my $ua = $c->user->ua; # Mojo::UserAgent->new; |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 148 | $ua->inactivity_timeout(30); |
| 149 | |
| 150 | # Build transaction |
| 151 | my $tx = $ua->build_tx(TRACE => $url); |
| 152 | |
| 153 | # non-blocking |
| 154 | if ($cb) { |
Nils Diewald | 791b590 | 2014-12-04 04:47:24 +0000 | [diff] [blame] | 155 | weaken $index; |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 156 | |
| 157 | # Trace non-blocking |
| 158 | $ua->start( |
| 159 | $tx => sub { |
Akron | 1b0c265 | 2017-04-27 15:28:49 +0200 | [diff] [blame] | 160 | $self->_process_response('trace', $index, pop); |
| 161 | return $cb->($index); |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 162 | }); |
| 163 | } |
| 164 | # Trace blocking |
| 165 | else { |
| 166 | my $tx = $ua->start($url); |
| 167 | return $self->_process_response('trace', $index, $tx); |
| 168 | }; |
| 169 | }; |
| 170 | |
| 171 | |
| 172 | # Get match info |
| 173 | sub match { |
| 174 | my $self = shift; |
| 175 | my $index = shift; |
| 176 | |
| 177 | # Get controller |
| 178 | my $c = $index->controller; |
| 179 | |
| 180 | # If there is a callback, do async |
| 181 | my $cb = pop if ref $_[-1] && ref $_[-1] eq 'CODE'; |
| 182 | |
| 183 | my %param = @_; |
| 184 | |
| 185 | my $url = Mojo::URL->new($index->api); |
| 186 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 187 | # Legacy: In old versions, doc_id contained text_id |
Akron | eb25016 | 2017-04-28 17:26:49 +0200 | [diff] [blame] | 188 | # $param{doc_id} .= '.' . $param{text_id} if $param{text_id}; |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 189 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 190 | # Use hash slice to create path |
Akron | eb25016 | 2017-04-28 17:26:49 +0200 | [diff] [blame] | 191 | $url->path(join('/', 'corpus', @param{qw/corpus_id doc_id text_id match_id/}, 'matchInfo')); |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 192 | |
| 193 | # Build match id |
| 194 | # $match = 'match-' . $corpus . '!' . $corpus . '_' . $doc . '-' . $match; |
| 195 | |
| 196 | my %query; |
| 197 | $query{foundry} = $param{foundry}; |
| 198 | $query{layer} = $param{layer} if defined $param{layer}; |
| 199 | $query{spans} = $param{spans} ? 'true' : 'false'; |
| 200 | |
| 201 | # Add query |
| 202 | $url->query(\%query); |
| 203 | |
| 204 | $c->app->log->debug('Match info: ' . $url); |
| 205 | |
| 206 | # Create new user agent and set timeout to 30 seconds |
Akron | 515851a | 2017-05-02 12:53:17 +0200 | [diff] [blame] | 207 | # my $ua = $c->user->ua; # Mojo::UserAgent->new; |
| 208 | # $ua->inactivity_timeout(30); |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 209 | |
| 210 | # non-blocking |
| 211 | if ($cb) { |
Akron | eb25016 | 2017-04-28 17:26:49 +0200 | [diff] [blame] | 212 | # $c->u |
| 213 | $c->user->auth_request(get => |
| 214 | # $ua->get( |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 215 | $url => sub { |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 216 | my $tx = pop; |
| 217 | $self->_process_response('match', $index, $tx); |
Akron | 515851a | 2017-05-02 12:53:17 +0200 | [diff] [blame] | 218 | weaken $index; |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 219 | return $cb->($index); |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 220 | }); |
| 221 | } |
| 222 | |
| 223 | # Match info blocking |
| 224 | else { |
Akron | eb25016 | 2017-04-28 17:26:49 +0200 | [diff] [blame] | 225 | my $tx = $c->user->auth_request(get => $url); |
Akron | 515851a | 2017-05-02 12:53:17 +0200 | [diff] [blame] | 226 | # my $tx = $ua->get($url); |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 227 | return $self->_process_response('match', $index, $tx); |
| 228 | }; |
| 229 | }; |
| 230 | |
| 231 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 232 | # Get resource information |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 233 | sub resource { |
| 234 | my $self = shift; |
| 235 | my $index = shift; |
| 236 | |
| 237 | # Get controller |
| 238 | my $c = $index->controller; |
| 239 | |
Akron | 1b0c265 | 2017-04-27 15:28:49 +0200 | [diff] [blame] | 240 | my $user = $c->stash('user') // 'not_logged_in'; |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 241 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 242 | # If there is a callback, do async |
| 243 | my $cb = pop if ref $_[-1] && ref $_[-1] eq 'CODE'; |
| 244 | |
| 245 | my %param = @_; |
| 246 | |
| 247 | # Rename info endpoints regarding resource |
| 248 | my $type = $param{type} // 'collection'; |
| 249 | $type = 'virtualcollection' if $type eq 'collection'; |
| 250 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 251 | # Create resource URL |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 252 | my $url = Mojo::URL->new($index->api)->path($type); |
| 253 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 254 | # Debugging |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 255 | $c->app->log->debug('Get resource info on '. $url); |
| 256 | |
| 257 | # Check for cached information |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 258 | if (my $json = $c->chi->get($user . $url->to_string)) { |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 259 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 260 | # TODO: That's unfortunate, as it prohibits caching of multiple resources |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 261 | $c->app->log->debug('Get resource info from cache'); |
| 262 | $c->stash('search.resource' => $json); |
| 263 | return $cb->($index) if $cb; |
| 264 | return $json; |
| 265 | }; |
| 266 | |
| 267 | $c->stash('search._resource_cache' => $url->to_string); |
| 268 | |
| 269 | # Create new user agent and set timeout to 30 seconds |
Akron | eb25016 | 2017-04-28 17:26:49 +0200 | [diff] [blame] | 270 | #my $ua = $c->ua; # Mojo::UserAgent->new; |
| 271 | #$ua->inactivity_timeout(30); |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 272 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 273 | # Get resource information async |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 274 | if ($cb) { |
Akron | eb25016 | 2017-04-28 17:26:49 +0200 | [diff] [blame] | 275 | $c->user->auth_request(get => |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 276 | $url => sub { |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 277 | $self->_process_response('resource', $index, pop); |
Akron | eb25016 | 2017-04-28 17:26:49 +0200 | [diff] [blame] | 278 | weaken $index; |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 279 | return $cb->($index); |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 280 | }) |
| 281 | } |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 282 | |
| 283 | # Get resource information blocking |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 284 | else { |
Akron | eb25016 | 2017-04-28 17:26:49 +0200 | [diff] [blame] | 285 | my $tx = $c->user->auth_request(get => $url); |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 286 | $self->_process_response('resource', $index, $tx); |
| 287 | }; |
| 288 | }; |
| 289 | |
| 290 | |
| 291 | # Process response - especially error messages etc. |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 292 | sub _process_response { |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 293 | my ($self, $type, $index, $tx) = @_; |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 294 | my $c = $index->controller; |
| 295 | |
| 296 | # An error has occurded |
| 297 | if (my $e = $tx->error) { |
Akron | 515851a | 2017-05-02 12:53:17 +0200 | [diff] [blame] | 298 | |
| 299 | # Send error |
| 300 | $self->_notify_on_error($c, 0, $tx->res->json); |
| 301 | |
| 302 | # $c->notify( |
| 303 | # error => |
| 304 | # ($e->{code} ? $e->{code} . ': ' : '') . |
| 305 | # $e->{message} . ' for ' . $type . ' (remote)' |
| 306 | # ); |
| 307 | $index->status($e->{code} // 0); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 308 | return; |
| 309 | }; |
| 310 | |
| 311 | # Response was fine |
| 312 | if (my $res = $tx->success) { |
| 313 | |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 314 | # Json failure |
| 315 | my $json; |
| 316 | unless ($json = $res->json) { |
| 317 | $c->notify(error => 'JSON response is invalid'); |
Akron | 515851a | 2017-05-02 12:53:17 +0200 | [diff] [blame] | 318 | $index->status(0); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 319 | return; |
| 320 | }; |
| 321 | |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 322 | # Set api response as jsonld |
| 323 | $index->api_response($json); |
| 324 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 325 | # expected response for matches |
| 326 | if ($type eq 'matches') { |
| 327 | $self->_process_response_matches($index, $json); |
| 328 | } |
| 329 | elsif ($type eq 'trace') { |
| 330 | $self->_process_response_trace($index, $json); |
| 331 | } |
| 332 | elsif ($type eq 'match') { |
| 333 | $self->_process_response_match($index, $json); |
| 334 | } |
| 335 | elsif ($type eq 'resource') { |
| 336 | $self->_process_response_resource($index, $json); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 337 | }; |
| 338 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 339 | return 1 if ref $json ne 'HASH'; |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 340 | |
Akron | f55504a | 2015-06-18 16:42:55 +0200 | [diff] [blame] | 341 | $self->_notify_on_warnings($c, $json); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 342 | $self->_notify_on_error($c, 0, $json); |
| 343 | } |
| 344 | |
| 345 | # Request failed |
| 346 | else { |
Akron | 515851a | 2017-05-02 12:53:17 +0200 | [diff] [blame] | 347 | $index->status(0); |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 348 | $self->_notify_on_error($c, 1, $tx->res); |
| 349 | }; |
| 350 | return 1; |
| 351 | }; |
| 352 | |
| 353 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 354 | # Handle match results |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 355 | sub _process_response_matches { |
| 356 | my ($self, $index, $json) = @_; |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 357 | |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 358 | # Process meta |
| 359 | my $meta = $json->{meta}; |
| 360 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 361 | # Reformat benchmark counter |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 362 | my $benchmark = $meta->{benchmark}; |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 363 | if ($benchmark && $benchmark =~ s/\s+(m)?s$//) { |
| 364 | $benchmark = sprintf("%.2f", $benchmark) . ($1 ? $1 : '') . 's'; |
| 365 | }; |
| 366 | |
| 367 | # Set benchmark |
| 368 | $index->benchmark($benchmark); |
| 369 | |
| 370 | # Set time exceeded |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 371 | if ($meta->{timeExceeded} && $meta->{timeExceeded} eq Mojo::JSON::true) { |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 372 | $index->time_exceeded(1); |
| 373 | }; |
| 374 | |
| 375 | # Set result values |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 376 | $index->items_per_page($meta->{itemsPerPage}); |
| 377 | |
Akron | bc213c0 | 2017-04-20 16:45:55 +0200 | [diff] [blame] | 378 | # Set authorization |
| 379 | $index->authorized($meta->{authorized}) if $meta->{authorized}; |
Akron | 9cc3eaf | 2015-06-10 22:15:52 +0200 | [diff] [blame] | 380 | |
Akron | c1457bf | 2015-06-11 19:24:00 +0200 | [diff] [blame] | 381 | # Bouncing query |
Akron | bc213c0 | 2017-04-20 16:45:55 +0200 | [diff] [blame] | 382 | # if ($json->{query}) { |
| 383 | # $index->query_jsonld($json->{query}); |
| 384 | # }; |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 385 | |
Akron | c1457bf | 2015-06-11 19:24:00 +0200 | [diff] [blame] | 386 | # Legacy |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 387 | # elsif ($json->{request}->{query}) { |
| 388 | # $index->query_jsonld($json->{request}->{query}); |
| 389 | # }; |
Akron | c1457bf | 2015-06-11 19:24:00 +0200 | [diff] [blame] | 390 | |
| 391 | # Bouncing collection query |
| 392 | if ($json->{collection}) { |
| 393 | $index->collection_jsonld($json->{collection}); |
| 394 | } |
| 395 | |
| 396 | # Legacy |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 397 | # elsif ($json->{request}->{collection}) { |
| 398 | # $index->collection_jsonld($json->{request}->{collection}); |
| 399 | # }; |
Akron | 48b1e4d | 2015-06-17 18:47:01 +0200 | [diff] [blame] | 400 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 401 | $index->results(_map_matches($json->{matches})); |
| 402 | |
| 403 | # Total results not set by stash |
| 404 | if ($index->total_results == -1) { |
| 405 | |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 406 | if ($meta->{totalResults} && $meta->{totalResults} > -1) { |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 407 | my $c = $index->controller; |
Akron | 089ae2c | 2017-04-17 17:59:23 +0200 | [diff] [blame] | 408 | |
| 409 | # TODO: Cache on auth_keys! |
Akron | 1b0c265 | 2017-04-27 15:28:49 +0200 | [diff] [blame] | 410 | my $user = $c->stash('user') // 'not_logged_in'; |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 411 | |
| 412 | $c->app->log->debug('Cache total result'); |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 413 | $c->chi->set($user . $index->_api_cache => $meta->{totalResults}, '120min'); |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 414 | $index->total_results($meta->{totalResults}); |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 415 | }; |
| 416 | }; |
| 417 | }; |
| 418 | |
| 419 | |
| 420 | # Process query serialization response |
| 421 | sub _process_response_match { |
| 422 | my ($self, $index, $json) = @_; |
| 423 | $index->results(_map_match($json)); |
| 424 | }; |
| 425 | |
| 426 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 427 | # Process trace response |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 428 | sub _process_response_trace { |
| 429 | my ($self, $index, $json) = @_; |
| 430 | $index->query_jsonld($json); |
| 431 | }; |
| 432 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 433 | |
| 434 | # Process resource response |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 435 | sub _process_response_resource { |
| 436 | my ($self, $index, $json) = @_; |
| 437 | my $c = $index->controller; |
| 438 | |
Akron | 1b0c265 | 2017-04-27 15:28:49 +0200 | [diff] [blame] | 439 | my $user = $c->stash('user') // 'not_logged_in'; |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 440 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 441 | # TODO: That's unfortunate, as it prohibits multiple resources |
| 442 | $c->stash('search.resource' => $json); |
| 443 | $c->app->log->debug('Cache resource info'); |
Akron | 3cd391e | 2017-03-29 23:42:54 +0200 | [diff] [blame] | 444 | $c->chi->set($user . $c->stash('search._resource_cache') => $json, '24 hours'); |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 445 | }; |
| 446 | |
| 447 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 448 | # Parse error messages and forward them to the user |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 449 | sub _notify_on_error { |
| 450 | my ($self, $c, $failure, $res) = @_; |
| 451 | my $json = $res; |
| 452 | |
| 453 | my $log = $c->app->log; |
| 454 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 455 | # Check if the response is already json |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 456 | if (blessed $res) { |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 457 | $json = $res->json if blessed $res ne 'Mojo::JSON'; |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 458 | }; |
| 459 | |
Akron | f55504a | 2015-06-18 16:42:55 +0200 | [diff] [blame] | 460 | # Check json response error message |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 461 | if ($json) { |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 462 | |
| 463 | # Legacy, but still in use by Kustvakt |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 464 | if ($json->{error}) { |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 465 | |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 466 | # Temp |
| 467 | $json->{error} =~ s/;\s+null$//; |
| 468 | $c->notify(error => $json->{error}); |
| 469 | return; |
| 470 | } |
| 471 | |
| 472 | # New error messages |
| 473 | elsif ($json->{errstr}) { |
| 474 | # Temp |
| 475 | $json->{errstr} =~ s/;\s+null$//; |
| 476 | $c->notify(error => $json->{errstr}); |
| 477 | return; |
| 478 | } |
| 479 | |
Akron | f55504a | 2015-06-18 16:42:55 +0200 | [diff] [blame] | 480 | elsif ($json->{errors}) { |
| 481 | my $errors = $json->{errors}; |
| 482 | # TODO: Check for ref! |
| 483 | foreach (@$errors) { |
Akron | bc213c0 | 2017-04-20 16:45:55 +0200 | [diff] [blame] | 484 | $c->notify( |
| 485 | error => |
| 486 | ($_->[0] ? $_->[0] . ': ' : '') . |
Akron | 515851a | 2017-05-02 12:53:17 +0200 | [diff] [blame] | 487 | ($_->[1] || 'Unknown') |
Akron | bc213c0 | 2017-04-20 16:45:55 +0200 | [diff] [blame] | 488 | ); |
Akron | f55504a | 2015-06-18 16:42:55 +0200 | [diff] [blame] | 489 | }; |
| 490 | } |
| 491 | |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 492 | # policy service error messages |
| 493 | elsif ($json->{status}) { |
| 494 | $c->notify(error => 'Middleware error ' . $json->{status}); |
| 495 | return; |
| 496 | }; |
| 497 | }; |
| 498 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 499 | # Doesn't matter what - there is a failure! |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 500 | if ($failure) { |
| 501 | $c->notify(error => ( |
| 502 | ($res->{code} ? $res->{code} . ': ' : '') . |
| 503 | ($res->{message} ? $res->{message} : 'Unknown error') . |
| 504 | ' (remote)' |
| 505 | )); |
| 506 | }; |
| 507 | }; |
| 508 | |
| 509 | |
Akron | f55504a | 2015-06-18 16:42:55 +0200 | [diff] [blame] | 510 | sub _notify_on_warnings { |
| 511 | my ($self, $c, $json) = @_; |
| 512 | |
| 513 | # Add warnings (Legacy) |
| 514 | if ($json->{warning}) { |
| 515 | $json->{warning} =~ s/;\s+null$//; |
| 516 | $c->notify(warn => $json->{warning}); |
| 517 | } |
| 518 | |
| 519 | # Add warnings |
| 520 | elsif ($json->{warnings}) { |
| 521 | |
| 522 | my $warnings = $json->{warnings}; |
| 523 | # TODO: Check for ref! |
| 524 | foreach (@$warnings) { |
| 525 | $c->notify( |
Akron | 46b9f21 | 2017-05-01 13:55:17 +0200 | [diff] [blame] | 526 | warn => |
| 527 | ($_->[0] ? $_->[0] . ': ' : '') . |
| 528 | $_->[1] |
| 529 | ); |
Akron | f55504a | 2015-06-18 16:42:55 +0200 | [diff] [blame] | 530 | }; |
| 531 | }; |
| 532 | }; |
| 533 | |
| 534 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 535 | # Cleanup array of matches |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 536 | sub _map_matches { |
| 537 | return () unless $_[0]; |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 538 | map { _map_match($_) } @{ shift() }; |
| 539 | }; |
| 540 | |
| 541 | |
| 542 | # Cleanup single match |
| 543 | sub _map_match { |
| 544 | my $x = shift or return; |
Akron | 7f613e0 | 2016-11-07 02:50:44 +0100 | [diff] [blame] | 545 | $x->{matchID} =~ s/^match\-(?:[^!]+!|[^_]+_)[^\.]+?\.[^-]+?-// or |
| 546 | $x->{matchID} =~ s!^match\-(?:[^\/]+\/){2}[^-]+?-!!; |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 547 | |
Akron | 30ee514 | 2015-06-26 01:50:14 +0200 | [diff] [blame] | 548 | ( |
| 549 | $x->{corpusID}, |
| 550 | $x->{docID}, |
| 551 | $x->{textID} |
| 552 | ) = ($x->{textSigle} =~ /^([^_]+?)_+([^\.]+?)\.(.+?)$/); |
| 553 | |
| 554 | # $x->{docID} =~ s/^[^_]+_//; |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 555 | # Legacy: In old versions the text_id was part of the doc_id |
Akron | 30ee514 | 2015-06-26 01:50:14 +0200 | [diff] [blame] | 556 | # unless ($x->{textID}) { |
| 557 | # ($x->{docID}, $x->{textID}) = split '\.', $x->{docID}; |
| 558 | # }; |
| 559 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 560 | $x; |
| 561 | }; |
| 562 | |
Nils Diewald | 8750783 | 2015-05-01 23:36:41 +0000 | [diff] [blame] | 563 | # Build query url |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 564 | sub _query_url { |
| 565 | my ($index, %param) = @_; |
| 566 | |
| 567 | # Set cutoff from param |
| 568 | $index->cutoff(delete $param{cutoff}); |
| 569 | |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 570 | # Set collection from param |
| 571 | $index->collection(delete $param{collection}); |
| 572 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 573 | # Set query language |
| 574 | $index->query_language(delete $param{query_language} // 'poliqarp'); |
| 575 | |
| 576 | # Should results be cached? Defaults to "yes" |
| 577 | $index->no_cache(1) if $param{no_cache}; |
| 578 | |
| 579 | # Init the query with stuff coming from the index |
| 580 | my %query; |
| 581 | $query{q} = $index->query; |
| 582 | $query{ql} = $index->query_language; |
| 583 | $query{page} = $index->start_page if $index->start_page; |
| 584 | $query{count} = $index->items_per_page if $index->items_per_page; |
Akron | 27ae9ec | 2015-06-23 00:43:21 +0200 | [diff] [blame] | 585 | $query{cq} = $index->collection if $index->collection; |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 586 | $query{cutoff} = 'true' if $index->cutoff; |
| 587 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 588 | # Create query url |
| 589 | my $url = Mojo::URL->new($index->api); |
| 590 | $url->query(\%query); |
| 591 | return $url; |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 592 | }; |
| 593 | |
| 594 | |
| 595 | 1; |
| 596 | |
Nils Diewald | 8f4b5da | 2014-12-03 22:13:39 +0000 | [diff] [blame] | 597 | |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 598 | __END__ |
| 599 | |
| 600 | =pod |
| 601 | |
Nils Diewald | 9dfe010 | 2015-05-19 16:14:06 +0000 | [diff] [blame] | 602 | =encoding utf8 |
Nils Diewald | 996aa55 | 2014-12-02 03:26:44 +0000 | [diff] [blame] | 603 | |
Nils Diewald | 9dfe010 | 2015-05-19 16:14:06 +0000 | [diff] [blame] | 604 | =head1 NAME |
| 605 | |
| 606 | Kalamar::API |
| 607 | |
| 608 | =head1 DESCRIPTION |
| 609 | |
| 610 | L<Kalamar::API> is a search engine class for L<Mojolicious::Plugin::Search> |
| 611 | that uses the KorAP Web API. |
| 612 | |
| 613 | B<The Web API as well as L<Mojolicious::Plugin::Search> are not stable yet, |
| 614 | so this class is expected to change in the near future. Do not rely on its API!> |
| 615 | |
| 616 | |
| 617 | =head1 METHODS |
| 618 | |
| 619 | L<Kalamar::API> inherits all methods from L<Mojolicious::Plugin> and |
| 620 | implements the following new ones. |
| 621 | |
| 622 | |
| 623 | =head2 register |
| 624 | |
| 625 | See L<Mojolicious::Plugin::Search> for registering search engines. |
| 626 | In addition to the mentioned query parameters, the following parameters are supported: |
| 627 | |
| 628 | |
| 629 | =over 2 |
| 630 | |
| 631 | =item B<query_language> |
| 632 | |
| 633 | One of the supported query languages, like C<poliqarp> or C<annis>. |
| 634 | |
| 635 | |
| 636 | =item B<cutoff> |
| 637 | |
| 638 | Cut off results following the current page (i.e. don't count the number of results). |
| 639 | |
| 640 | |
| 641 | =item B<no_cache> |
| 642 | |
| 643 | Do not cache search results. Defaults to C<0>. |
| 644 | |
| 645 | |
| 646 | =back |
| 647 | |
| 648 | In addition to the mentioned index attributes, the following attributes are supported: |
| 649 | |
Akron | 456abd9 | 2015-06-02 15:07:21 +0200 | [diff] [blame] | 650 | =over 2 |
Nils Diewald | 9dfe010 | 2015-05-19 16:14:06 +0000 | [diff] [blame] | 651 | |
| 652 | =item B<api> |
| 653 | |
| 654 | The API address. |
| 655 | |
| 656 | |
| 657 | =item B<time_exceeded> |
| 658 | |
| 659 | Report on time outs, that may mean, not all results were retrieved. |
| 660 | |
| 661 | |
| 662 | =item B<api_request> |
| 663 | |
| 664 | Report the whole API request. |
| 665 | |
| 666 | |
| 667 | =item B<api_response> |
| 668 | |
| 669 | Report the whole API response (a KoralQuery object). |
| 670 | |
| 671 | |
| 672 | =item B<benchmarks> |
| 673 | |
| 674 | Report on processing time for benchmarking. |
| 675 | |
| 676 | |
| 677 | =item B<query_jsonld> |
| 678 | |
| 679 | The KoralQuery realization of the C<query> object. |
| 680 | |
| 681 | =back |
| 682 | |
| 683 | =head2 search |
| 684 | |
| 685 | Search the index. |
| 686 | |
| 687 | =head2 trace |
| 688 | |
| 689 | Trace query serializations. |
| 690 | |
| 691 | =head2 match |
| 692 | |
| 693 | Get match information. |
| 694 | |
| 695 | =head2 resource |
| 696 | |
| 697 | Get resource information. |
| 698 | |
| 699 | |
| 700 | =head1 COPYRIGHT AND LICENSE |
| 701 | |
Akron | 46b9f21 | 2017-05-01 13:55:17 +0200 | [diff] [blame] | 702 | Copyright (C) 2015-2017, L<IDS Mannheim|http://www.ids-mannheim.de/> |
Nils Diewald | 9dfe010 | 2015-05-19 16:14:06 +0000 | [diff] [blame] | 703 | Author: L<Nils Diewald|http://nils-diewald.de/> |
| 704 | |
| 705 | Kalamar is developed as part of the L<KorAP|http://korap.ids-mannheim.de/> |
| 706 | Corpus Analysis Platform at the |
| 707 | L<Institute for the German Language (IDS)|http://ids-mannheim.de/>, |
| 708 | member of the |
| 709 | L<Leibniz-Gemeinschaft|http://www.leibniz-gemeinschaft.de/en/about-us/leibniz-competition/projekte-2011/2011-funding-line-2/> |
| 710 | and supported by the L<KobRA|http://www.kobra.tu-dortmund.de> project, |
| 711 | funded by the |
| 712 | L<Federal Ministry of Education and Research (BMBF)|http://www.bmbf.de/en/>. |
| 713 | |
| 714 | Kalamar is free software published under the |
Akron | 456abd9 | 2015-06-02 15:07:21 +0200 | [diff] [blame] | 715 | L<BSD-2 License|https://raw.githubusercontent.com/KorAP/Kalamar/master/LICENSE>. |
Nils Diewald | 9dfe010 | 2015-05-19 16:14:06 +0000 | [diff] [blame] | 716 | |
| 717 | =cut |