Merge branch 'master' into repetition-query
diff --git a/Changes b/Changes
index c0dd87a..76b694d 100644
--- a/Changes
+++ b/Changes
@@ -1,163 +1,175 @@
-0.55.6 2016-08-06
+0.55.7 2016-10-11
+ - [bugfix] Throw error on optional operands in distance
+ queries (diewald)
+ - [performance] Remember solved problematic queries in the
+ query planner (diewald)
+ - [performance] Treat term queries like ".+?", ".+", ".*?", and ".*"
+ as any-terms (diewald)
+ - [feature] Introduced SpanExpansionQueryWrapper (diewald)
+ - [performance] Sequences of empty tokens will now be merged into
+ a single extension, if possible (diewald)
+
+0.55.6 2016-08-10
- [bugfix] distance with key "t" uses default foundry (diewald)
- - [cleanup] Renamed fromJson() to fromKoral() (diewald)
- - [cleanup] Removed deprecated methods in Krill:
- setSpanQuery(), getRequest() (diewald)
- - [cleanup] Removed deprecated method in KrillIndex:
- search() (diewald)
+ - [cleanup] Renamed fromJson() to fromKoral() (diewald)
+ - [cleanup] Removed deprecated methods in Krill:
+ setSpanQuery(), getRequest() (diewald)
+ - [cleanup] Removed deprecated method in KrillIndex:
+ search() (diewald)
- [documentation] Refer to KoralQuery instead of Koral (diewald)
- - [cleanup] Removed deprecated method in KrillCollection:
- getCount() (diewald)
- - [feature] Added getDoc() method to KrillIndex (diewald)
+ - [cleanup] Removed deprecated method in KrillCollection:
+ getCount() (diewald)
+ - [feature] Added getDoc() method to KrillIndex (diewald)
- [bugfix] Fixed UID handling (diewald)
- - [feature] Added document method to Web-API (diewald)
- - [feature] Added experimental KrillStats class (diewald)
- - [bugfix] Escape quotes in JSON strings (diewald)
- - [bugfix] Escape XML and Brackets in snippets correctly (diewald)
- - [feature] Differ between matching scope and match highlighting,
- in case the match was extended to the sentence scope (diewald)
- - [bugfix] KoralQuery bugs are now correctly moved to responses (diewald)
- - [security] Do not mirror arbitrary JSON objects (diewald)
+ - [feature] Added document method to Web-API (diewald)
+ - [feature] Added experimental KrillStats class (diewald)
+ - [bugfix] Escape quotes in JSON strings (diewald)
+ - [bugfix] Escape XML and Brackets in snippets correctly (diewald)
+ - [feature] Differ between matching scope and match highlighting,
+ in case the match was extended to the sentence scope (diewald)
+ - [bugfix] KoralQuery bugs are now correctly moved to responses (diewald)
+ - [security] Do not mirror arbitrary JSON objects (diewald)
+ - [bugfix] Fixed zero-boundary repetition bug (diewald)
0.55.5 2016-05-02
- - [performance] Changed to a dynamic window for sorting in FocusSpans (margaretha)
- - [bugfix] store skipped spans in Repetitionspans as candidates
- (margaretha)
- - [bugfix] Store lazy loading embedded spans for startsWith in
- WithinSpans (diewald)
- - [bugfix] Fixed loading krill.info in case krill.properties is given
- (diewald)
- - [bugfix] Remove element payloads in SubSpans (similar to Focus)
- to prevent wrong highlighting (diewald)
- - [bugfix] Sort SubSpans (margaretha)
+ - [performance] Changed to a dynamic window for sorting in FocusSpans (margaretha)
+ - [bugfix] store skipped spans in Repetitionspans as candidates
+ (margaretha)
+ - [bugfix] Store lazy loading embedded spans for startsWith in
+ WithinSpans (diewald)
+ - [bugfix] Fixed loading krill.info in case krill.properties is given
+ (diewald)
+ - [bugfix] Remove element payloads in SubSpans (similar to Focus)
+ to prevent wrong highlighting (diewald)
+ - [bugfix] Sort SubSpans (margaretha)
0.55.4 2016-04-22
- - [bugfix] Wrap <base/s=t> boundaries around right extended queries
- (diewald)
- - [bugfix] Lazy loading bug in WithinSpans (diewald)
- - [feature] Matching relation nodes with attributes (margaretha)
- - [bugfix] Removed span id in SegmentSpans, added in NextSpans (margaretha)
- - [feature] Updated relation deserialization (margaretha)
- - [feature] Added tests for relation queries and relation deserialization
- (margaretha)
- - [bugfix] Workaround for Nullpointers in SpanOrQuery (diewald)
+ - [bugfix] Wrap <base/s=t> boundaries around right extended queries
+ (diewald)
+ - [bugfix] Lazy loading bug in WithinSpans (diewald)
+ - [feature] Matching relation nodes with attributes (margaretha)
+ - [bugfix] Removed span id in SegmentSpans, added in NextSpans (margaretha)
+ - [feature] Updated relation deserialization (margaretha)
+ - [feature] Added tests for relation queries and relation deserialization
+ (margaretha)
+ - [bugfix] Workaround for Nullpointers in SpanOrQuery (diewald)
0.55.3 2016-02-22
- - [bugfix] Use base foundry for default sentences and paragraphs
- (diewald, margaretha)
- - [bugfix] Distance constraint exclusion serialized (diewald)
- - [bugfix] Use indexer without krill.properties packaged (diewald)
- - [bugfix] Clean pom.xml regarding overlapping dependencies (diewald)
+ - [bugfix] Use base foundry for default sentences and paragraphs
+ (diewald, margaretha)
+ - [bugfix] Distance constraint exclusion serialized (diewald)
+ - [bugfix] Use indexer without krill.properties packaged (diewald)
+ - [bugfix] Clean pom.xml regarding overlapping dependencies (diewald)
0.55.2 2016-02-13
- [bugfix] Fixed support for corpus and doc IDs containing
- dashes (diewald)
- - [bugfix] Support "s" layer without meaning "surface" -
- this is a hack (diewald)
+ dashes (diewald)
+ - [bugfix] Support "s" layer without meaning "surface" -
+ this is a hack (diewald)
0.55.1 2016-02-10
- [bugfix] Fixed attribute serialization in MatchInfo (diewald)
0.55 2016-02-04
- [feature] Introducing payload identifiers to the index
- structure (diewald, margaretha)
- ! This is a major release, introducing index features !
- ! not compatible with indices of old versions of Krill !
+ structure (diewald, margaretha)
+ ! This is a major release, introducing index features !
+ ! not compatible with indices of old versions of Krill !
0.54 2015-11-13
- - [performance] Updated Lucene dependency from 4.10.3
- to 5.0.0 (diewald)
+ - [performance] Updated Lucene dependency from 4.10.3
+ to 5.0.0 (diewald)
- [bugfix] Minor fixes regarding invalid tokens in the
- test suite (diewald)
- - [bugfix] Fixed krill.properties test (diewald)
- - [bugfix] Fixed multiple distances in disjunction query
- (GitHub issue #2; margaretha)
+ test suite (diewald)
+ - [bugfix] Fixed krill.properties test (diewald)
+ - [bugfix] Fixed multiple distances in disjunction query
+ (GitHub issue #2; margaretha)
0.53 2015-09-25
- [feature] Implemented new KrillCollection (diewald)
- This should fix a lot of issues with deleted
- documents and negation in virtual collections.
- - [cleanup] REMOVED deprecated collection filtering (diewald)
- - [cleanup] REMOVED deprecated termRelation API (diewald)
- - [feature] Added removal methods for documents (diewald)
- - [cleanup] REMOVED search method in KrillCollection (diewald)
+ This should fix a lot of issues with deleted
+ documents and negation in virtual collections.
+ - [cleanup] REMOVED deprecated collection filtering (diewald)
+ - [cleanup] REMOVED deprecated termRelation API (diewald)
+ - [feature] Added removal methods for documents (diewald)
+ - [cleanup] REMOVED search method in KrillCollection (diewald)
- [bugfix] Fixed collection statistics (diewald)
- - [feature] Improved packaging (diewald)
- - [feature] Introduced cmd options for server (diewald)
- - [feature] Return meta data on server-based indexation (diewald)
+ - [feature] Improved packaging (diewald)
+ - [feature] Introduced cmd options for server (diewald)
+ - [feature] Return meta data on server-based indexation (diewald)
0.52 2015-07-08
- [bugfix] Fixed payload filtering in FocusSpans (margaretha)
- - [workaround] Reintroduced empty collection support,
- as Koral still creates them (diewald)
- - [bugfix] Escaped characters now supported in MutiTerm (diewald)
- - [feature] Deserialization of flags (diewald)
- - [feature] Made responses valid KoralQueries (diewald)
- - [bugfix] Fixed sequence deserialization bug for simple unordered
- constraint (diewald)
- - [bugfix] Mirror collection (diewald)
- - [bugfix] Updated default fields for meta data (diewald)
- - [bugfix] Updated match identifier for "sigle" data (diewald)
- - [workaround] Support corpusID/docID and textSigle match strings
- (diewald)
- - [workaround] Support matches starting with "contains"
- (Kustvakt bug/diewald)
- - [bugfix] Fixed treatment of several collection types (diewald)
- - [bugfix] Remove legacy error handling in Match (diewald)
- - [bugfix] JSON-serialization bug in match in case of
- error messages (diewald)
- - [bugfix] 'fields' serialization (diewald)
- - [bugfix] Deserialization of group:disjunction (diewald)
- - [bugfix] Deserialization of wrapped spans (diewald)
- - [bugfix] Fix KoralQuery mirroring in case of
- parsing failures (diewald)
- - [feature] Support '@all' as a 'fields' value for all meta
- data fields (diewald)
- - [bugfix] Fix case sensitivity bug in filters (diewald)
- - [bugfix] Fix sentence expansion bug for new data (diewald)
- - [bugfix] Deserialization of 'containsnot' instead of 'excludes'
- (diewald)
+ - [workaround] Reintroduced empty collection support,
+ as Koral still creates them (diewald)
+ - [bugfix] Escaped characters now supported in MutiTerm (diewald)
+ - [feature] Deserialization of flags (diewald)
+ - [feature] Made responses valid KoralQueries (diewald)
+ - [bugfix] Fixed sequence deserialization bug for simple unordered
+ constraint (diewald)
+ - [bugfix] Mirror collection (diewald)
+ - [bugfix] Updated default fields for meta data (diewald)
+ - [bugfix] Updated match identifier for "sigle" data (diewald)
+ - [workaround] Support corpusID/docID and textSigle match strings
+ (diewald)
+ - [workaround] Support matches starting with "contains"
+ (Kustvakt bug/diewald)
+ - [bugfix] Fixed treatment of several collection types (diewald)
+ - [bugfix] Remove legacy error handling in Match (diewald)
+ - [bugfix] JSON-serialization bug in match in case of
+ error messages (diewald)
+ - [bugfix] 'fields' serialization (diewald)
+ - [bugfix] Deserialization of group:disjunction (diewald)
+ - [bugfix] Deserialization of wrapped spans (diewald)
+ - [bugfix] Fix KoralQuery mirroring in case of
+ parsing failures (diewald)
+ - [feature] Support '@all' as a 'fields' value for all meta
+ data fields (diewald)
+ - [bugfix] Fix case sensitivity bug in filters (diewald)
+ - [bugfix] Fix sentence expansion bug for new data (diewald)
+ - [bugfix] Deserialization of 'containsnot' instead of 'excludes'
+ (diewald)
0.51 2015-03-17
- This is a major version (prepared for the GitHub release)
- [cleanup] Changed groupID to "de.ids_mannheim.korap",
- renamed korap-style.xml to Format.xml (diewald)
- - [bugfix] Updated Format.xml to work with
- recent versions of Eclipse (diewald)
+ renamed korap-style.xml to Format.xml (diewald)
+ - [bugfix] Updated Format.xml to work with
+ recent versions of Eclipse (diewald)
- [documentation] Improved documentation for API classes (diewald)
- - [bugfix] Updated AttributeSpans (margaretha)
- - [bugfix] Updated SpanRelationQuery (margaretha)
- - [cleanup] Autoformat (diewald)
+ - [bugfix] Updated AttributeSpans (margaretha)
+ - [bugfix] Updated SpanRelationQuery (margaretha)
+ - [cleanup] Autoformat (diewald)
- [documentation] References added to the Readme (diewald)
- - [bugfix] Improved failing for missing property file (kupietz)
- - [bugfix] Fixed tests for server responses to not use
- Jackson deserialization (diewald)
- - [cleanup] No more jersey logging in tests (diewald)
+ - [bugfix] Improved failing for missing property file (kupietz)
+ - [bugfix] Fixed tests for server responses to not use
+ Jackson deserialization (diewald)
+ - [cleanup] No more jersey logging in tests (diewald)
0.50.1 2015-03-02
- - [feature] Deserialization of arbitrary elements with
- attributes (margaretha)
- - [cleanup] Extracted KrillMeta from Krill,
- renamed KorapQuery to KrillQuery,
- extracted QueryBuilder from KrillQuery,
- renamed KorapIndexer to index/Indexer,
- renamed KorapDocument to model/AbstractDocument,
- renamed KorapMatch to response/Match,
- renamed KorapIndex to KrillIndex,
- renamed KorapCollection to KrillCollection,
- moved collector classes from index to response,
- renamed KorapDate to KrillDate,
- renamed KorapArray to KrillArray,
- renamed KorapByte to KrillByte,
- renamed KorapString to KrillString,
- moved KorapNode to server/Node,
- moved KorapResult to response/Result,
- renamed KorapResponse to Response,
- moved model/* to index,
- moved meta/SearchContext to response/SearchContext,
- krillified property files,
- added Errorcodes,
- removed KrillIndex#addDocFile (diewald)
+ - [feature] Deserialization of arbitrary elements with
+ attributes (margaretha)
+ - [cleanup] Extracted KrillMeta from Krill,
+ renamed KorapQuery to KrillQuery,
+ extracted QueryBuilder from KrillQuery,
+ renamed KorapIndexer to index/Indexer,
+ renamed KorapDocument to model/AbstractDocument,
+ renamed KorapMatch to response/Match,
+ renamed KorapIndex to KrillIndex,
+ renamed KorapCollection to KrillCollection,
+ moved collector classes from index to response,
+ renamed KorapDate to KrillDate,
+ renamed KorapArray to KrillArray,
+ renamed KorapByte to KrillByte,
+ renamed KorapString to KrillString,
+ moved KorapNode to server/Node,
+ moved KorapResult to response/Result,
+ renamed KorapResponse to Response,
+ moved model/* to index,
+ moved meta/SearchContext to response/SearchContext,
+ krillified property files,
+ added Errorcodes,
+ removed KrillIndex#addDocFile (diewald)
- [documentation] Improved documentation for API classes (diewald)
0.50 2015-02-23
@@ -165,150 +177,150 @@
0.49.4 2015-02-20
- [documentation] Improved documentation for API classes (diewald)
- - [cleanup] Improved test coverage for utility classes (diewald)
- - [performance] Updated Lucene dependency from 4.5.1 to 4.10.3,
- Updated Jackson dependency from 2.4.0 to 2.4.4,
- Updated Jersey dependency from 2.4.1 to 2.15 (diewald)
+ - [cleanup] Improved test coverage for utility classes (diewald)
+ - [performance] Updated Lucene dependency from 4.5.1 to 4.10.3,
+ Updated Jackson dependency from 2.4.0 to 2.4.4,
+ Updated Jersey dependency from 2.4.1 to 2.15 (diewald)
- [feature] Presorting of element terms in the index for coherent
- SpanQuery sorting; Removed buffering of element candidates (diewald)
- Warning: This is a breaking change!
- - [feature] Added CorpusDataException to deal with data importing bugs;
+ SpanQuery sorting; Removed buffering of element candidates (diewald)
+ Warning: This is a breaking change!
+ - [feature] Added CorpusDataException to deal with data importing bugs;
made some imports break more easily though (diewald)
- - [performance] Payloads in ElementSpans can now be lazily loaded,
- MatchModifyClassSpans no longer rely on payload copies (diewald)
- - [cleanup] Renamed /filter to /collection,
- merge KorapHTML and KorapString,
- removed KorapSpan, KorapTermSpan and KorapLongSpan,
- renamed /analysis to /model,
- renamed shrink() to focus(),
- removed KorapPrimaryData,
+ - [performance] Payloads in ElementSpans can now be lazily loaded,
+ MatchModifyClassSpans no longer rely on payload copies (diewald)
+ - [cleanup] Renamed /filter to /collection,
+ merge KorapHTML and KorapString,
+ removed KorapSpan, KorapTermSpan and KorapLongSpan,
+ renamed /analysis to /model,
+ renamed shrink() to focus(),
+ removed KorapPrimaryData,
fixed a lot of wrong tests for WithinSpans,
- renamed KorapFilter to /collection/CollectionBuilder,
- renamed SpanMatchModify to SpanFocus,
- changed KoralQuery root prefix to "koral",
- renamed different kinds of "queries" throughout classes,
- renamed KorapSearch#run to KorapSearch#apply (diewald)
+ renamed KorapFilter to /collection/CollectionBuilder,
+ renamed SpanMatchModify to SpanFocus,
+ changed KoralQuery root prefix to "koral",
+ renamed different kinds of "queries" throughout classes,
+ renamed KorapSearch#run to KorapSearch#apply (diewald)
- [feature] Improved deserialization of SpanSubSpanQueries
(margaretha)
- - [feature] Introducing the potential need for resorting queries
- on focussing (e.g., set by relations) (diewald)
- - [bugfix] Hopefully fixing a resource related bug for
- server instantiation (diewald)
- - [bugfix] Changed attribute spans to contain element's
- width (margaretha)
- - [feature] Improved highlighting to use semantic html5 (#193; diewald)
+ - [feature] Introducing the potential need for resorting queries
+ on focussing (e.g., set by relations) (diewald)
+ - [bugfix] Hopefully fixing a resource related bug for
+ server instantiation (diewald)
+ - [bugfix] Changed attribute spans to contain element's
+ width (margaretha)
+ - [feature] Improved highlighting to use semantic html5 (#193; diewald)
0.49.3 2015-02-03
- [documentation] Improved documentation for
- API classes (diewald)
- - [documentation] Improved documentation for
- various queries (margaretha)
- - [feature] Added deserialization of SpanSubSpanQueries
+ API classes (diewald)
+ - [documentation] Improved documentation for
+ various queries (margaretha)
+ - [feature] Added deserialization of SpanSubSpanQueries
(margaretha, diewald)
- - [bugfix] Null filters are now correctly extended (diewald)
- - [cleanup] Refactoring of KorapResult, KorapResponse, KorapQuery,
- deprecated operation:or in favor of operation:junction,
- updating test suite for class and frame attributes (diewald)
- - [bugfix] Empty class segments are now correctly serialized
- in sequences (diewald)
- - [feature] Started support for position frames (diewald)
- - [bugfix] Fixed #134 Nullpointer exception in WithinSpans
- (diewald, margaretha)
+ - [bugfix] Null filters are now correctly extended (diewald)
+ - [cleanup] Refactoring of KorapResult, KorapResponse, KorapQuery,
+ deprecated operation:or in favor of operation:junction,
+ updating test suite for class and frame attributes (diewald)
+ - [bugfix] Empty class segments are now correctly serialized
+ in sequences (diewald)
+ - [feature] Started support for position frames (diewald)
+ - [bugfix] Fixed #134 Nullpointer exception in WithinSpans
+ (diewald, margaretha)
0.49.2 2014-12-05
- - [documentation] Improved documentation for
+ - [documentation] Improved documentation for
various queries (margaretha)
- - [feature] Added @Experimental support (margaretha)
- - [feature] Added Eclipse based code formatting (diewald)
+ - [feature] Added @Experimental support (margaretha)
+ - [feature] Added Eclipse based code formatting (diewald)
0.49.1 2014-11-25
- [cleanup] Major refactoring for new response messages,
- removed deprecated error messaging,
+ removed deprecated error messaging,
removed long-term deprecated methods in KorapResult,
- removed long-term deprecated methods in KorapMatch,
- renamed toJSON->toJsonString
- renamed toJSONnode->toJsonNode,
- renamed fromJSON->fromJson (diewald)
- - [documentation] Improved documentation for notifications (diewald)
+ removed long-term deprecated methods in KorapMatch,
+ renamed toJSON->toJsonString
+ renamed toJSONnode->toJsonNode,
+ renamed fromJSON->fromJson (diewald)
+ - [documentation] Improved documentation for notifications (diewald)
0.49 2014-11-14
- This is a major version (prepared for the Evaluation Meeting)
- [bugfix] Rewrite works in repeating expansions (margaretha)
- - [feature] Clean timeout with parameter support (diewald)
- - [workaround] Support unknown overlap frame,
- if frames are known (diewald)
- - [bugfix] Class payload fixed in SpanRepetitions (margaretha)
- - [bugfix] #179 Expansion bug fixed (margaretha)
- - [bugfix] Corrected small bug where null warnings where
- returned in JSON (diewald)
- - [cleanup] Converted error to failure (diewald)
- - [bugfix] Added const layer rewrite (diewald)
+ - [feature] Clean timeout with parameter support (diewald)
+ - [workaround] Support unknown overlap frame,
+ if frames are known (diewald)
+ - [bugfix] Class payload fixed in SpanRepetitions (margaretha)
+ - [bugfix] #179 Expansion bug fixed (margaretha)
+ - [bugfix] Corrected small bug where null warnings where
+ returned in JSON (diewald)
+ - [cleanup] Converted error to failure (diewald)
+ - [bugfix] Added const layer rewrite (diewald)
0.48 2014-11-07
- This is a major version (prepared for the "Tag der Offenen Tür")
- [feature] Retrieval of token lists (diewald)
- - [bugfix] Classes can now be highlighted to 127.
- - [bugfix] Sequences with distances won't be flattened in
- SpanSequenceQueryWrapper (diewald)
- - [bugfix] Sequences with [problem][problem][anchor] can now
- be deserialized (diewald)
- - [bugfix] Queries with regular expressions in spanNext() are now
- correctly rewritten (diewald)
- - [bugfix] Ignore foundry for orth layer (diewald)
- - [feature] Support fields in meta (diewald)
- - [workaround] Support more legacy APIs (diewald)
- - [bugfix] Check for the existence of @type in JSON-LD groups
- to avoid NullPointer (diewald)
- - [cleanup] Validated query and collection input - limited
- to a "don't break anyting" policy (diewald)
+ - [bugfix] Classes can now be highlighted to 127.
+ - [bugfix] Sequences with distances won't be flattened in
+ SpanSequenceQueryWrapper (diewald)
+ - [bugfix] Sequences with [problem][problem][anchor] can now
+ be deserialized (diewald)
+ - [bugfix] Queries with regular expressions in spanNext() are now
+ correctly rewritten (diewald)
+ - [bugfix] Ignore foundry for orth layer (diewald)
+ - [feature] Support fields in meta (diewald)
+ - [workaround] Support more legacy APIs (diewald)
+ - [bugfix] Check for the existence of @type in JSON-LD groups
+ to avoid NullPointer (diewald)
+ - [cleanup] Validated query and collection input - limited
+ to a "don't break anyting" policy (diewald)
0.47 2014-11-05
- [feature] Support new index format with more metadata (diewald)
- - [bugfix] #142 Fixed class spans for skipTo() skips (margaretha)
- - [feature] Added attribute support to relations (margaretha)
- - [bugfix] Make classes and shrink default to 1 instead of 0 (diewald)
- - [bugfix] shrink now removes all non-classes payloads (diewald)
+ - [bugfix] #142 Fixed class spans for skipTo() skips (margaretha)
+ - [feature] Added attribute support to relations (margaretha)
+ - [bugfix] Make classes and shrink default to 1 instead of 0 (diewald)
+ - [bugfix] shrink now removes all non-classes payloads (diewald)
0.46 2014-10-21
- [feature] Support for relation queries (margaretha)
- - [feature] Wrapping of extension queries
- (not completely tested yet; diewald)
+ - [feature] Wrapping of extension queries
+ (not completely tested yet; diewald)
0.45 2014-10-06
- [bugfix] Correctly respond request in JSON results (diewald)
- [cleanup] Made SpanQueryWrapper an abstract class instead
- of an interface (diewald)
+ of an interface (diewald)
0.44.1 2014-09-29
- [cleanup] Prepare test suite for publication (diewald)
0.44 2014-09-28
- [feature] Match collector using database for
- distributed search (diewald)
- - [bugfix] Unified boundary handling and prevent nullpointer
- exceptions on parsing (diewald)
- - [bugfix] Ordering of payloads reversed to make nesting highlights
- nest correctly (diewald)
- - [performance] Updated Lucene dependency from 4.3.1 to 4.5.1 (diewald)
- - [performance] Updated Jackson dependency from 2.2.2 to 2.4.0 (diewald)
- - [bugfix] Return matches correctly with JSON (diewald)
- - [bugfix] Repare database tests working with temporary
- databases (diewald)
+ distributed search (diewald)
+ - [bugfix] Unified boundary handling and prevent nullpointer
+ exceptions on parsing (diewald)
+ - [bugfix] Ordering of payloads reversed to make nesting highlights
+ nest correctly (diewald)
+ - [performance] Updated Lucene dependency from 4.3.1 to 4.5.1 (diewald)
+ - [performance] Updated Jackson dependency from 2.2.2 to 2.4.0 (diewald)
+ - [bugfix] Return matches correctly with JSON (diewald)
+ - [bugfix] Repare database tests working with temporary
+ databases (diewald)
0.43 2014-09-23
- [cleanup] Made a lot of stuff rely on KorapResponse (diewald)
- - [bugfix] Small but ancient collection bug fixed (diewald)
- - [feature] Collect first matches of documents based on UIDs (diewald)
+ - [bugfix] Small but ancient collection bug fixed (diewald)
+ - [feature] Collect first matches of documents based on UIDs (diewald)
0.42 2014-09-19
- [cleanup] Extracted private classes from KorapMatch (diewald)
- - [bugfix] Fix query rewrite in contains-queries (diewald)
+ - [bugfix] Fix query rewrite in contains-queries (diewald)
0.41 2014-09-17
- [feature] getStartPos(classNr) and getEndPos(classNr)
in KorapMatch (diewald)
- [bugfix] SpanExpansionQuery with fixed Constructor
- and Exclusion handling (margaretha)
+ and Exclusion handling (margaretha)
- [feature] Early implementation of match collector (diewald)
0.40 2014-09-03
@@ -320,10 +332,10 @@
0.32 2014-08-01
- [bugfix] Fix for repetition candidates (margaretha)
- - [bugfix] Some fixes for negative segments (diewald)
- - [bugfix] SpanMatchModify now skips in case the
- requested class isn't found (diewald)
- - [performance] Some logging issues fixed (diewald)
+ - [bugfix] Some fixes for negative segments (diewald)
+ - [bugfix] SpanMatchModify now skips in case the
+ requested class isn't found (diewald)
+ - [performance] Some logging issues fixed (diewald)
0.31.8 2014-07-24
- [feature] Added optionality to querys for quantifiers (diewald)
@@ -334,31 +346,31 @@
0.31.6 2014-06-30
- [feature] Support for korap:reference deserialization (diewald)
- - [feature] Preliminary support for attributes in elements (margaretha)
- - [feature] Preliminary support operation:repetition deserialization
- (unfinished and untested; diewald)
+ - [feature] Preliminary support for attributes in elements (margaretha)
+ - [feature] Preliminary support operation:repetition deserialization
+ (unfinished and untested; diewald)
0.31.5 2014-06-17
- [feature] Batch return of defined foundries and layers in
- getMatchInfo() (diewald)
- - [security] Improve foundry and layer check before injection in
- regex (diewald)
+ getMatchInfo() (diewald)
+ - [security] Improve foundry and layer check before injection in
+ regex (diewald)
0.31.4 2014-06-16
- [feature] MatchModifyClassQuery now can extend (diewald)
- - [bugfix] Make matchinfo a bit more robust (diewald)
+ - [bugfix] Make matchinfo a bit more robust (diewald)
0.31.3 2014-06-11
- [bugfix] Minor bugfix regarding treatment of
- failed search results (diewald)
+ failed search results (diewald)
0.31.2 2014-05-28
- [feature] Request parameter "itemsPerResource" limits
- hits per document (diewald)
+ hits per document (diewald)
0.31.1 2014-05-21
- [bugfix] Minor bugfix for wrongly serialized contexts
- in responses (diewald)
+ in responses (diewald)
0.31.0 2014-05-20
- [feature] span contexts in JSON-LD are now supported
@@ -368,39 +380,39 @@
0.30.9 2014-04-11
- [bugfix] Prevent multiple identifiers in
- relation highlighting (diewald)
+ relation highlighting (diewald)
0.30.8 2014-04-10
- Added getTermRelationJSON() to KorapCollection (diewald)
- This is likely to get deprecated!
+ This is likely to get deprecated!
0.30.7 2014-04-09
- Moved JSON interpretation from BooleanFilter to KorapFilter (diewald)
- - Added 'or' and group nesting to KorapFilter (diewald)
+ - Added 'or' and group nesting to KorapFilter (diewald)
0.30.6 2014-03-20
- Refactoring and cleaning of ElementSpans (diewald)
- - [bugfix] Stabilizing KorapMatch (diewald)
+ - [bugfix] Stabilizing KorapMatch (diewald)
0.30.5 2014-03-18
- - Support for Quantifier Queries (margaretha)
- - Some benchmarks (margaretha)
+ - Support for Quantifier Queries (margaretha)
+ - Some benchmarks (margaretha)
- [bugfix] WithinSpans respect match payloads,
- and are now way more correct ;) (diewald)
- - Support for overlap queries (diewald)
- - Refactoring of WithinSpans (diewald)
- - Introducing lazy loading for wrapper .end() in WithinSpans (diewald)
+ and are now way more correct ;) (diewald)
+ - Support for overlap queries (diewald)
+ - Refactoring of WithinSpans (diewald)
+ - Introducing lazy loading for wrapper .end() in WithinSpans (diewald)
0.30.4 2014-02-26
- [cleanup] Making MultiTerm* more robust (diewald)
0.30.3 2014-02-20
- Return json request in response if given (diewald)
- - [bugfix] ClassSpans and WithinSpans check skipTo
- in advance now (diewald)
- - Compilerfriendly logging rules, resulting
- in significant performance improvements (diewald)
- - Small refactorings (diewald)
+ - [bugfix] ClassSpans and WithinSpans check skipTo
+ in advance now (diewald)
+ - Compilerfriendly logging rules, resulting
+ in significant performance improvements (diewald)
+ - Small refactorings (diewald)
0.30.2 2014-02-14
- [bugfix] wrapper uses correct distance constraint (diewald)
@@ -410,112 +422,112 @@
0.30 2014-02-13
- This is a major version
- (prepared for the IDS meeting on the 17th of february)
- - Improved stringification for distance queries (margaretha)
+ (prepared for the IDS meeting on the 17th of february)
+ - Improved stringification for distance queries (margaretha)
0.26.4 2014-02-12
- Serialize SpanDistanceQueries and SpanMultipleDistanceQueries
(diewald)
- - [temporarily] Added default foundry "mate" in KorapQuery (diewald)
+ - [temporarily] Added default foundry "mate" in KorapQuery (diewald)
0.26.3 2014-02-12
- [bugfix] Only open writer in case it's needed (diewald)
0.26.2 2014-02-12
- Added sequence queries with defined gaps (margaretha)
- - Support for multiple distance constraints (margaretha)
- - Updated json-ld deserialization with nested meta (diewald)
- - [bugfix] Wildcards in segments were ignored (diewald)
+ - Support for multiple distance constraints (margaretha)
+ - Updated json-ld deserialization with nested meta (diewald)
+ - [bugfix] Wildcards in segments were ignored (diewald)
0.26.1 2014-01-21
- Distinct Match retrieval including spans
- and dependency relations (diewald)
+ and dependency relations (diewald)
0.26 2014-01-16
- Introduced standalone SpanSegmentQueries (margaretha)
- [bugfix] SpanNextQueries (margaretha)
- - Support for Match Identifiers (diewald)
- - Support for distinct Match retrieval (diewald)
+ - Support for Match Identifiers (diewald)
+ - Support for distinct Match retrieval (diewald)
0.25.3 2014-01-10
- Updated Lucene to 4.3.1 (diewald)
- - Welcome! Eliza Margaretha has joined the team!
+ - Welcome! Eliza Margaretha has joined the team!
0.25.2 2013-12-29
- Fixed highlight bug with empty reopened tags
0.25.1 2013-12-28
- Support for pubDate collections
- - Changed versioning scheme
+ - Changed versioning scheme
0.25 2013-12-20
- Support for Wildcard Queries
- - Improved support for regular expressions
- - Introduced keyword fields that store no positions
+ - Improved support for regular expressions
+ - Introduced keyword fields that store no positions
0.24_1 2013-12-05
- This is a pseudo version for demo versions with dirty hacks,
- meant to be rolled back!
- - There is now an ignored broken test in TestKorapCollection
+ meant to be rolled back!
+ - There is now an ignored broken test in TestKorapCollection
0.24 2013-12-05
- Json deserializer finished for virtual collections
0.23 2013-12-05
- Error handling corrected in KorapResult
- - Json deserializer finished for queries
+ - Json deserializer finished for queries
0.22 2013-12-04
- Introduced KorapSearch
- - Json deserializer (started)
+ - Json deserializer (started)
0.21 2013-11-28
- Virtual collections can now be defined,
- searched, nested and extended
+ searched, nested and extended
0.20 2013-11-18
- Unboxing from sandbox repository
0.11 2013-11-14
- - JSON file importer (plain and gzip)
- - [bugfix] Element Query
- - [bugfix] Sequence Query
+ - JSON file importer (plain and gzip)
+ - [bugfix] Element Query
+ - [bugfix] Sequence Query
0.10 2013-11-11
- Added JSON input to indexer
0.09 2013-11-07
- Reconstruction of all wrapper queries using an interface
- - Finished KorapQuery (preliminary feature freeze)
+ - Finished KorapQuery (preliminary feature freeze)
0.08 2013-10-18
- Changed methods to attributes in KorapMatch
- - Changed SimpleJSON to Jackson
- - Fixed Highlighting
- - Some Refactoring of result classes
- - Introduced KorapDocument
- - Introduced KorapHTML utility
+ - Changed SimpleJSON to Jackson
+ - Fixed Highlighting
+ - Some Refactoring of result classes
+ - Introduced KorapDocument
+ - Introduced KorapHTML utility
0.07 2013-10-14
- Added position to offset matching
0.06 2013-10-01
- Added SpanElementQuery for working WithinQeries
- - Added KorapResult and KorapMatching
+ - Added KorapResult and KorapMatching
0.05 2013-09-18
- Fixed bug in indexing (MultiTermTokenStream)
- - Fixed SpanNext
- - Added KorapIndex
- - First draft for KorapFilter
+ - Fixed SpanNext
+ - Added KorapIndex
+ - First draft for KorapFilter
0.04 2013-09-01
- Introduced optimized SpanNext class (not working atm),
- as SpanNear does not always work with nested synonyms
+ as SpanNear does not always work with nested synonyms
0.03 2013-08-28
- Added within query (not working atm)
- - Pushed query part to public repo
+ - Pushed query part to public repo
0.02 2013-08-26
- Added sequences
diff --git a/misc/payloads.md b/misc/payloads.md
index f041466..9fb1109 100644
--- a/misc/payloads.md
+++ b/misc/payloads.md
@@ -2,10 +2,10 @@
Apache Lucene supports payloads as arbitrary byte sequences to store information for terms specific to any token position. Krill uses payloads to store various information in a compact way. This documents describes the payload information for index payloads (payloads stored in the index for different term concepts) and computed payloads (payloads created during the retrieval phase).
## Payload Type Identifier (PTI)
-Payloads (both indexed and computed) have a leading byte indicating the type of the payload sequence. This is necessary because the origin (i.e. the requested term) of a payload is lost during the retrieval phase. Payload type identifiers range between 0 and 255 and have the length of a byte (\<b\>). In case a term has no payload, no payload type identifier is stored.
+Payloads (both indexed and computed) have a leading byte indicating the type of the payload sequence. This is necessary because the origin (i.e. the requested term) of a payload is lost during the retrieval phase. Payload type identifiers range between 0 and 255 and have the length of a byte (\<b\>). In case a token has no payload, no payload type identifier is stored.
## Token-unique Identifier (TUI)
-Terms, elements and relations in the index may contain token-unique identifiers (TUI) to distinguish between lucene-terms starting at the same token position. TUIs are used for matching attributes to terms, elements and relations, and to refer to terms and elements from relations. TUIs have the length of a short (\<s\>).
+Tokens, spans and relations in the index may contain token-unique identifiers (TUI) to distinguish between Lucene-tokens starting at the same token position. TUIs are used for matching attributes to tokens, spans and relations, and to refer to tokens and spans from relations. TUIs have the length of a short (\<s\>).
## Index Payloads
@@ -13,38 +13,38 @@
A token always has a special character payload storing the start and end offset of the token. The special character is a reference symbol for this payload, which is an underscore followed by the corresponding token position. For example, the _1$\<i\>0\<i\>3 is the special character payload for the token in position 1 describing that the token ranges from 0 to 3. This offset information is stored in integer.
Token payloads are not retrieved via SpanQueries and therefore do not have a PTI.
-### Term payloads
-Some terms are indexed with a TUI in their payloads, if they take part in a relation.
-Terms may also have a certainty value attached, expressed as a byte (with a ranging certainty value from probably incorrect to probably correct (0-255)). For example:
+### Token payloads
+Some tokens are indexed with a TUI in their payloads, if they take part in a relation.
+Tokens may also have a certainty value attached, expressed as a byte (with a ranging certainty value from probably incorrect to probably correct (0-255)). For example:
pos:NN$<b>128<s>1
pos:NN$<b>129<b>34
pos:NN$<b>130<s>1<b>34
-*PTIs* (it’s a term payload, if the first bit is set):
+*PTIs* (it is a token payload, if the first bit is set):
- 128: Term with a TUI
- 129: Term with certainty value
- 130: Term with TUI and certainty value
+ 128: Token with a TUI
+ 129: Token with certainty value
+ 130: Token with TUI and certainty value
-### Element payloads
-Each element has payloads consisting of start and end character
+### Span payloads
+Each span has payloads consisting of start and end character
offset information, the PTI, the token position which is the end of
-the element, and the depth (in an abstract tree, with 0 being root).
+the span, and the depth (in an abstract tree, with 0 being root).
In addition, it may have a TUI and a certainty value.
-The offset information and the end element position are stored in
+The offset information and the end position are stored in
integer, whereas the TUI is stored in short, and the depth and certainty
-information is stored as byte values. The stored data type for the end
-element, the depth, the TUI and the certainty are written explicitly:
+information is stored as byte values. The stored data type for the end,
+the depth, the TUI and the certainty are written explicitly:
\<i\> for integer (4 bytes), \<s\> for short (2 bytes), and \<b\> for
byte (1 byte). For example:
<>:s$<b>64<i>0<i>38<i>7<b>0
-means that element \<s\> starts from character offset position 0 and
-ends at character offset position 38. The element ends at token
+means that span \<s\> starts from character offset position 0 and
+ends at character offset position 38. The span ends at token
position 7 (i.e. it includes the 7th token) which is stored in integer.
-It is a root element or no
+It is a root span or no
further information on a tree level is given (depth=0).
<>:s$<b>64<i>0<i>38<i>7<b>0<s>1
@@ -59,16 +59,16 @@
means \<s\> has an additional TUI and a certainty value.
-Elements may also be empty - meaning they behave as milestones.
+Spans may also be empty - meaning they behave as milestones.
In that case, character offsets are only given once.
<>:s$<b>65<i>38<b>0
means \<s\> is a milestone at position 38 in root.
-*PTIs* (It’s an element payload if the second bit is set):
+*PTIs* (It is a span payload if the second bit is set):
- 64 Element (with optional TUI and certainty)
+ 64 Span (with optional TUI and certainty)
65 Milestone (with optional TUI and certainty)
### Relation payloads
@@ -88,13 +88,13 @@
right parts, which again can be either a source or a target of the
relation.
-* If the left part of a relation is an element, the end position
- of the element has to be stored in payload; if it is a token,
+* If the left part of a relation is a span, the end position
+ of the span has to be stored in payload; if it is a token,
nothing has to be stored additionally.
* The same applies for the right part of a relation.
These positions are always stored in integer. Besides token position,
-character offsets of relations with element parts
+character offsets of relations with span parts
are also stored in integer.
Left-part TUI reference, right-part TUI reference, and relation TUI can
@@ -106,7 +106,7 @@
the right-part TUI reference, or the relation TUI), all TUIs have to be set.
If the TUIs do not refer to anything, they have to be set to ```0```.
-1) Term to term relation has
+1) Token to token relation has
* 1 byte for PTI (32),
* 1 integer for the right part token position,
@@ -121,11 +121,11 @@
has a token as the right part at (end) position 3, the source TUI reference 5,
no target TUI reference and the relation TUI 3.
-2) Term to element relation has
+2) Token to span relation has
* 1 byte for PTI (33),
-* 1 integer for the start element offset of the right part,
-* 1 integer for the end element offset of the right part,
+* 1 integer for the start span offset of the right part,
+* 1 integer for the end span offset of the right part,
* 1 integer for the start position of the right part,
* 1 integer for the end position of the right part,
* and 0-3 TUIs as above.
@@ -137,11 +137,11 @@
means the right part starts at token position 1 and ends at token
position 3.
-3) Element to term relation has
+3) Span to token relation has
* 1 byte for PTI (34),
-* 1 integer for the start element offset of the left part,
-* 1 integer for the end element offset of the left part,
+* 1 integer for the start span offset of the left part,
+* 1 integer for the end span offset of the left part,
* 1 integer for end position of the left part,
* 1 integer for end position of the right part, and
* and 0-3 TUIs as above.
@@ -151,15 +151,15 @@
>:dependency$<b>34<i>27<i>34<i>2<i>3<s>5<s>4<s>3
means the left part ends at token position 2, and right part is a
-term ending at position 3.
+token ending at position 3.
-4) Element to element relation has
+4) Span to span relation has
* 1 byte for PTI (35),
-* 1 integer for the start element offset of the left part,
-* 1 integer for the end element offset of the left part,
-* 1 integer for the start element offset of the right part,
-* 1 integer for the end element offset of the right part,
+* 1 integer for the start span offset of the left part,
+* 1 integer for the end span offset of the left part,
+* 1 integer for the start span offset of the right part,
+* 1 integer for the end span offset of the right part,
* 1 integer for end position of the left part,
* 1 integer for the start position of the right part,
* 1 integer for end position of the right part,
@@ -169,34 +169,34 @@
>:dependency$<b>35<i>27<i>34<i>35<i>40<i>2<i>3<i>4<s>5<s>4<s>3
-means the left part ends at token position 2, the right part is an
-element starting at position 3 and ending at position 4.
+means the left part ends at token position 2, the right part is a
+span starting at position 3 and ending at position 4.
*PTIs* (it’s a relation payload if the third bit is set):
- 32 term to term relation (with optional relation TUI and certainty)
- 33 term to element relation (with optional relation TUI and certainty)
- 34 element to term relation (with optional relation TUI and certainty)
- 35 element to element relation (with optional relation TUI and certainty)
+ 32 token to token relation (with optional relation TUI and certainty)
+ 33 token to span relation (with optional relation TUI and certainty)
+ 34 span to token relation (with optional relation TUI and certainty)
+ 35 span to span relation (with optional relation TUI and certainty)
### Attribute payloads
Each attribute has two payloads:
-* the TUI of the term, element or relation to which the attribute
+* the TUI of the token, span or relation to which the attribute
belongs to (stored in short)
-* for elements: the corresponding element end position (stored in integer)
+* for spans: the corresponding span end position (stored in integer)
For example:
@:class=header$<b>17<i>6<s>1
-means the attribute belongs to the element in the
+means the attribute belongs to the span in the
same token position whose TUI is 1 and end position is 6.
*PTIs* (it’s an attribute payload, if the fourth bit is set):
- 16. Attribute for Terms
- 17. Attribute for Elements
+ 16. Attribute for Tokens
+ 17. Attribute for Spans
18. Attribute for Relations
## Computed Payloads
diff --git a/pom.xml b/pom.xml
index 4b2b844..31efb78 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Krill</artifactId>
- <version>0.55.6</version>
+ <version>0.55.7</version>
<packaging>jar</packaging>
<name>Krill</name>
diff --git a/src/main/java/de/ids_mannheim/korap/Krill.java b/src/main/java/de/ids_mannheim/korap/Krill.java
index de55528..437e1e9 100644
--- a/src/main/java/de/ids_mannheim/korap/Krill.java
+++ b/src/main/java/de/ids_mannheim/korap/Krill.java
@@ -178,10 +178,10 @@
// Throw an error, in case the query matches everywhere
if (qw.isEmpty()) {
this.addError(780, "This query matches everywhere");
- }
- else if (qw.isNull()) {
- this.addError(783, "This query can't match anywhere");
- }
+ }
+ else if (qw.isNull()) {
+ this.addError(783, "This query can't match anywhere");
+ }
else {
diff --git a/src/main/java/de/ids_mannheim/korap/KrillCollection.java b/src/main/java/de/ids_mannheim/korap/KrillCollection.java
index d839422..d49c4d6 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillCollection.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillCollection.java
@@ -8,7 +8,6 @@
import de.ids_mannheim.korap.util.QueryException;
import de.ids_mannheim.korap.response.Result;
-import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.*;
import org.apache.lucene.index.*;
import org.apache.lucene.util.BytesRef;
diff --git a/src/main/java/de/ids_mannheim/korap/KrillQuery.java b/src/main/java/de/ids_mannheim/korap/KrillQuery.java
index 3185603..19fd398 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillQuery.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillQuery.java
@@ -115,7 +115,7 @@
// Set max boundary
this.max = json.has("max") ? json.get("max").asInt(defaultMax)
: defaultMax;
-
+
if (DEBUG)
log.trace("Found koral:boundary with {}:{}", min, max);
};
@@ -723,12 +723,12 @@
// Check relation between min and max
if (min > max)
max = max;
-
+
SpanQueryWrapper sqw = this._fromKoral(operands.get(0));
if (sqw.maybeExtension())
return sqw.setMin(min).setMax(max);
-
+
return new SpanRepetitionQueryWrapper(sqw, min, max);
};
@@ -1193,18 +1193,23 @@
// Branch on type
switch (json.get("type").asText()) {
- case "type:regex":
- return qb.seg(qb.re(value.toString(), isCaseInsensitive));
+ case "type:regex": {
- case "type:wildcard":
- return qb.seq(qb.wc(value.toString(), isCaseInsensitive));
+ // The regex can be rewritten to an any token
+ if (value.toString().matches("^[si]:\\.[\\+\\*]\\??$")) {
+ return new SpanRepetitionQueryWrapper();
+ };
+ return qb.seg(qb.re(value.toString(), isCaseInsensitive));
+ }
+ case "type:wildcard":
+ return qb.seq(qb.wc(value.toString(), isCaseInsensitive));
- case "type:string":
- break;
+ case "type:string":
+ break;
- default:
- this.addWarning(746,
- "Term type is not supported - treated as a string");
+ default:
+ this.addWarning(746,
+ "Term type is not supported - treated as a string");
};
};
diff --git a/src/main/java/de/ids_mannheim/korap/index/FieldDocument.java b/src/main/java/de/ids_mannheim/korap/index/FieldDocument.java
index f1f412d..19f1ec5 100644
--- a/src/main/java/de/ids_mannheim/korap/index/FieldDocument.java
+++ b/src/main/java/de/ids_mannheim/korap/index/FieldDocument.java
@@ -23,8 +23,14 @@
import java.util.*;
/*
- Todo: Store primary data at base/cons field.
+ TODO: Store primary data at base/cons field.
All other Termvectors should have no stored field!
+
+ TODO: Currently Character offsets are stored with positional
+ information in the token stream. This is bad!
+ The character offset may need a special encoding in Lucene
+ To store the character offsets directly (not in the payloads),
+ to make this less messy and speed things up.
*/
/**
diff --git a/src/main/java/de/ids_mannheim/korap/query/QueryBuilder.java b/src/main/java/de/ids_mannheim/korap/query/QueryBuilder.java
index 72cfb23..96e4eb5 100644
--- a/src/main/java/de/ids_mannheim/korap/query/QueryBuilder.java
+++ b/src/main/java/de/ids_mannheim/korap/query/QueryBuilder.java
@@ -257,7 +257,7 @@
return ssq;
};
-
+
/**
* Create an empty query segment.
*
diff --git a/src/main/java/de/ids_mannheim/korap/query/wrap/SpanAlterQueryWrapper.java b/src/main/java/de/ids_mannheim/korap/query/wrap/SpanAlterQueryWrapper.java
index a7b55cf..8fd18cd 100644
--- a/src/main/java/de/ids_mannheim/korap/query/wrap/SpanAlterQueryWrapper.java
+++ b/src/main/java/de/ids_mannheim/korap/query/wrap/SpanAlterQueryWrapper.java
@@ -83,7 +83,6 @@
if (term.maybeUnsorted())
this.maybeUnsorted = true;
-
this.isNull = false;
return this;
};
diff --git a/src/main/java/de/ids_mannheim/korap/query/wrap/SpanExpansionQueryWrapper.java b/src/main/java/de/ids_mannheim/korap/query/wrap/SpanExpansionQueryWrapper.java
new file mode 100644
index 0000000..7c4cf61
--- /dev/null
+++ b/src/main/java/de/ids_mannheim/korap/query/wrap/SpanExpansionQueryWrapper.java
@@ -0,0 +1,68 @@
+package de.ids_mannheim.korap.query.wrap;
+
+import org.apache.lucene.search.RegexpQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.util.automaton.RegExp;
+import org.apache.lucene.index.Term;
+import de.ids_mannheim.korap.query.wrap.SpanQueryWrapper;
+import de.ids_mannheim.korap.query.SpanExpansionQuery;
+import de.ids_mannheim.korap.util.QueryException;
+
+
+import java.util.*;
+
+/*
+ * TODO: SpanExpansionQueryWrapper currently does not support negative extensions!
+ */
+/**
+ * @author diewald
+ */
+
+public class SpanExpansionQueryWrapper extends SpanQueryWrapper {
+ private SpanQueryWrapper anchor;
+
+ // < 0 to the left of anchor span
+ // >= 0 to the right of anchor span
+ private int direction;
+
+ // if > 0, collect expansion offsets
+ // using this label
+ private byte classNumber;
+
+
+ public SpanExpansionQueryWrapper (SpanQueryWrapper anchor,
+ int min,
+ int max,
+ int direction,
+ byte classNumber
+ ) {
+ this.anchor = anchor;
+ this.isNull = false;
+ this.min = min;
+ this.max = max;
+ this.direction = direction;
+ this.classNumber = classNumber;
+ this.isExtended = true;
+ if (direction >= 0)
+ this.isExtendedToTheRight = true;
+ };
+
+ @Override
+ public boolean isNull () {
+ // Needs to be overwritten, as min and max do not indicate null value
+ return this.isNull;
+ };
+
+
+ @Override
+ public SpanQuery toFragmentQuery () throws QueryException {
+ return new SpanExpansionQuery(
+ this.anchor.retrieveNode(this.retrieveNode).toFragmentQuery(),
+ this.getMin(),
+ this.getMax(),
+ this.direction,
+ this.classNumber,
+ true
+ );
+ };
+};
diff --git a/src/main/java/de/ids_mannheim/korap/query/wrap/SpanQueryWrapper.java b/src/main/java/de/ids_mannheim/korap/query/wrap/SpanQueryWrapper.java
index 0855885..f0221d5 100644
--- a/src/main/java/de/ids_mannheim/korap/query/wrap/SpanQueryWrapper.java
+++ b/src/main/java/de/ids_mannheim/korap/query/wrap/SpanQueryWrapper.java
@@ -7,6 +7,7 @@
import de.ids_mannheim.korap.query.SpanWithinQuery;
// TODO: Add warnings and errors - using KrillQuery
+// TODO: Rename isEmpty to isAny!
/**
* A wrapper base class for Lucene SpanQueries,
@@ -35,7 +36,7 @@
protected boolean hasClass = false, isNull = true, isOptional = false,
isNegative = false, isEmpty = false, isExtended = false,
isExtendedToTheRight = false, maybeUnsorted = false,
- retrieveNode = false;
+ retrieveNode = false, isProblematic = false;
/**
@@ -46,7 +47,6 @@
* @throws QueryException
*/
public SpanQuery toFragmentQuery () throws QueryException {
- System.err.println("||||||||||||||||||||||||||");
return (SpanQuery) null;
};
@@ -61,8 +61,9 @@
*/
public SpanQuery toQuery () throws QueryException {
- if (this.isNull() || this.isEmpty())
+ if (this.isNull() || this.isEmpty()) {
return null;
+ };
// Wrap the query in a <base/s=t>, if it's extended to the right
if (this.isExtendedToTheRight()) {
@@ -94,11 +95,18 @@
return this.isOptional;
};
+
+ public SpanQueryWrapper isOptional (boolean opt) {
+ this.isOptional = opt;
+ return this;
+ };
+
+
/**
* Boolean value indicating that the wrapped query is
* <tt>null</tt>, meaning it doesn't match anything at
- * all.
+ * all (i.e. a zero-width assertion).
*
* For example the segment denoting an adjective
* in the following Poliqarp expression doen't match
@@ -211,7 +219,6 @@
return this;
};
-
/**
* Check, if the wrapped query can be used as an
* anchor query in a sequence, i.e. a query that
@@ -412,8 +419,10 @@
*/
public String toString () {
String string = "" + (this.isNull() ? "isNull" : "notNull") + "-"
- + (this.isEmpty() ? "isEmpty" : "notEmpty") + "-"
- + (this.isOptional() ? "isOptional" : "notOptional");
+ + (this.isEmpty() ? "isEmpty" : "notEmpty") + "-"
+ + (this.isOptional() ? "isOptional" : "notOptional") + "-"
+ + (this.isExtendedToTheRight() ? "isExtendedToTheRight" : "notExtendedToTheRight");
+ ;
return string;
};
};
diff --git a/src/main/java/de/ids_mannheim/korap/query/wrap/SpanSequenceQueryWrapper.java b/src/main/java/de/ids_mannheim/korap/query/wrap/SpanSequenceQueryWrapper.java
index 72fadb2..c5aac76 100644
--- a/src/main/java/de/ids_mannheim/korap/query/wrap/SpanSequenceQueryWrapper.java
+++ b/src/main/java/de/ids_mannheim/korap/query/wrap/SpanSequenceQueryWrapper.java
@@ -18,7 +18,6 @@
/*
TODO: Make isNegative work!
- TODO: Make isEmpty work!
TODO: Evaluate if spanNext(spanNext(a,b),spanNext(c,d)) is faster
than spanNext(spanNext(spanNext(a,b),c),d)
TODO: Improve support for SpanElementQueryWrapper in constraints!
@@ -612,6 +611,7 @@
public boolean isEmpty () {
+
if (this.segments.size() == 1)
return this.segments.get(0).isEmpty();
@@ -727,7 +727,7 @@
if (query == null)
return (SpanQuery) null;
- // NextQueries:
+ // NextQueries
if (!this.hasConstraints() && this.isInOrder()) {
for (; i < this.segments.size(); i++) {
@@ -741,6 +741,12 @@
return (SpanQuery) query;
};
+ // DistanceQueries with problems
+ if (this.hasConstraints() && this.isProblematic) {
+ throw new QueryException(613,
+ "Distance constraints not supported with empty, optional or negative operands");
+ };
+
// DistanceQueries
if (this.constraints.size() == 1) {
DistanceConstraint constraint = this.constraints.get(0);
@@ -753,6 +759,11 @@
if (this.segments.get(i).isExtended())
throw new QueryException(613, limitationError);
+ /* Maybe important
+ if (this.segments.get(i).isOptional())
+ throw new QueryException(613, limitationError);
+ */
+
SpanQuery sq = (SpanQuery) this.segments.get(i)
.retrieveNode(this.retrieveNode).toFragmentQuery();
if (sq == null)
@@ -772,6 +783,11 @@
if (this.segments.get(i).isExtended())
throw new QueryException(613, limitationError);
+ /* May be necessary
+ if (this.segments.get(i).isOptional())
+ throw new QueryException(613, limitationError);
+ */
+
SpanQuery sq = (SpanQuery) this.segments.get(i)
.retrieveNode(this.retrieveNode).toFragmentQuery();
if (sq == null)
@@ -817,6 +833,8 @@
- look for an anchor next to it
- merge the problematic segment with the anchor
- go on
+
+ - This does not work for distance constraints!
*/
private boolean _solveProblematicSequence () {
int size = this.segments.size();
@@ -838,6 +856,18 @@
if (DEBUG)
log.trace("segment {} is problematic", i);
+ // Sequences with distance constraints do not support empty or optional
+ // elemets
+ if (this.hasConstraints()) {
+ if (DEBUG) {
+ log.trace("Sequence has constraints, "
+ + "that do not allow empty or optional elements");
+ };
+ this.isSolved = true;
+ this.isProblematic = true;
+ return false;
+ };
+
// [problem][anchor]
if (i < (size - 1) && this.segments.get(i + 1).maybeAnchor()) {
if (DEBUG)
@@ -853,6 +883,8 @@
// An error occurred while solving the problem
catch (QueryException e) {
+ this.isSolved = true;
+ this.isProblematic = true;
return false;
};
@@ -880,6 +912,8 @@
true));
}
catch (QueryException e) {
+ this.isSolved = true;
+ this.isProblematic = true;
return false;
};
@@ -916,15 +950,17 @@
return _solveProblematicSequence();
this.isSolved = true;
- return true;
+ this.isProblematic = true;
+ return false; // true;
};
this.isSolved = true;
+ this.isProblematic = false;
return false;
};
- // Todo: Deal with negative and optional!
+ // Todo: Deal with negative, empty and optional!
// [base=der][base!=Baum]?
private SpanQueryWrapper _merge (SpanQueryWrapper anchor,
SpanQueryWrapper problem, boolean mergeLeft) throws QueryException {
@@ -944,15 +980,46 @@
log.trace("Problem is empty with class {}",
problem.getClassNumber());
- query = new SpanExpansionQuery(anchor.retrieveNode(
- this.retrieveNode).toFragmentQuery(),
- problem.isOptional() ? 0 : problem.getMin(),
- problem.getMax(), direction,
- problem.hasClass() ? problem.getClassNumber() : (byte) 0,
- true);
- SpanQueryWrapper sqw = new SpanSimpleQueryWrapper(query)
- .isExtended(true);
+ // Merge extensions!
+ if (!problem.hasClass() &&
+ !anchor.hasClass() &&
+ anchor.isExtended()) {
+ if (DEBUG)
+ log.trace("It may be possible to extend anchor with problem");
+
+ if (
+ // Further extend to the right ...
+ (direction >= 0 && anchor.isExtendedToTheRight) ||
+
+ // or the left
+ (direction < 0 && !anchor.isExtendedToTheRight)) {
+
+ if (DEBUG)
+ log.trace("Readjust min and max");
+
+ // Readjust the anchor
+ anchor.setMin(anchor.getMin() + problem.getMin());
+ anchor.setMax(anchor.getMax() + problem.getMax());
+
+ /*
+ * This is wrong - min is only relevant for extensions
+ if (anchor.getMin() > 0)
+ anchor.isOptional = false;
+ */
+ return anchor;
+ };
+ };
+
+ // Can't merge extensions
+ SpanQueryWrapper sqw = new SpanExpansionQueryWrapper(
+ anchor,
+ problem.isOptional() ? 0 : problem.getMin(),
+ problem.getMax(),
+ direction,
+ problem.hasClass() ? problem.getClassNumber() : (byte) 0
+ ).isExtended(true);
+
// Set right extension
if (direction >= 0)
sqw.isExtendedToTheRight(true);
@@ -969,15 +1036,17 @@
log.trace("Problem is negative with class {}",
problem.getClassNumber());
- query = new SpanExpansionQuery(anchor.retrieveNode(
- this.retrieveNode).toFragmentQuery(), problem.retrieveNode(
- this.retrieveNode).toFragmentQuery(), problem.getMin(),
- problem.getMax(), direction,
- problem.hasClass() ? problem.getClassNumber() : (byte) 0,
- true);
+ // TODO: Should probably wrapped as well!
+ // A sequence of negative tokens may expand jointly!
+ query = new SpanExpansionQuery(anchor.retrieveNode(
+ this.retrieveNode).toFragmentQuery(), problem.retrieveNode(
+ this.retrieveNode).toFragmentQuery(), problem.getMin(),
+ problem.getMax(), direction,
+ problem.hasClass() ? problem.getClassNumber() : (byte) 0,
+ true);
SpanQueryWrapper sqw = new SpanSimpleQueryWrapper(query)
- .isExtended(true);
+ .isExtended(true);
// Set right extension
if (direction >= 0)
@@ -989,24 +1058,29 @@
if (DEBUG)
log.trace("Problem is optional");
+ // [base=der][][base=Baum]?
// [base=der][base=baum]?
// [base=der]
- SpanAlterQueryWrapper saqw = new SpanAlterQueryWrapper(this.field,
- anchor);
+ SpanAlterQueryWrapper saqw = new SpanAlterQueryWrapper(
+ this.field,
+ anchor
+ );
// [base=der]
SpanSequenceQueryWrapper ssqw = new SpanSequenceQueryWrapper(
- this.field, anchor);
+ this.field,
+ anchor
+ );
// [base=der][base=baum]
- if (mergeLeft)
- ssqw.append(new SpanSimpleQueryWrapper(problem.retrieveNode(
- this.retrieveNode).toFragmentQuery()));
+ if (mergeLeft) {
+ ssqw.append(problem.isOptional(false));
+ }
// [base=baum][base=der]
- else
- ssqw.prepend(new SpanSimpleQueryWrapper(problem.retrieveNode(
- this.retrieveNode).toFragmentQuery()));
+ else {
+ ssqw.prepend(problem.isOptional(false));
+ }
saqw.or(ssqw);
diff --git a/src/main/java/de/ids_mannheim/korap/query/wrap/SpanSimpleQueryWrapper.java b/src/main/java/de/ids_mannheim/korap/query/wrap/SpanSimpleQueryWrapper.java
index 61608e5..3d9c76a 100644
--- a/src/main/java/de/ids_mannheim/korap/query/wrap/SpanSimpleQueryWrapper.java
+++ b/src/main/java/de/ids_mannheim/korap/query/wrap/SpanSimpleQueryWrapper.java
@@ -3,6 +3,7 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
+import de.ids_mannheim.korap.util.QueryException;
public class SpanSimpleQueryWrapper extends SpanQueryWrapper {
private SpanQuery query;
@@ -25,6 +26,20 @@
this.query = query;
};
+ // This is similar to a clone
+ public SpanSimpleQueryWrapper (SpanQueryWrapper query) throws QueryException {
+ this.hasClass = query.hasClass();
+ this.isOptional = query.isOptional();
+ this.isNegative = query.isNegative();
+ this.isEmpty = query.isEmpty();
+ this.isExtended = query.isExtended();
+ this.isExtendedToTheRight = query.isExtendedToTheRight();
+ this.maybeUnsorted = query.maybeUnsorted();
+ this.retrieveNode = query.retrieveNode;
+ this.query = query.toFragmentQuery();
+ this.isNull = query.isNull();
+ };
+
@Override
public SpanQuery toFragmentQuery () {
diff --git a/src/main/java/de/ids_mannheim/korap/util/KrillString.java b/src/main/java/de/ids_mannheim/korap/util/KrillString.java
index 602709b..d3da39d 100644
--- a/src/main/java/de/ids_mannheim/korap/util/KrillString.java
+++ b/src/main/java/de/ids_mannheim/korap/util/KrillString.java
@@ -1,7 +1,9 @@
package de.ids_mannheim.korap.util;
import java.io.*;
+import java.net.URLDecoder;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
@@ -26,7 +28,10 @@
*/
public static String StringfromFile (String path, Charset encoding)
throws IOException {
- byte[] encoded = Files.readAllBytes(Paths.get(path));
+ path = URLDecoder.decode(path,"UTF-8");
+ path = path.replaceFirst("^/(.:/)", "$1");
+ Path p = Paths.get(path);
+ byte[] encoded = Files.readAllBytes(p);
return new String(encoded, encoding);
};
diff --git a/src/test/java/de/ids_mannheim/korap/TestSimple.java b/src/test/java/de/ids_mannheim/korap/TestSimple.java
index 17179aa..6451ae4 100644
--- a/src/test/java/de/ids_mannheim/korap/TestSimple.java
+++ b/src/test/java/de/ids_mannheim/korap/TestSimple.java
@@ -2,6 +2,7 @@
import java.util.*;
import java.io.*;
+import java.net.URLDecoder;
import static org.junit.Assert.*;
@@ -95,7 +96,7 @@
SpanQueryWrapper sqwi;
try {
- String json = getString(jsonFile);
+ String json = getJsonString(jsonFile);
sqwi = new KrillQuery("tokens").fromKoral(json);
}
catch (QueryException e) {
@@ -107,9 +108,11 @@
// Get string
- public static String getString (String path) {
+ public static String getJsonString (String path) {
+
StringBuilder contentBuilder = new StringBuilder();
try {
+ path = URLDecoder.decode(path, "UTF-8");
BufferedReader in = new BufferedReader(new FileReader(path));
String str;
while ((str = in.readLine()) != null) {
diff --git a/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionJSON.java b/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionJSON.java
index 9232711..8b08bda 100644
--- a/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionJSON.java
+++ b/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionJSON.java
@@ -186,7 +186,7 @@
// Legacy collections reflect old tests, that were adopted to the new scheme
@Test
public void metaQuery1Legacy () {
- String metaQuery = getString(getClass().getResource(
+ String metaQuery = getJsonString(getClass().getResource(
"/queries/metaquery.jsonld").getFile());
KrillCollection kc = new KrillCollection(metaQuery);
@@ -211,7 +211,7 @@
@Test
public void metaQuery2Legacy () {
- String metaQuery = getString(getClass().getResource(
+ String metaQuery = getJsonString(getClass().getResource(
"/queries/metaquery2.jsonld").getFile());
KrillCollection kc = new KrillCollection(metaQuery);
/*
@@ -227,7 +227,7 @@
@Test
public void metaQuery3Legacy () {
- String metaQuery = getString(getClass().getResource(
+ String metaQuery = getJsonString(getClass().getResource(
"/queries/metaquery4.jsonld").getFile());
KrillCollection kc = new KrillCollection(metaQuery);
/*
@@ -245,7 +245,7 @@
@Test
public void metaQuery7Legacy () {
- String metaQuery = getString(getClass().getResource(
+ String metaQuery = getJsonString(getClass().getResource(
"/queries/metaquery7.jsonld").getFile());
KrillCollection kc = new KrillCollection(metaQuery);
/*
@@ -266,7 +266,7 @@
@Test
public void metaQuery9 () {
- String metaQuery = getString(getClass().getResource(
+ String metaQuery = getJsonString(getClass().getResource(
"/queries/metaquery9.jsonld").getFile());
KrillCollection kc = new KrillCollection(metaQuery);
/*
@@ -279,6 +279,6 @@
private String _getJSONString (String file) {
- return getString(getClass().getResource(path + file).getFile());
+ return getJsonString(getClass().getResource(path + file).getFile());
};
};
diff --git a/src/test/java/de/ids_mannheim/korap/highlight/TestClass.java b/src/test/java/de/ids_mannheim/korap/highlight/TestClass.java
index 0b1eb95..2371d3d 100644
--- a/src/test/java/de/ids_mannheim/korap/highlight/TestClass.java
+++ b/src/test/java/de/ids_mannheim/korap/highlight/TestClass.java
@@ -6,6 +6,7 @@
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
+import java.net.URLDecoder;
import org.apache.lucene.search.spans.SpanQuery;
import org.junit.Test;
@@ -29,8 +30,8 @@
@Test
public void queryJSONpoly1 () throws QueryException, IOException {
- String jsonPath = getClass().getResource("/queries/poly1.json")
- .getFile();
+ String jsonPath = URLDecoder.decode(getClass().getResource("/queries/poly1.json")
+ .getFile(),"UTF-8");
String jsonQuery = readFile(jsonPath);
SpanQueryWrapper sqwi = new KrillQuery("tokens").fromKoral(jsonQuery);
@@ -64,8 +65,8 @@
@Test
public void queryJSONpoly4 () throws QueryException, IOException {
- String jsonPath = getClass().getResource("/queries/poly4.json")
- .getFile();
+ String jsonPath = URLDecoder.decode(getClass().getResource("/queries/poly4.json")
+ .getFile(),"UTF-8");
String jsonQuery = readFile(jsonPath);
SpanQueryWrapper sqwi = new KrillQuery("tokens").fromKoral(jsonQuery);
SpanQuery sq = sqwi.toQuery();
diff --git a/src/test/java/de/ids_mannheim/korap/highlight/TestHighlight.java b/src/test/java/de/ids_mannheim/korap/highlight/TestHighlight.java
index 45ae9e3..43c15eb 100644
--- a/src/test/java/de/ids_mannheim/korap/highlight/TestHighlight.java
+++ b/src/test/java/de/ids_mannheim/korap/highlight/TestHighlight.java
@@ -277,7 +277,7 @@
ki.commit();
// 15
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bugs/greater_highlights_15.jsonld").getFile());
Krill ks = new Krill(json);
@@ -292,7 +292,7 @@
kr.getMatch(0).getSnippetHTML(),
"<span class=\"context-left\"><span class=\"more\"></span>2. Herkunft Die aus dem proto-semitischen </span><span class=\"match\"><mark><mark class=\"class-15 level-0\">Alphabet</mark></mark></span><span class=\"context-right\"> stammende Urform des Buchstaben ist wahrscheinlich<span class=\"more\"></span></span>");
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/bugs/greater_highlights_16.jsonld").getFile());
// 16
@@ -309,7 +309,7 @@
"<span class=\"context-left\"><span class=\"more\"></span>2. Herkunft Die aus dem proto-semitischen </span><span class=\"match\"><mark><mark class=\"class-16 level-0\">Alphabet</mark></mark></span><span class=\"context-right\"> stammende Urform des Buchstaben ist wahrscheinlich<span class=\"more\"></span></span>");
// 127
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/bugs/greater_highlights_127.jsonld").getFile());
ks = new Krill(json);
@@ -325,7 +325,7 @@
"<span class=\"context-left\"><span class=\"more\"></span>2. Herkunft Die aus dem proto-semitischen </span><span class=\"match\"><mark><mark class=\"class-127 level-0\">Alphabet</mark></mark></span><span class=\"context-right\"> stammende Urform des Buchstaben ist wahrscheinlich<span class=\"more\"></span></span>");
// 255
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/bugs/greater_highlights_255.jsonld").getFile());
ks = new Krill(json);
@@ -341,7 +341,7 @@
"<span class=\"context-left\"><span class=\"more\"></span>2. Herkunft Die aus dem proto-semitischen </span><span class=\"match\"><mark>Alphabet</mark></span><span class=\"context-right\"> stammende Urform des Buchstaben ist wahrscheinlich<span class=\"more\"></span></span>");
// 300
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/bugs/greater_highlights_300.jsonld").getFile());
ks = new Krill(json);
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestElementDistanceIndex.java b/src/test/java/de/ids_mannheim/korap/index/TestElementDistanceIndex.java
index 026a86a..7716322 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestElementDistanceIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestElementDistanceIndex.java
@@ -5,6 +5,7 @@
import java.util.*;
import java.io.*;
+import java.net.URLDecoder;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.spans.SpanQuery;
@@ -205,8 +206,9 @@
ki.addDoc(getClass().getResourceAsStream("/wiki/00001.json.gz"), true);
ki.commit();
- SpanQueryWrapper sqwi = jsonQuery(getClass().getResource(
- "/queries/cosmas1.json").getFile());
+ String jsonPath = URLDecoder.decode(getClass().getResource("/queries/cosmas1.json")
+ .getFile(),"UTF-8");
+ SpanQueryWrapper sqwi = jsonQuery(jsonPath);
kr = ki.search(sqwi.toQuery(), (short) 10);
assertEquals((long) 3, kr.getTotalResults());
@@ -233,8 +235,9 @@
assertEquals(kr.getTotalResults(), 1);
assertEquals("[[ecebdc]]", kr.getMatch(0).getSnippetBrackets());
- sqwi = jsonQuery(getClass().getResource(
- "/queries/distances/in-same-t.jsonld").getFile());
+ String jsonPath = URLDecoder.decode(getClass().getResource("/queries/distances/in-same-t.jsonld")
+ .getFile(),"UTF-8");
+ sqwi = jsonQuery(jsonPath);
assertEquals(
"spanElementDistance(tokens:s:c, tokens:s:e, [(base/s:t[0:0], ordered, notExcluded)])",
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java b/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java
index 0962e64..a570e93 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java
@@ -6,6 +6,7 @@
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
+import java.net.URLDecoder;
import org.apache.lucene.search.spans.SpanQuery;
import org.junit.Test;
@@ -198,9 +199,11 @@
};
ki.commit();
+ String jsonPath = URLDecoder.decode(getClass().getResource("/queries/bsp18.jsonld")
+ .getFile(),"UTF-8");
+
// {1:der} \w0:5 nicht
- SpanQueryWrapper sqwi = jsonQuery(getClass().getResource(
- "/queries/bsp18.jsonld").getFile());
+ SpanQueryWrapper sqwi = jsonQuery(jsonPath);
Result kr = ki.search(sqwi.toQuery(), 0, (short) 5, true, (short) 2,
false, (short) 5);
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestMatchIdentifier.java b/src/test/java/de/ids_mannheim/korap/index/TestMatchIdentifier.java
index 7859614..072a633 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestMatchIdentifier.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestMatchIdentifier.java
@@ -274,6 +274,13 @@
assertEquals("tokens", res.at("/field").asText());
assertEquals("GOE_AGX.00002", res.at("/textSigle").asText());
assertEquals("Goethe, Johann Wolfgang von", res.at("/author").asText());
+
+ /*
+ km = ki.getMatchInfo(
+ "match-GOE!GOE_AGX.00002-p10-20",
+ "tokens", true, (String) null, (String) null, true, true, false);
+ assertEquals("", km.toJsonString());
+ */
};
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestRepetitionIndex.java b/src/test/java/de/ids_mannheim/korap/index/TestRepetitionIndex.java
index a76bc4d..fd6b3b3 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestRepetitionIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestRepetitionIndex.java
@@ -257,6 +257,7 @@
// }
}
+
@Test
public void testCase5 () throws IOException {
ki = new KrillIndex();
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestSpanExpansionIndex.java b/src/test/java/de/ids_mannheim/korap/index/TestSpanExpansionIndex.java
index 08f56a8..7af71f0 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestSpanExpansionIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestSpanExpansionIndex.java
@@ -6,6 +6,7 @@
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
+import java.net.URLDecoder;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.RegexpQuery;
@@ -295,9 +296,8 @@
ki.addDoc(getClass().getResourceAsStream("/wiki/00001.json.gz"), true);
ki.addDoc(getClass().getResourceAsStream("/wiki/00002.json.gz"), true);
ki.commit();
-
- String jsonPath = getClass().getResource("/queries/poly3.json")
- .getFile();
+ String jsonPath = URLDecoder.decode(getClass().getResource("/queries/poly3.json")
+ .getFile(),"UTF-8");
String jsonQuery = readFile(jsonPath);
SpanQueryWrapper sqwi = new KrillQuery("tokens").fromKoral(jsonQuery);
@@ -405,9 +405,9 @@
ki.addDoc(createFieldDoc3());
ki.addDoc(createFieldDoc4());
ki.commit();
-
- String json = readFile(getClass().getResource(
- "/queries/bugs/expansion_bug_3.jsonld").getFile());
+ String jsonPath = URLDecoder.decode(getClass().getResource("/queries/bugs/expansion_bug_3.jsonld")
+ .getFile(),"UTF-8");
+ String json = readFile(jsonPath);
KrillQuery kq = new KrillQuery("base");
SpanQuery sq = kq.fromKoral(json).toQuery();
assertEquals(
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestWithinIndex.java b/src/test/java/de/ids_mannheim/korap/index/TestWithinIndex.java
index f77f42a..7960560 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestWithinIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestWithinIndex.java
@@ -6,6 +6,7 @@
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
+import java.net.URLDecoder;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.spans.SpanQuery;
@@ -1107,8 +1108,8 @@
// contains(<s>, (es wird | wird es))
@Test
public void queryJSONpoly2 () throws QueryException, IOException {
- String jsonPath = getClass().getResource("/queries/poly2.json")
- .getFile();
+ String jsonPath = URLDecoder.decode(getClass().getResource("/queries/poly2.json")
+ .getFile(),"UTF-8");
String jsonPQuery = readFile(jsonPath);
SpanQueryWrapper sqwi = new KrillQuery("tokens").fromKoral(jsonPQuery);
@@ -1144,8 +1145,8 @@
at de.ids_mannheim.korap.Krill.apply(Krill.java:304)
*/
- String jsonPath = getClass().getResource(
- "/queries/bugs/span_or_bug.jsonld").getFile();
+ String jsonPath = URLDecoder.decode(getClass().getResource(
+ "/queries/bugs/span_or_bug.jsonld").getFile(),"UTF-8");
String jsonPQuery = readFile(jsonPath);
SpanQueryWrapper sqwi = new KrillQuery("tokens").fromKoral(jsonPQuery);
diff --git a/src/test/java/de/ids_mannheim/korap/query/TestKrillQueryJSON.java b/src/test/java/de/ids_mannheim/korap/query/TestKrillQueryJSON.java
index 31a3612..d66be4f 100644
--- a/src/test/java/de/ids_mannheim/korap/query/TestKrillQueryJSON.java
+++ b/src/test/java/de/ids_mannheim/korap/query/TestKrillQueryJSON.java
@@ -2,6 +2,7 @@
import java.util.*;
import java.io.*;
+import java.net.URLDecoder;
import org.apache.lucene.search.spans.SpanQuery;
import de.ids_mannheim.korap.query.wrap.SpanQueryWrapper;
@@ -571,15 +572,17 @@
};
};
- @Test
+
+ @Test
public void queryJSONdistancesWithRegexes () throws QueryException {
- // "der" []{2,3} [opennlp/p="NN"]
+ // "der" []{2,3} [opennlp/p="NN"]
try {
String json = getString(getClass().getResource(
"/queries/bugs/distances_with_regex_bug.jsonld").getFile());
KrillQuery kq = new KrillQuery("tokens");
- assertEquals(kq.fromKoral(json).toQuery().toString(),
+ assertEquals(
+ kq.fromKoral(json).toQuery().toString(),
"spanDistance(SpanMultiTermQueryWrapper(tokens:/s:der/), SpanMultiTermQueryWrapper(tokens:/opennlp/p:NN/), [(w[3:4], ordered, notExcluded)])");
}
catch (QueryException e) {
@@ -587,10 +590,35 @@
};
};
+ @Test
+ public void queryJSONregexRewrite1 () throws QueryException {
+ // "der" [.+?]
+ String json = getString(getClass().getResource(
+ "/queries/sequence/regex-rewrite-1.jsonld").getFile());
+ KrillQuery kq = new KrillQuery("tokens");
+
+ assertEquals(
+ kq.fromKoral(json).toQuery().toString(),
+ "focus(254: spanContain(<tokens:base/s:t />, {254: spanExpansion(tokens:s:der, []{1, 1}, right)}))");
+ };
+
+ @Test
+ public void queryJSONregexRewrite2 () throws QueryException {
+ // "der" [.*] [.*?] [.+] [.+?]
+ String json = getString(getClass().getResource(
+ "/queries/sequence/regex-rewrite-2.jsonld").getFile());
+ KrillQuery kq = new KrillQuery("tokens");
+
+ assertEquals(
+ kq.fromKoral(json).toQuery().toString(),
+ "focus(254: spanContain(<tokens:base/s:t />, {254: spanExpansion(tokens:s:der, []{4, 4}, right)}))");
+ };
+
+
public static String getString (String path) {
StringBuilder contentBuilder = new StringBuilder();
try {
- BufferedReader in = new BufferedReader(new FileReader(path));
+ BufferedReader in = new BufferedReader(new FileReader(URLDecoder.decode(path,"UTF-8")));
String str;
while ((str = in.readLine()) != null) {
contentBuilder.append(str);
diff --git a/src/test/java/de/ids_mannheim/korap/query/TestSpanSequenceQueryJSON.java b/src/test/java/de/ids_mannheim/korap/query/TestSpanSequenceQueryJSON.java
index a38aec2..aff5cf8 100644
--- a/src/test/java/de/ids_mannheim/korap/query/TestSpanSequenceQueryJSON.java
+++ b/src/test/java/de/ids_mannheim/korap/query/TestSpanSequenceQueryJSON.java
@@ -283,7 +283,7 @@
}
catch (QueryException qe) {
assertEquals(
- "Distance constraints not supported with empty or negative operands",
+ "Distance constraints not supported with empty, optional or negative operands",
qe.getMessage());
};
};
@@ -352,6 +352,34 @@
};
+ @Test
+ public void queryJSONkoralOptionalityInDistanceBug () {
+ try {
+ // Sonne [] Mond?
+ SpanQueryWrapper sqwi = jsonQueryFile("distance-with-optionality.jsonld");
+ sqwi.toQuery().toString();
+ }
+ catch (QueryException qe) {
+ assertEquals(
+ "Distance constraints not supported with empty, optional or negative operands",
+ qe.getMessage());
+ }
+ // Could also be a distance at the end ... that's a query planner thing.
+ };
+
+
+ @Test
+ public void queryJSONkoralOptionalityAfterEmptyBug () throws QueryException {
+ // Sonne [] Mond?
+ SpanQueryWrapper sqwi = jsonQueryFile("empty-followed-by-optionality.jsonld");
+ assertEquals(
+ "focus(254: spanContain(<tokens:base/s:t />, {254: spanOr([spanExpansion(tokens:s:Sonne, []{1, 1}, right), spanNext(spanExpansion(tokens:s:Sonne, []{1, 1}, right), tokens:s:Mond)])}))",
+ sqwi.toQuery().toString());
+ // Could also be a distance at the end ... that's a query planner thing.
+ };
+
+
+
// get query wrapper based on json file
public SpanQueryWrapper jsonQueryFile (String filename) {
return getJSONQuery(getClass().getResource(path + filename).getFile());
diff --git a/src/test/java/de/ids_mannheim/korap/query/TestTemporaryQueryLimitations.java b/src/test/java/de/ids_mannheim/korap/query/TestTemporaryQueryLimitations.java
index 72ae83e..d651318 100644
--- a/src/test/java/de/ids_mannheim/korap/query/TestTemporaryQueryLimitations.java
+++ b/src/test/java/de/ids_mannheim/korap/query/TestTemporaryQueryLimitations.java
@@ -48,7 +48,7 @@
FieldDocument fd = ki.addDoc(json);
ki.commit();
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/bugs/cosmas_classrefcheck.jsonld").getFile());
Krill ks = new Krill(json);
diff --git a/src/test/java/de/ids_mannheim/korap/search/TestKrill.java b/src/test/java/de/ids_mannheim/korap/search/TestKrill.java
index 1fb1f2b..ab52dd0 100644
--- a/src/test/java/de/ids_mannheim/korap/search/TestKrill.java
+++ b/src/test/java/de/ids_mannheim/korap/search/TestKrill.java
@@ -1,6 +1,6 @@
package de.ids_mannheim.korap.search;
-import static de.ids_mannheim.korap.TestSimple.getString;
+import static de.ids_mannheim.korap.TestSimple.getJsonString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
@@ -129,7 +129,7 @@
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/metaquery3.jsonld").getFile());
Krill ks = new Krill(json);
@@ -161,7 +161,7 @@
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/metaquery4.jsonld").getFile());
Krill ks = new Krill(json);
@@ -176,14 +176,14 @@
assertEquals(kr.getTotalResults(), 5);
- json = getString(getClass().getResource("/queries/metaquery5.jsonld")
+ json = getJsonString(getClass().getResource("/queries/metaquery5.jsonld")
.getFile());
ks = new Krill(json);
kr = ks.apply(ki);
assertEquals(kr.getTotalResults(), 1);
- json = getString(getClass().getResource("/queries/metaquery6.jsonld")
+ json = getJsonString(getClass().getResource("/queries/metaquery6.jsonld")
.getFile());
ks = new Krill(json);
kr = ks.apply(ki);
@@ -231,7 +231,7 @@
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bsp-fail1.jsonld").getFile());
Result kr = new Krill(json).apply(ki);
@@ -254,7 +254,7 @@
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bsp-fail2.jsonld").getFile());
Result kr = new Krill(json).apply(ki);
@@ -271,7 +271,7 @@
public void queryJSONmirrorTestBug () throws IOException {
// Construct index
KrillIndex ki = new KrillIndex();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bugs/failing_mirror.jsonld").getFile());
Krill ks = new Krill(json);
Result kr = ks.apply(ki);
@@ -281,7 +281,7 @@
assertEquals("Unable to parse JSON", res.at("/errors/0/1").asText());
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/bugs/failing_mirror_2.jsonld").getFile());
ks = new Krill(json);
kr = ks.apply(ki);
@@ -311,7 +311,7 @@
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bsp-context.jsonld").getFile());
Krill ks = new Krill(json);
@@ -330,7 +330,7 @@
assertEquals(5, kr.getStartIndex());
assertEquals(5, kr.getItemsPerPage());
- json = getString(getClass()
+ json = getJsonString(getClass()
.getResource("/queries/bsp-context-2.jsonld").getFile());
kr = new Krill(json).apply(ki);
@@ -363,7 +363,7 @@
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bsp-paging.jsonld").getFile());
Krill ks = new Krill(json);
@@ -372,7 +372,7 @@
assertEquals(5, kr.getStartIndex());
assertEquals(5, kr.getItemsPerPage());
- json = getString(getClass().getResource("/queries/bsp-cutoff.jsonld")
+ json = getJsonString(getClass().getResource("/queries/bsp-cutoff.jsonld")
.getFile());
ks = ks = new Krill(json);
kr = ks.apply(ki);
@@ -380,7 +380,7 @@
assertEquals(2, kr.getStartIndex());
assertEquals(2, kr.getItemsPerPage());
- json = getString(getClass().getResource("/queries/metaquery9.jsonld")
+ json = getJsonString(getClass().getResource("/queries/metaquery9.jsonld")
.getFile());
KrillCollection kc = new KrillCollection(json);
kc.setIndex(ki);
@@ -400,7 +400,7 @@
true);
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bsp-itemsPerResource.jsonld").getFile());
Krill ks = new Krill(json);
@@ -480,7 +480,7 @@
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bsp-uid-example.jsonld").getFile());
Krill ks = new Krill(json);
@@ -665,7 +665,7 @@
getClass().getResourceAsStream("/bzk/D59-00089.json.gz"), true);
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bugs/cosmas_boundary.jsonld").getFile());
QueryBuilder kq = new QueryBuilder("tokens");
@@ -723,7 +723,7 @@
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bugs/multiple_classes.jsonld").getFile());
Krill ks = new Krill(json);
@@ -753,7 +753,7 @@
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bugs/multiple_classes.jsonld").getFile());
Krill ks = new Krill(json);
@@ -795,7 +795,7 @@
assertEquals("BZK", fd.getCorpusSigle());
// [tt/p="A.*"]{0,3}[tt/p="N.*"]
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bugs/multiterm_rewrite.jsonld").getFile());
Krill ks = new Krill(json);
@@ -858,7 +858,7 @@
ki.commit();
// ({1:Sonne []* Erde} | {2: Erde []* Sonne})
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bugs/tokendistancespan_bug.jsonld").getFile());
Krill ks = new Krill(json);
@@ -881,7 +881,7 @@
true);
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/metaquery8-nocollection.jsonld").getFile());
Krill ks = new Krill(json);
@@ -890,7 +890,7 @@
assertEquals(0, kr.getStartIndex());
assertEquals(10, kr.getItemsPerPage());
- json = getString(getClass().getResource("/queries/metaquery8.jsonld")
+ json = getJsonString(getClass().getResource("/queries/metaquery8.jsonld")
.getFile());
ks = new Krill(json);
@@ -901,7 +901,7 @@
assertEquals(0, kr.getStartIndex());
assertEquals(10, kr.getItemsPerPage());
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/metaquery8-filtered.jsonld").getFile());
ks = new Krill(json);
@@ -912,7 +912,7 @@
assertEquals(0, kr.getStartIndex());
assertEquals(10, kr.getItemsPerPage());
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/metaquery8-filtered-further.jsonld").getFile());
ks = new Krill(json);
@@ -923,7 +923,7 @@
assertEquals(10, kr.getItemsPerPage());
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/metaquery8-filtered-nested.jsonld").getFile());
ks = new Krill(json);
@@ -958,7 +958,7 @@
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bsp-context-2.jsonld").getFile());
Krill ks = new Krill(json);
@@ -978,7 +978,7 @@
assertFalse(kr.getContext().toJsonNode().toString()
.equals("\"base/s:s\""));
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/bsp-context-sentence.jsonld").getFile());
kr = new Krill(json).apply(ki);
@@ -1010,7 +1010,7 @@
};
ki.commit();
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bsp-bug.jsonld").getFile());
Result kr = new Krill(json).apply(ki);
@@ -1019,28 +1019,29 @@
"Operation needs operand list");
};
-
- @Test
+
+ @Test
public void searchJSONdistanceWithRegexesBug () throws IOException {
// Construct index
KrillIndex ki = new KrillIndex();
// Indexing test files
for (String i : new String[] { "00001" }) {
- // , "00002", "00003", "00004", "00005", "00006", "02439"
+ // , "00002", "00003", "00004", "00005", "00006", "02439"
ki.addDoc(
getClass().getResourceAsStream("/wiki/" + i + ".json.gz"),
true);
};
ki.commit();
- // "der" []{2,3} [opennlp/p="NN"]
- String json = getString(getClass().getResource(
+ // "der" []{2,3} [opennlp/p="NN"]
+ String json = getJsonString(getClass().getResource(
"/queries/bugs/distances_with_regex_bug.jsonld").getFile());
Result kr = new Krill(json).apply(ki);
- assertEquals(kr.getMatch(0).getSnippetBrackets(),
- "Mit Ausnahme von Fremdwörtern und Namen ist das A der einzige Buchstabe im Deutschen, [[der zweifach am Anfang]] eines Wortes stehen darf, etwa im Wort Aal.");
+ assertEquals(
+ kr.getMatch(0).getSnippetBrackets(),
+ "Mit Ausnahme von Fremdwörtern und Namen ist das A der einzige Buchstabe im Deutschen, [[der zweifach am Anfang]] eines Wortes stehen darf, etwa im Wort Aal.");
};
@@ -1058,7 +1059,7 @@
// Expansion bug
// der alte Digraph Aa durch Å
- String json = getString(getClass().getResource(
+ String json = getJsonString(getClass().getResource(
"/queries/bugs/expansion_bug_2.jsonld").getFile());
Result kr = new Krill(json).apply(ki);
@@ -1078,7 +1079,7 @@
// der alte Digraph Aa durch []
// Works with one document
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/bugs/expansion_bug.jsonld").getFile());
kr = new Krill(json).apply(ki);
@@ -1103,7 +1104,7 @@
// Expansion bug
// der alte Digraph Aa durch Å
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/bugs/expansion_bug_2.jsonld").getFile());
kr = new Krill(json).apply(ki);
@@ -1116,7 +1117,7 @@
assertEquals(kr.getTotalResults(), 1);
// der alte Digraph Aa durch []
- json = getString(getClass().getResource(
+ json = getJsonString(getClass().getResource(
"/queries/bugs/expansion_bug.jsonld").getFile());
kr = new Krill(json).apply(ki);
@@ -1128,22 +1129,24 @@
assertEquals(kr.getTotalResults(), 1);
};
-
- @Test
+
+ @Test
public void queryJSONzeroRepetitionBug () throws IOException {
- // der{0}
- KrillIndex ki = new KrillIndex();
- ki.addDoc(getClass().getResourceAsStream("/wiki/00001.json.gz"), true);
- ki.commit();
-
- String json = getString(getClass().getResource(
- "/queries/bugs/zero_repetition_bug.jsonld").getFile());
+ // der{0}
+ KrillIndex ki = new KrillIndex();
+ ki.addDoc(getClass().getResourceAsStream("/wiki/00001.json.gz"), true);
+ ki.commit();
- Result kr = new Krill(json).apply(ki);
+ String json = getJsonString(getClass().getResource(
+ "/queries/bugs/zero_repetition_bug.jsonld").getFile());
- assertEquals(783, kr.getError(0).getCode());
- assertEquals("This query can't match anywhere", kr.getError(0).getMessage());
- };
+ Result kr = new Krill(json).apply(ki);
+
+ assertEquals(783, kr.getError(0).getCode());
+ assertEquals("This query can't match anywhere", kr.getError(0)
+ .getMessage());
+ };
+
/**
* This is a Schreibgebrauch ressource that didn't work for
diff --git a/src/test/java/de/ids_mannheim/korap/search/TestMetaFields.java b/src/test/java/de/ids_mannheim/korap/search/TestMetaFields.java
index 5e15394..b297672 100644
--- a/src/test/java/de/ids_mannheim/korap/search/TestMetaFields.java
+++ b/src/test/java/de/ids_mannheim/korap/search/TestMetaFields.java
@@ -47,7 +47,7 @@
};
ki.commit();
- String jsonString = getString(getClass().getResource(
+ String jsonString = getJsonString(getClass().getResource(
"/queries/metas/fields.jsonld").getFile());
Krill ks = new Krill(jsonString);
@@ -87,7 +87,7 @@
assertTrue(res.at("/matches/0/layerInfos").isMissingNode());
assertTrue(res.at("/matches/0/tokenization").isMissingNode());
- jsonString = getString(getClass().getResource(
+ jsonString = getJsonString(getClass().getResource(
"/queries/metas/fields_2.jsonld").getFile());
ks = new Krill(jsonString);
kr = ks.apply(ki);
@@ -125,7 +125,7 @@
ki.addDoc(getClass().getResourceAsStream("/goe/AGX-00002.json"), false);
ki.commit();
- String jsonString = getString(getClass().getResource(
+ String jsonString = getJsonString(getClass().getResource(
"/queries/metas/fields_no.jsonld").getFile());
Krill ks = new Krill(jsonString);
@@ -153,7 +153,7 @@
// All fields
- jsonString = getString(getClass().getResource(
+ jsonString = getJsonString(getClass().getResource(
"/queries/metas/fields_all.jsonld").getFile());
ks = new Krill(jsonString);
@@ -184,7 +184,7 @@
// @All fields
- jsonString = getString(getClass().getResource(
+ jsonString = getJsonString(getClass().getResource(
"/queries/metas/fields_at_all.jsonld").getFile());
ks = new Krill(jsonString);
@@ -245,7 +245,7 @@
ki.commit();
// textClass = reisen & wissenschaft
- String jsonString = getString(getClass().getResource(
+ String jsonString = getJsonString(getClass().getResource(
"/queries/collections/collection_textClass.jsonld").getFile());
Krill ks = new Krill(jsonString);
KrillCollection kc = ks.getCollection();
@@ -253,7 +253,7 @@
assertEquals(1, kc.numberOf("documents"));
// textClass = reisen
- jsonString = getString(getClass().getResource(
+ jsonString = getJsonString(getClass().getResource(
"/queries/collections/collection_textClass_2.jsonld").getFile());
ks = new Krill(jsonString);
kc = ks.getCollection();
@@ -276,7 +276,7 @@
*/
// author = wolfgang
- jsonString = getString(getClass().getResource(
+ jsonString = getJsonString(getClass().getResource(
"/queries/collections/collection_goethe.jsonld").getFile());
ks = new Krill(jsonString);
kc = ks.getCollection();
@@ -284,7 +284,7 @@
assertEquals(1, kc.numberOf("documents"));
// author = Wolfgang
- jsonString = getString(getClass().getResource(
+ jsonString = getJsonString(getClass().getResource(
"/queries/collections/collection_goethe_2.jsonld").getFile());
ks = new Krill(jsonString);
kc = ks.getCollection();
@@ -303,7 +303,7 @@
public void searchMetaContext () throws IOException {
// All fields
- String jsonString = getString(getClass().getResource(
+ String jsonString = getJsonString(getClass().getResource(
"/queries/metas/context_paragraph.jsonld").getFile());
Krill ks = new Krill(jsonString);
diff --git a/src/test/java/de/ids_mannheim/korap/search/TestResult.java b/src/test/java/de/ids_mannheim/korap/search/TestResult.java
index a106fae..12e5b3d 100644
--- a/src/test/java/de/ids_mannheim/korap/search/TestResult.java
+++ b/src/test/java/de/ids_mannheim/korap/search/TestResult.java
@@ -2,6 +2,7 @@
import java.util.*;
import java.io.*;
+import java.net.URLDecoder;
import org.apache.lucene.search.spans.SpanQuery;
@@ -263,7 +264,7 @@
public static String getString (String path) {
StringBuilder contentBuilder = new StringBuilder();
try {
- BufferedReader in = new BufferedReader(new FileReader(path));
+ BufferedReader in = new BufferedReader(new FileReader(URLDecoder.decode(path,"UTF-8")));
String str;
while ((str = in.readLine()) != null) {
contentBuilder.append(str);
diff --git a/src/test/java/de/ids_mannheim/korap/server/TestResource.java b/src/test/java/de/ids_mannheim/korap/server/TestResource.java
index 633a277..908f3e9 100644
--- a/src/test/java/de/ids_mannheim/korap/server/TestResource.java
+++ b/src/test/java/de/ids_mannheim/korap/server/TestResource.java
@@ -17,6 +17,7 @@
import org.junit.Test;
import java.io.FileInputStream;
+import java.net.URLDecoder;
import de.ids_mannheim.korap.server.Node;
import de.ids_mannheim.korap.response.Result;
@@ -267,7 +268,7 @@
public static String getString (String path) {
StringBuilder contentBuilder = new StringBuilder();
try {
- BufferedReader in = new BufferedReader(new FileReader(path));
+ BufferedReader in = new BufferedReader(new FileReader(URLDecoder.decode(path,"UTF-8")));
String str;
while ((str = in.readLine()) != null) {
contentBuilder.append(str);
diff --git a/src/test/resources/queries/sequence/distance-with-optionality.jsonld b/src/test/resources/queries/sequence/distance-with-optionality.jsonld
new file mode 100644
index 0000000..c1ec846
--- /dev/null
+++ b/src/test/resources/queries/sequence/distance-with-optionality.jsonld
@@ -0,0 +1,52 @@
+{
+ "@context": "http://korap.ids-mannheim.de/ns/KoralQuery/v0.3/context.jsonld",
+ "query": {
+ "@type": "koral:group",
+ "distances": [
+ {
+ "@type": "koral:distance",
+ "boundary": {
+ "@type": "koral:boundary",
+ "max": 1,
+ "min": 1
+ },
+ "key": "w",
+ "max": 1,
+ "min": 1
+ }
+ ],
+ "inOrder": true,
+ "operands": [
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "key": "Sonne",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ },
+ {
+ "@type": "koral:group",
+ "boundary": {
+ "@type": "koral:boundary",
+ "max": 1,
+ "min": 0
+ },
+ "operands": [
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "key": "Mond",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ }
+ ],
+ "operation": "operation:repetition"
+ }
+ ],
+ "operation": "operation:sequence"
+ }
+}
diff --git a/src/test/resources/queries/sequence/empty-followed-by-optionality.jsonld b/src/test/resources/queries/sequence/empty-followed-by-optionality.jsonld
new file mode 100644
index 0000000..0d60cd1
--- /dev/null
+++ b/src/test/resources/queries/sequence/empty-followed-by-optionality.jsonld
@@ -0,0 +1,42 @@
+{
+ "@context": "http://korap.ids-mannheim.de/ns/KoralQuery/v0.3/context.jsonld",
+ "query": {
+ "@type": "koral:group",
+ "inOrder": true,
+ "operation": "operation:sequence",
+ "operands": [
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "key": "Sonne",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ },
+ {
+ "@type": "koral:token"
+ },
+ {
+ "@type": "koral:group",
+ "boundary": {
+ "@type": "koral:boundary",
+ "max": 1,
+ "min": 0
+ },
+ "operands": [
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "key": "Mond",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ }
+ ],
+ "operation": "operation:repetition"
+ }
+ ]
+ }
+}
diff --git a/src/test/resources/queries/sequence/regex-rewrite-1.jsonld b/src/test/resources/queries/sequence/regex-rewrite-1.jsonld
new file mode 100644
index 0000000..e3cf3bf
--- /dev/null
+++ b/src/test/resources/queries/sequence/regex-rewrite-1.jsonld
@@ -0,0 +1,29 @@
+{
+ "@context":"http://ids-mannheim.de/ns/KorAP/json-ld/v0.3/context.jsonld",
+ "collection":null,
+ "query":{
+ "@type": "koral:group",
+ "operation": "operation:sequence",
+ "operands": [
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "key": "der",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ },
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "type" : "type:regex",
+ "key": ".+?",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ }
+ ]
+ }
+}
diff --git a/src/test/resources/queries/sequence/regex-rewrite-2.jsonld b/src/test/resources/queries/sequence/regex-rewrite-2.jsonld
new file mode 100644
index 0000000..a073d3f
--- /dev/null
+++ b/src/test/resources/queries/sequence/regex-rewrite-2.jsonld
@@ -0,0 +1,59 @@
+{
+ "@context":"http://ids-mannheim.de/ns/KorAP/json-ld/v0.3/context.jsonld",
+ "collection":null,
+ "query":{
+ "@type": "koral:group",
+ "operation": "operation:sequence",
+ "operands": [
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "key": "der",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ },
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "type" : "type:regex",
+ "key": ".*",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ },
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "type" : "type:regex",
+ "key": ".*?",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ },
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "type" : "type:regex",
+ "key": ".+",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ },
+ {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "type" : "type:regex",
+ "key": ".+?",
+ "layer": "orth",
+ "match": "match:eq"
+ }
+ }
+ ]
+ }
+}