Added more drukola tests
Change-Id: I1c13d117064bd673828bd64be2ce474333625043
diff --git a/t/corpus/BBU/BLOG/83709_a_82384/base/tokens_aggr.xml b/t/corpus/BBU/BLOG/83709_a_82384/base/tokens_aggr.xml
new file mode 100644
index 0000000..c1eb2ff
--- /dev/null
+++ b/t/corpus/BBU/BLOG/83709_a_82384/base/tokens_aggr.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="span.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<layer docid="BBU_BLOG.83709_a_82384" xmlns="http://ids-mannheim.de/ns/KorAP" version="KorAP-0.4">
+ <spanList>
+ <span id="t_0" from="0" to="7" />
+ <span id="t_1" from="8" to="13" />
+ <span id="t_2" from="14" to="20" />
+ <span id="t_3" from="21" to="30" />
+ <span id="t_4" from="31" to="33" />
+ <span id="t_5" from="34" to="40" />
+ <span id="t_6" from="41" to="49" />
+ <span id="t_7" from="50" to="52" />
+ <span id="t_8" from="53" to="54" />
+ <span id="t_9" from="54" to="61" />
+ <span id="t_10" from="62" to="67" />
+ <span id="t_11" from="67" to="68" />
+ <span id="t_12" from="69" to="71" />
+ <span id="t_13" from="72" to="78" />
+ <span id="t_14" from="79" to="83" />
+ <span id="t_15" from="83" to="84" />
+ </spanList>
+</layer>
\ No newline at end of file
diff --git a/t/corpus/BBU/BLOG/83709_a_82384/base/tokens_conservative.xml b/t/corpus/BBU/BLOG/83709_a_82384/base/tokens_conservative.xml
new file mode 100644
index 0000000..3e84414
--- /dev/null
+++ b/t/corpus/BBU/BLOG/83709_a_82384/base/tokens_conservative.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="span.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<layer docid="BBU_BLOG.83709_a_82384" xmlns="http://ids-mannheim.de/ns/KorAP" version="KorAP-0.4">
+ <spanList>
+ <span id="t_0" from="0" to="7" />
+ <span id="t_1" from="8" to="13" />
+ <span id="t_2" from="14" to="20" />
+ <span id="t_3" from="21" to="30" />
+ <span id="t_4" from="31" to="33" />
+ <span id="t_5" from="34" to="40" />
+ <span id="t_6" from="41" to="49" />
+ <span id="t_7" from="50" to="52" />
+ <span id="t_8" from="54" to="61" />
+ <span id="t_9" from="62" to="67" />
+ <span id="t_10" from="69" to="71" />
+ <span id="t_11" from="72" to="78" />
+ <span id="t_12" from="79" to="83" />
+ </spanList>
+</layer>
\ No newline at end of file
diff --git a/t/corpus/BBU/BLOG/83709_a_82384/data.xml b/t/corpus/BBU/BLOG/83709_a_82384/data.xml
new file mode 100644
index 0000000..7cbc1df
--- /dev/null
+++ b/t/corpus/BBU/BLOG/83709_a_82384/data.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="text.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<raw_text docid="BBU_BLOG.83709_a_82384" xmlns="http://ids-mannheim.de/ns/KorAP">
+ <metadata file="metadata.xml" />
+ <text>Schimbă vorba Dragoș Bucurenci Cu Lucian Mândruță la “Schimbă vorba” pe Gândul Live.</text>
+</raw_text>
\ No newline at end of file
diff --git a/t/corpus/BBU/BLOG/83709_a_82384/drukola/morpho.xml b/t/corpus/BBU/BLOG/83709_a_82384/drukola/morpho.xml
new file mode 100644
index 0000000..4753ff5
--- /dev/null
+++ b/t/corpus/BBU/BLOG/83709_a_82384/drukola/morpho.xml
@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="span.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<layer docid="BBU_BLOG.83709_a_82384" xmlns="http://ids-mannheim.de/ns/KorAP" version="KorAP-0.4">
+ <spanList>
+ <span id="s0" from="0" to="7" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">schimba</f>
+ <f name="pos">VERB</f>
+ <f name="msd">msd=Vmis3s|number=singular|type=predicative|person=third|tense=past|mood=indicative</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s1" from="8" to="13" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">vorbă</f>
+ <f name="pos">NOUN</f>
+ <f name="msd">msd=Ncfsry|case=Ncfsry|definiteness=yes|gender=feminine|number=singular|type=common</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s2" from="14" to="20" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">dragoș</f>
+ <f name="pos">NOUN</f>
+ <f name="msd">msd=Np|type=proper</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s3" from="21" to="30" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">bucurenci</f>
+ <f name="pos">NOUN</f>
+ <f name="msd">msd=Np|type=proper</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s4" from="31" to="33" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">cu</f>
+ <f name="pos">ADPOSITION</f>
+ <f name="msd">msd=Sp</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s5" from="34" to="40" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">lucian</f>
+ <f name="pos">NOUN</f>
+ <f name="msd">msd=Np|type=proper</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s6" from="41" to="49" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">mândruț</f>
+ <f name="pos">ADJECTIVE</f>
+ <f name="msd">msd=Afpfsrn|case=Afpfsrn|definiteness=no|gender=feminine|number=singular</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s7" from="50" to="52" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">la</f>
+ <f name="pos">ADPOSITION</f>
+ <f name="msd">msd=Sp</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s8" from="53" to="54" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">“</f>
+ <f name="pos">NOUN</f>
+ <f name="msd">msd=Np|type=proper</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s9" from="55" to="61" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">schimba</f>
+ <f name="pos">VERB</f>
+ <f name="msd">msd=Vmis3s|number=singular|type=predicative|person=third|tense=past|mood=indicative</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s10" from="62" to="67" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">vorbă</f>
+ <f name="pos">NOUN</f>
+ <f name="msd">msd=Ncfsry|case=Ncfsry|definiteness=yes|gender=feminine|number=singular|type=common</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s11" from="68" to="68" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">”</f>
+ <f name="pos">PUNCTUATION</f>
+ <f name="msd"></f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s12" from="69" to="71" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">pe</f>
+ <f name="pos">ADPOSITION</f>
+ <f name="msd">msd=Sp</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s13" from="72" to="78" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">gând</f>
+ <f name="pos">NOUN</f>
+ <f name="msd">msd=Ncmsry|case=Ncmsry|definiteness=yes|gender=masculine|number=singular|type=common</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s14" from="79" to="83" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">Live</f>
+ <f name="pos">NOUN</f>
+ <f name="msd">msd=Np|type=proper</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s15" from="84" to="84" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">.</f>
+ <f name="pos">PUNCTUATION</f>
+ <f name="msd"></f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ </spanList>
+</layer>
\ No newline at end of file
diff --git a/t/corpus/BBU/BLOG/83709_a_82384/header.xml b/t/corpus/BBU/BLOG/83709_a_82384/header.xml
new file mode 100644
index 0000000..b492d5d
--- /dev/null
+++ b/t/corpus/BBU/BLOG/83709_a_82384/header.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="text" pattern="text" status="new" version="1.1" TEIform="teiHeader">
+ <fileDesc>
+ <titleStmt>
+ <textSigle>BBU/BLOG.83709_a_82384</textSigle>
+ <t.title>Schimbă vorba</t.title>
+ </titleStmt>
+ <publicationStmt>
+ <distributor/>
+ <pubAddress/>
+ <availability region="world">QAO-NC</availability>
+ <pubDate/>
+ </publicationStmt>
+ <sourceDesc>
+ <biblStruct>
+ <monogr>
+ <h.author>Dragoș Bucurenci</h.author>
+ <editor role="translator">-</editor>
+ <imprint>
+ <pubDate type="year">2013</pubDate>
+ <pubDate type="month">10</pubDate>
+ <pubDate type="day">5</pubDate>
+ <pubPlace>URL:http://www.bucurenci.ro</pubPlace>
+ </imprint>
+ </monogr>
+ </biblStruct>
+ </sourceDesc>
+ </fileDesc>
+ </idsHeader>
\ No newline at end of file
diff --git a/t/corpus/BBU/BLOG/83709_a_82384/struct/structure.xml b/t/corpus/BBU/BLOG/83709_a_82384/struct/structure.xml
new file mode 100644
index 0000000..e7c201b
--- /dev/null
+++ b/t/corpus/BBU/BLOG/83709_a_82384/struct/structure.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="span.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<layer docid="BBU_BLOG.83709_a_82384" xmlns="http://ids-mannheim.de/ns/KorAP" version="KorAP-0.4">
+ <spanList>
+ <span id="s0" from="0" to="84" l="1">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">text</f>
+ </fs>
+ </span>
+ <span id="s1" from="0" to="84" l="2">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">body</f>
+ </fs>
+ </span>
+ <span id="s2" from="0" to="13" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">p</f>
+ </fs>
+ </span>
+ <span id="s3" from="0" to="13" l="4">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">s</f>
+ </fs>
+ </span>
+ <span id="s6" from="14" to="30" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">p</f>
+ </fs>
+ </span>
+ <span id="s7" from="14" to="30" l="4">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">s</f>
+ </fs>
+ </span>
+ <span id="s10" from="31" to="84" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">p</f>
+ </fs>
+ </span>
+ <span id="s11" from="31" to="84" l="4">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">s</f>
+ </fs>
+ </span>
+ </spanList>
+</layer>
\ No newline at end of file
diff --git a/t/corpus/BBU/BLOG/header.xml b/t/corpus/BBU/BLOG/header.xml
new file mode 100644
index 0000000..ff755da
--- /dev/null
+++ b/t/corpus/BBU/BLOG/header.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="document" pattern="text" status="new" version="1.1" TEIform="teiHeader">
+ <fileDesc>
+ <titleStmt>
+ <dokumentSigle>BBU/BLOG</dokumentSigle>
+ <d.title>[...]</d.title>
+ </titleStmt>
+ <publicationStmt>
+ <distributor/>
+ <pubAddress/>
+ <availability region="world">[...]</availability>
+ <pubDate/>
+ </publicationStmt>
+ <sourceDesc>
+ <biblStruct>
+ <monogr>
+ <h.title type="main"/>
+ <edition>
+ <further/>
+ <kind/>
+ <appearance/>
+ </edition>
+ <imprint/>
+ </monogr>
+ </biblStruct>
+ </sourceDesc>
+ </fileDesc>
+ <profileDesc>
+ <textDesc>
+ <textType>[DOCUMENTTEXTSTYLE]</textType>
+ <textTypeRef>[SOURCENAME]</textTypeRef>
+ </textDesc>
+ <textClass>
+ <catRef target="[DOCUMENTTEXTDOMAIN].[DOCUMENTTEXTSUBDOMAIN]" scheme="topic"/>
+ </textClass>
+ </profileDesc>
+ </idsHeader>
\ No newline at end of file
diff --git a/t/corpus/BBU/header.xml b/t/corpus/BBU/header.xml
new file mode 100644
index 0000000..b9ff5ba
--- /dev/null
+++ b/t/corpus/BBU/header.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="corpus" status="new" version="1.1" TEIform="teiHeader">
+ <fileDesc>
+ <titleStmt>
+ <korpusSigle>BBU</korpusSigle>
+ <c.title>[SOURCENAME]</c.title>
+ </titleStmt>
+ <sourceDesc>
+ <biblStruct>
+ <monogr>
+ <h.author>[AUTHORNAME]</h.author>
+ <editor role="translator">[TRANSLATOR]</editor>
+ </monogr>
+ </biblStruct>
+ </sourceDesc>
+ </fileDesc>
+ <profileDesc>
+ <langUsage>
+ <language id="ro" usage="100">Romanian</language>
+ </langUsage>
+ <!--
+ <textDesc>
+ This element is suppressed in p5!
+ <channel mode="w">[MEDIUM]</channel>
+ </textDesc>
+ -->
+ </profileDesc>
+ </idsHeader>
\ No newline at end of file
diff --git a/t/real/drukola.t b/t/real/drukola.t
new file mode 100644
index 0000000..244101a
--- /dev/null
+++ b/t/real/drukola.t
@@ -0,0 +1,109 @@
+use strict;
+use warnings;
+use Test::More;
+use Data::Dumper;
+use JSON::XS;
+
+use Benchmark qw/:hireswallclock/;
+
+my $t = Benchmark->new;
+
+use utf8;
+use lib 'lib', '../lib';
+
+use File::Basename 'dirname';
+use File::Spec::Functions 'catdir';
+
+use_ok('KorAP::XML::Krill');
+
+# This will Check DRuKoLa-Files
+
+# New
+# BBU/BLOG/83709_a_82384
+my $path = catdir(dirname(__FILE__), '../corpus/BBU/BLOG/83709_a_82384');
+
+ok(my $doc = KorAP::XML::Krill->new( path => $path . '/' ), 'Load Korap::Document');
+ok($doc->parse, 'Parse document');
+
+is($doc->text_sigle, 'BBU/BLOG/83709_a_82384', 'Correct text sigle');
+is($doc->doc_sigle, 'BBU/BLOG', 'Correct document sigle');
+is($doc->corpus_sigle, 'BBU', 'Correct corpus sigle');
+
+my $meta = $doc->meta;
+is($meta->{title}, 'Schimbă vorba', 'Title');
+is($meta->{pub_place}, 'URL:http://www.bucurenci.ro', 'PubPlace');
+is($meta->{pub_date}, '20131005', 'Creation Date');
+ok(!$meta->{sub_title}, 'SubTitle');
+is($meta->{author}, 'Dragoș Bucurenci', 'Author');
+
+ok(!$meta->{publisher}, 'Publisher');
+ok(!$meta->{editor}, 'Editor');
+#is($meta->{text_type}, 'Autobiographie', 'Correct Text Type');
+ok(!$meta->{text_type_art}, 'Correct Text Type Art');
+# is($meta->{text_type_ref}, '', 'Correct Text Type Ref');
+ok(!$meta->{text_column}, 'Correct Text Column');
+ok(!$meta->{text_domain}, 'Correct Text Domain');
+ok(!$meta->{creation_date}, 'Creation Date');
+
+ok(!$meta->{pages}, 'Pages');
+ok(!$meta->{file_edition_statement}, 'File Ed Statement');
+ok(!$meta->{bibl_edition_statement}, 'Bibl Ed Statement');
+ok(!$meta->{reference}, 'Reference');
+is($meta->{language}, 'ro', 'Language');
+
+#is($meta->{corpus_title}, 'Goethes Werke', 'Correct Corpus title');
+ok(!$meta->{corpus_sub_title}, 'Correct Corpus Sub title');
+#is($meta->{corpus_author}, 'Goethe, Johann Wolfgang von', 'Correct Corpus author');
+#is($meta->{corpus_editor}, 'Trunz, Erich', 'Correct Corpus editor');
+
+#is($meta->{doc_title}, 'Goethe: Autobiographische Schriften II, (1817-1825, 1832)',
+# 'Correct Doc title');
+ok(!$meta->{doc_sub_title}, 'Correct Doc Sub title');
+ok(!$meta->{doc_author}, 'Correct Doc author');
+ok(!$meta->{doc_editor}, 'Correct Doc editor');
+
+# Tokenization
+use_ok('KorAP::XML::Tokenizer');
+
+my ($token_base_foundry, $token_base_layer) = (qw/Base Tokens_conservative/);
+
+# Get tokenization
+my $tokens = KorAP::XML::Tokenizer->new(
+ path => $doc->path,
+ doc => $doc,
+ foundry => $token_base_foundry,
+ layer => $token_base_layer,
+ name => 'tokens'
+);
+ok($tokens, 'Token Object is fine');
+ok($tokens->parse, 'Token parsing is fine');
+
+my $output = decode_json( $tokens->to_json );
+
+## Base
+$tokens->add('DeReKo', 'Structure', 'base_sentences_paragraphs');
+ok($tokens->add('DRuKoLa', 'Morpho'), 'Add Drukola');
+
+$output = $tokens->to_data;
+
+is($output->{data}->{foundries}, 'dereko dereko/structure dereko/structure/base_sentences_paragraphs drukola drukola/morpho', 'Foundries');
+
+is($output->{data}->{layerInfos}, 'dereko/s=spans drukola/l=tokens drukola/m=tokens drukola/p=tokens', 'layerInfos');
+
+my $token = join('||', @{$output->{data}->{stream}->[7]});
+
+like($token, qr!drukola/l:la!, 'data');
+like($token, qr!drukola/m:msd:Sp!, 'data');
+like($token, qr!drukola/p:ADPOSITION!, 'data');
+
+$token = join('||', @{$output->{data}->{stream}->[9]});
+
+like($token, qr!i:vorba!, 'data');
+like($token, qr!drukola/l:vorbă!, 'data');
+like($token, qr!drukola/m:case:Ncfsry!, 'data');
+like($token, qr!drukola/m:definiteness:yes!, 'data');
+like($token, qr!drukola/m:gender:feminine!, 'data');
+like($token, qr!drukola/p:NOUN!, 'data');
+
+done_testing;
+__END__