src/main/java/de/ids_mannheim/korap/openthesaurus/ThesaurusDictionary.java

package de.ids_mannheim.korap.openthesaurus;

import java.io.*;
import java.util.*;
import java.util.regex.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class ThesaurusDictionary {

    // The pattern is instantiated here for the sake of efficiency (gets reused lots of times)
    static String regex = "(\\(.+\\) *)*([^\\(\\)]+[^\\(\\ )])( *\\(.+\\))*";
    static Pattern withoutParenthesesPattern = Pattern.compile(regex);

    /**
     * Extracts words without parentheses from a string. Text file provided by
     * OpenThesaurus contains not only synonyms on a line, but also some
     * explanations/annotations, e.g. "(sich etwas) versagen (geh.)".
     * Those are useless for querying and are, therefore, removed together
     * with surrounding spaces.
     * @param  str string to extract a substring from
     * @return substring without words in parentheses
     */
    static String cleanString(String str) {
        Matcher withoutParentheses = withoutParenthesesPattern.matcher(str);
        if (withoutParentheses.matches()) {
            return withoutParentheses.group(2);
        }
        return str;
    }

    /**
     * Reads in the text file with OpenThesaurus data and filters it as follows:
     * 1) Ignores all lines, which start with a "#", as those are comments.
     * 2) Reads in the lines before a line with "==" signs appears. This is the beginning
     * of the licensing section, which should be removed.
     * @param  thesaurusInputStream InputStream created from a path to OpenThesaurus data
     * @return Stream<String> with relevant lines from the OpenThesaurus data file.
     * @throws IOException
     */
    public static Stream<String> getAndFilterLines(InputStream thesaurusInputStream) throws IOException {
        BufferedReader bf = new BufferedReader(new InputStreamReader(thesaurusInputStream));
        return bf.lines()
                 .takeWhile(line -> !line.startsWith("=="))
                 .filter(line -> !line.startsWith("#"));
    }

    /**
     * Processes a single line of text from the OpenThesaurus data file,
     * creating a dictionary, where every word on the line appears as a
     * key, having all other words as a list of values.
     * @param  line a single line in the OpenThesaurus file
     * @return HashMap<String, List<String>> with all combinations of
     *         words on the line as keys, and their neighbors as values
     *         (of course, ignoring the ordering in values)
     */
    public static Map<String, List<String>> processLine(String line) {
        String[] lineSplit = line.split(";");
        Map<String, List<String>> dictFromLine = new HashMap<>();
        if (lineSplit.length > 1) {
            // this creates a dictionary entry for every word on the line
            // with all the other words as a List of synonyms
            for (int i = 0; i < lineSplit.length; i++) {
                // don't include explanations and word usage as "(ugs.)"
                String currentKey = cleanString(lineSplit[i]);
                List<String> synonyms = new ArrayList<>();
                for (int j = 0; j < lineSplit.length; j++) {
                    // include all words except the current key
                    if (j != i) synonyms.add(cleanString(lineSplit[j]));
                }
                dictFromLine.put(currentKey, synonyms);
            }
        }
        return dictFromLine;
    }

    /**
     * Combines all the above into a pipeline and returns a dictionary, which
     * can be queried for synonyms to get a list of synsets.
     * @param thesaurusFileStream InputStream created from OpenThesaurus data file
     * @return HashMap<String, List<List<String>>> with a word as a key, and
     *         a List of Lists as values (each of them represents a synset)
     */
    public static Map<String, List<List<String>>> createThesaurusDictionary(InputStream thesaurusFileStream) {
        Map<String, List<List<String>>> thesaurusDict = null;

        try {
            Stream<String> fileLines = getAndFilterLines(thesaurusFileStream);
            thesaurusDict = fileLines
                    .parallel()
                    .flatMap(line -> processLine(line).entrySet().stream())
                    .collect(Collectors.groupingBy(Map.Entry::getKey,
                             Collectors.mapping(Map.Entry::getValue, Collectors.toList())));
        }
        catch (IOException e) {
            e.printStackTrace();
        }

        return thesaurusDict;
    }

    /**
     * Overloads the InputStream version of the method in case a filepath String to
     * OpenThesaurus data is provided (the alternative is when the default cached data
     * from resources/static are used, obtained via "getResourceAsStream")
     * @param  thesaurusPath filepath to the OpenThesaurus data file
     * @return HashMap<String, List<List<String>>> with a word as a key, and
     *         a List of Lists as values (each of them represents a synset)
     */
    public static Map<String, List<List<String>>> createThesaurusDictionary(String thesaurusPath) {
        InputStream FIS = null;
        try {
            FIS = new FileInputStream(thesaurusPath);
        }
        catch (IOException e) {
            e.printStackTrace();
        }
        return createThesaurusDictionary(FIS);
    }
    /*
    public static void main(String[] args) {
        InputStream thesaurusInputStream = ThesaurusDictionary.class.getResourceAsStream("/static/openthesaurus.txt");
        Map<String, List<List<String>>> thesaurusDict = createThesaurusDictionary(thesaurusInputStream);

        System.out.println(thesaurusDict.get("laufen"));
    }

     */
}