Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 1 | #ifdef __cplusplus |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 2 | #include <typeinfo> |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 3 | #include "rocksdb/db.h" |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 4 | #endif |
| 5 | #include <stdint.h> |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 6 | |
| 7 | #define IS_BIG_ENDIAN (*(uint16_t *)"\0\xff" < 0x100) |
| 8 | #define encodeCollocation(w1, w2, dist) (((uint64_t)dist << 56) | ((uint64_t)w2 << 24) | w1) |
| 9 | #define W1(key) (uint64_t)(key & 0xffffff) |
| 10 | #define W2(key) (uint64_t)((key >> 24) & 0xffffff) |
| 11 | #define DIST(key) (int8_t)((uint64_t)((key >> 56) & 0xff)) |
| 12 | |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 13 | #ifdef __cplusplus |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 14 | namespace rocksdb { |
Marc Kupietz | 3400aa5 | 2018-06-05 10:28:55 +0200 | [diff] [blame] | 15 | class Collocator { |
| 16 | public: |
| 17 | uint64_t w2; |
| 18 | uint64_t raw; |
| 19 | double pmi; |
| 20 | double npmi; |
| 21 | double llr; |
| 22 | double lfmd; |
| 23 | double fpmi; |
| 24 | double left_lfmd; |
| 25 | double right_lfmd; |
| 26 | double left_npmi; |
| 27 | double right_npmi; |
| 28 | }; |
| 29 | |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 30 | class CollocatorIterator : public Iterator { |
| 31 | public: |
| 32 | CollocatorIterator(const Iterator& it); |
| 33 | void SeekToFirst(); |
| 34 | void SeekToLast(); |
| 35 | void Seek(const rocksdb::Slice&); |
| 36 | void Prev(); |
| 37 | bool isValid(); |
| 38 | uint64_t intValue(); |
| 39 | uint64_t intKey(); |
| 40 | }; |
| 41 | |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 42 | extern "C" { |
Marc Kupietz | 6aec768 | 2018-01-10 09:47:48 +0100 | [diff] [blame] | 43 | class CollocatorDB { |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 44 | public: |
Marc Kupietz | 3400aa5 | 2018-06-05 10:28:55 +0200 | [diff] [blame] | 45 | std::string getWord(uint32_t w1); |
| 46 | std::vector<Collocator> get_collocators(uint32_t w1); |
Marc Kupietz | bd96619 | 2018-10-13 14:14:37 +0200 | [diff] [blame] | 47 | std::vector<Collocator> get_collocators(uint32_t w1, uint32_t max_w2); |
Marc Kupietz | 3400aa5 | 2018-06-05 10:28:55 +0200 | [diff] [blame] | 48 | void dumpSparseLlr(uint32_t w1, uint32_t min_cooccur); |
| 49 | CollocatorDB(const char *db_name, const bool read_only); |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 50 | void inc(const uint32_t w1, const uint32_t w2, const uint8_t dist); |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 51 | void dump(const uint32_t w1, const uint32_t w2, const uint8_t dist); |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 52 | CollocatorIterator* SeekIterator(uint64_t w1, uint64_t w2, int8_t dist); |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 53 | }; |
| 54 | |
| 55 | } |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 56 | } |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 57 | |
Marc Kupietz | 6aec768 | 2018-01-10 09:47:48 +0100 | [diff] [blame] | 58 | typedef rocksdb::CollocatorDB COLLOCATORDB; |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 59 | |
| 60 | #else |
Marc Kupietz | 6aec768 | 2018-01-10 09:47:48 +0100 | [diff] [blame] | 61 | typedef struct COLLOCATORDB COLLOCATORDB; |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 62 | #endif |
| 63 | |
Marc Kupietz | 6aec768 | 2018-01-10 09:47:48 +0100 | [diff] [blame] | 64 | extern COLLOCATORDB *open_collocatordb(char *s); |
| 65 | extern COLLOCATORDB *open_collocatordb_for_write(char *s); |
| 66 | extern void inc_collocator(COLLOCATORDB *db, uint64_t w1, uint64_t w2, int8_t dist); |
| 67 | extern void dump_collocators(COLLOCATORDB *db, uint32_t w1, uint32_t w2, int8_t dist); |
| 68 | extern void get_collocators(COLLOCATORDB *db, uint32_t w1); |
| 69 | extern char *get_collocators_as_json(COLLOCATORDB *db, uint32_t w1); |
Marc Kupietz | ca3a52e | 2018-06-05 14:16:23 +0200 | [diff] [blame] | 70 | extern char *get_word(COLLOCATORDB *db, uint32_t w1); |