Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 1 | #ifdef __cplusplus |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 2 | #include <typeinfo> |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 3 | #include "rocksdb/db.h" |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 4 | #endif |
| 5 | #include <stdint.h> |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 6 | |
| 7 | #define IS_BIG_ENDIAN (*(uint16_t *)"\0\xff" < 0x100) |
| 8 | #define encodeCollocation(w1, w2, dist) (((uint64_t)dist << 56) | ((uint64_t)w2 << 24) | w1) |
| 9 | #define W1(key) (uint64_t)(key & 0xffffff) |
| 10 | #define W2(key) (uint64_t)((key >> 24) & 0xffffff) |
| 11 | #define DIST(key) (int8_t)((uint64_t)((key >> 56) & 0xff)) |
| 12 | |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 13 | #ifdef __cplusplus |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 14 | namespace rocksdb { |
Marc Kupietz | 3400aa5 | 2018-06-05 10:28:55 +0200 | [diff] [blame] | 15 | class Collocator { |
| 16 | public: |
| 17 | uint64_t w2; |
| 18 | uint64_t raw; |
| 19 | double pmi; |
| 20 | double npmi; |
| 21 | double llr; |
| 22 | double lfmd; |
Marc Kupietz | 4188045 | 2019-01-22 15:29:06 +0100 | [diff] [blame^] | 23 | double md; |
Marc Kupietz | 3400aa5 | 2018-06-05 10:28:55 +0200 | [diff] [blame] | 24 | double left_lfmd; |
| 25 | double right_lfmd; |
| 26 | double left_npmi; |
| 27 | double right_npmi; |
Marc Kupietz | 4188045 | 2019-01-22 15:29:06 +0100 | [diff] [blame^] | 28 | double dice; |
| 29 | double logdice; |
Marc Kupietz | 3400aa5 | 2018-06-05 10:28:55 +0200 | [diff] [blame] | 30 | }; |
| 31 | |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 32 | class CollocatorIterator : public Iterator { |
| 33 | public: |
| 34 | CollocatorIterator(const Iterator& it); |
| 35 | void SeekToFirst(); |
| 36 | void SeekToLast(); |
| 37 | void Seek(const rocksdb::Slice&); |
| 38 | void Prev(); |
| 39 | bool isValid(); |
| 40 | uint64_t intValue(); |
| 41 | uint64_t intKey(); |
| 42 | }; |
| 43 | |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 44 | extern "C" { |
Marc Kupietz | 6aec768 | 2018-01-10 09:47:48 +0100 | [diff] [blame] | 45 | class CollocatorDB { |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 46 | public: |
Marc Kupietz | 3400aa5 | 2018-06-05 10:28:55 +0200 | [diff] [blame] | 47 | std::string getWord(uint32_t w1); |
| 48 | std::vector<Collocator> get_collocators(uint32_t w1); |
Marc Kupietz | bd96619 | 2018-10-13 14:14:37 +0200 | [diff] [blame] | 49 | std::vector<Collocator> get_collocators(uint32_t w1, uint32_t max_w2); |
Marc Kupietz | 3400aa5 | 2018-06-05 10:28:55 +0200 | [diff] [blame] | 50 | void dumpSparseLlr(uint32_t w1, uint32_t min_cooccur); |
| 51 | CollocatorDB(const char *db_name, const bool read_only); |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 52 | void inc(const uint32_t w1, const uint32_t w2, const uint8_t dist); |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 53 | void dump(const uint32_t w1, const uint32_t w2, const uint8_t dist); |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 54 | CollocatorIterator* SeekIterator(uint64_t w1, uint64_t w2, int8_t dist); |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 55 | }; |
| 56 | |
| 57 | } |
Marc Kupietz | 4b799e9 | 2018-01-02 11:04:56 +0100 | [diff] [blame] | 58 | } |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 59 | |
Marc Kupietz | 6aec768 | 2018-01-10 09:47:48 +0100 | [diff] [blame] | 60 | typedef rocksdb::CollocatorDB COLLOCATORDB; |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 61 | |
| 62 | #else |
Marc Kupietz | 6aec768 | 2018-01-10 09:47:48 +0100 | [diff] [blame] | 63 | typedef struct COLLOCATORDB COLLOCATORDB; |
Marc Kupietz | 06c9a9f | 2018-01-02 16:56:43 +0100 | [diff] [blame] | 64 | #endif |
| 65 | |
Marc Kupietz | 6aec768 | 2018-01-10 09:47:48 +0100 | [diff] [blame] | 66 | extern COLLOCATORDB *open_collocatordb(char *s); |
| 67 | extern COLLOCATORDB *open_collocatordb_for_write(char *s); |
| 68 | extern void inc_collocator(COLLOCATORDB *db, uint64_t w1, uint64_t w2, int8_t dist); |
| 69 | extern void dump_collocators(COLLOCATORDB *db, uint32_t w1, uint32_t w2, int8_t dist); |
| 70 | extern void get_collocators(COLLOCATORDB *db, uint32_t w1); |
| 71 | extern char *get_collocators_as_json(COLLOCATORDB *db, uint32_t w1); |
Marc Kupietz | ca3a52e | 2018-06-05 14:16:23 +0200 | [diff] [blame] | 72 | extern char *get_word(COLLOCATORDB *db, uint32_t w1); |