Make the installation easier, with a script that builds rocksdb
scripts/build-rocksdb.sh builds rocksdb once, with the shared and the
static library, into a prefix of its own. It builds the version of the
rocksdb package of the distribution where there is one, so that the
static library matches what collocatordb is compiled against, and a
pinned one where there is none (Rocky Linux, RHEL). Older rocksdb
versions are made to compile with newer compilers, e.g. Fedora's 10.2.1
with gcc 16.
-DSTATIC_ROCKSDB=ON links rocksdb and collocatordb statically, which
makes counting collocations about 22% faster, without naming the two
libraries by their paths.
The installation section of the README is rewritten as four steps that
cover Fedora, Rocky Linux 9 and 10, RHEL and MacOS.
Change-Id: Idbb588ad517c626fc387973b676126480edb6c2e
diff --git a/scripts/build-rocksdb.sh b/scripts/build-rocksdb.sh
new file mode 100755
index 0000000..36e4565
--- /dev/null
+++ b/scripts/build-rocksdb.sh
@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+#
+# Builds rocksdb once, with the shared and the static library, and installs it
+# into a prefix of its own (default: $HOME/rocksdb). Works on Fedora, Rocky
+# Linux, RHEL, Debian, Ubuntu and MacOS.
+#
+# The version to build is, in this order:
+# 1. the one given with -v
+# 2. the one of the rocksdb package of the distribution, so that the static
+# library matches the headers and the shared library collocatordb was
+# built against
+# 3. a pinned one that is known to compile with the compilers of Rocky Linux
+# 9 (gcc 11) up to Fedora (gcc 16) and MacOS (clang)
+#
+# Examples:
+# scripts/build-rocksdb.sh # build the version of the distribution
+# scripts/build-rocksdb.sh -v 10.2.1 # build a specific version
+# scripts/build-rocksdb.sh -p /opt/rocksdb # install somewhere else
+#
+set -euo pipefail
+
+PINNED_VERSION=10.2.1
+PREFIX=$HOME/rocksdb
+VERSION=
+SOURCE_DIR=
+STATIC_ONLY=0
+
+usage() {
+ sed -n '2,17p' "$0"
+ exit "${1:-0}"
+}
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ -p|--prefix) PREFIX=$2; shift 2 ;;
+ -v|--version) VERSION=$2; shift 2 ;;
+ -s|--source) SOURCE_DIR=$2; shift 2 ;;
+ --static-only) STATIC_ONLY=1; shift ;;
+ -h|--help) usage 0 ;;
+ *) echo "Unknown option: $1" >&2; usage 1 ;;
+ esac
+done
+
+SOURCE_DIR=${SOURCE_DIR:-$PREFIX-src}
+
+# --- the version -------------------------------------------------------------
+if [ -z "$VERSION" ]; then
+ if command -v rpm > /dev/null && rpm -q rocksdb > /dev/null 2>&1; then
+ VERSION=$(rpm -q --qf '%{VERSION}' rocksdb)
+ elif command -v dpkg-query > /dev/null && dpkg-query -W librocksdb-dev > /dev/null 2>&1; then
+ VERSION=$(dpkg-query -W -f '${Version}' librocksdb-dev | cut -d- -f1)
+ elif command -v brew > /dev/null && brew list --versions rocksdb > /dev/null 2>&1; then
+ VERSION=$(brew list --versions rocksdb | awk '{print $2}' | cut -d' ' -f1)
+ else
+ VERSION=$PINNED_VERSION
+ echo "No rocksdb package found, building the pinned version $VERSION."
+ echo "Give another one with -v, see https://github.com/facebook/rocksdb/tags"
+ fi
+fi
+echo "Building rocksdb $VERSION"
+
+# --- the prerequisites -------------------------------------------------------
+missing=
+for tool in git cmake make; do
+ command -v $tool > /dev/null || missing="$missing $tool"
+done
+if ! command -v c++ > /dev/null && ! command -v g++ > /dev/null && ! command -v clang++ > /dev/null; then
+ missing="$missing c++"
+fi
+# The compression libraries rocksdb is built with below. Where they are
+# missing the build still succeeds, just without them, so this is a hint.
+compression_hint=
+if command -v pkg-config > /dev/null; then
+ for lib in snappy zlib bzip2 liblz4 libzstd; do
+ pkg-config --exists $lib 2> /dev/null || compression_hint="$compression_hint $lib"
+ done
+fi
+if [ -n "$missing" ]; then
+ echo "Missing build tools:$missing" >&2
+ echo >&2
+ echo "Install them and the compression libraries:" >&2
+ echo " Fedora, Rocky, RHEL: sudo dnf install cmake gcc-c++ git snappy-devel zlib-devel bzip2-devel lz4-devel libzstd-devel" >&2
+ echo " Debian, Ubuntu: sudo apt-get install cmake g++ git libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev" >&2
+ echo " MacOS: brew install cmake snappy zlib bzip2 lz4 zstd" >&2
+ exit 1
+fi
+if [ -n "$compression_hint" ]; then
+ echo "Warning: pkg-config does not find:$compression_hint" >&2
+ echo "rocksdb will be built without them. The devel packages named above provide them." >&2
+fi
+
+# --- the sources -------------------------------------------------------------
+if [ -d "$SOURCE_DIR/.git" ]; then
+ echo "Reusing the checkout in $SOURCE_DIR"
+ git -C "$SOURCE_DIR" fetch --depth 1 origin "refs/tags/v$VERSION"
+ git -C "$SOURCE_DIR" checkout -q FETCH_HEAD
+else
+ git clone https://github.com/facebook/rocksdb.git --branch "v$VERSION" --depth 1 "$SOURCE_DIR"
+fi
+
+# --- the build ---------------------------------------------------------------
+# lib64 where the distribution has one (Fedora, Rocky, RHEL, openSUSE), lib
+# elsewhere. Both are searched for a prefix by cmake, this is for the user.
+if [ -d /usr/lib64 ] && [ ! -e /etc/debian_version ]; then
+ LIBDIR=lib64
+else
+ LIBDIR=lib
+fi
+
+# WITH_GFLAGS and WITH_LIBURING are off so that the static library does not
+# drag them into everything that links it. FAIL_ON_WARNINGS off so that a
+# newer compiler does not turn rocksdb's own warnings into errors. And
+# -include cstdint because older rocksdb versions declare uint64_t and friends
+# without including it, which newer compilers no longer tolerate - Fedora 44
+# packages 10.2.1 and builds it with gcc 16, for instance. Where it is already
+# included, the flag changes nothing.
+cmake -S "$SOURCE_DIR" -B "$SOURCE_DIR/build" \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_CXX_FLAGS="-include cstdint" \
+ -DFAIL_ON_WARNINGS=OFF \
+ -DWITH_TESTS=OFF -DWITH_ALL_TESTS=OFF \
+ -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_CORE_TOOLS=OFF \
+ -DWITH_GFLAGS=OFF -DWITH_LIBURING=OFF \
+ -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON -DWITH_ZSTD=ON -DWITH_BZ2=ON \
+ -DROCKSDB_BUILD_SHARED=$([ $STATIC_ONLY = 1 ] && echo OFF || echo ON) \
+ -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+ -DCMAKE_INSTALL_LIBDIR="$LIBDIR"
+cmake --build "$SOURCE_DIR/build" -j "$(getconf _NPROCESSORS_ONLN 2> /dev/null || sysctl -n hw.ncpu)"
+cmake --install "$SOURCE_DIR/build"
+
+# --- what to do with it ------------------------------------------------------
+echo
+echo "Installed rocksdb $VERSION:"
+echo " headers: $PREFIX/include"
+echo " static: $PREFIX/$LIBDIR/librocksdb.a"
+[ $STATIC_ONLY = 1 ] || echo " shared: $PREFIX/$LIBDIR/$( [ "$(uname)" = Darwin ] && echo librocksdb.dylib || echo librocksdb.so )"
+echo
+echo "Build collocatordb against it with:"
+echo " cmake -S . -B build -DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_INSTALL_PREFIX=/usr/local"
+echo
+echo "Build dereko2vec against it with:"
+echo " cmake -S . -B build -DSTATIC_ROCKSDB=ON -DCMAKE_PREFIX_PATH=$PREFIX"
+if [ $STATIC_ONLY = 0 ] && [ "$(uname)" = Linux ]; then
+ echo
+ echo "Programs that use the shared library find it after:"
+ echo " echo $PREFIX/$LIBDIR | sudo tee /etc/ld.so.conf.d/rocksdb.conf && sudo ldconfig"
+fi