Install everything with one script, and use it in the CI
scripts/install.sh runs the whole installation: the packages of the
distribution, rocksdb where there is none or none with the static
library, collocatordb, and dereko2vec with the fast static link, tested
and installed below /usr/local. Options: --shared, --skip-deps,
--skip-tests, --static-binary, --prefix, --rocksdb-prefix,
--rocksdb-version, -j.
The GitHub workflow (Ubuntu, MacOS) and the GitLab CI (Debian, with the
completely static binary as the artifact) now call it instead of
repeating the steps.
build-rocksdb.sh passes the homebrew prefix to cmake on MacOS, so that
the compression libraries are found.
Change-Id: Idbe7d7e598001d91aef8aaba045ac1ace61bf92a
diff --git a/scripts/install.sh b/scripts/install.sh
new file mode 100755
index 0000000..bb076ea
--- /dev/null
+++ b/scripts/install.sh
@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+#
+# Installs dereko2vec and everything it needs, in one go:
+#
+# 1. the packages of the distribution (dnf, apt-get or brew)
+# 2. rocksdb, but only where the distribution has none or none with the
+# static library (scripts/build-rocksdb.sh)
+# 3. collocatordb
+# 4. dereko2vec, with the fast static link of rocksdb and collocatordb
+#
+# Works on Fedora, Rocky Linux, RHEL, Debian, Ubuntu and MacOS.
+#
+# Options:
+# --shared link rocksdb and collocatordb shared (default:
+# static, which is about 22% faster when indexing)
+# --prefix DIR install there instead of /usr/local
+# --rocksdb-prefix DIR build rocksdb into DIR instead of $HOME/rocksdb
+# --rocksdb-version V build rocksdb version V instead of the one of the
+# distribution
+# --skip-deps do not install the packages of the distribution
+# --skip-tests do not run ctest
+# --static-binary build dereko2vec as a completely static binary
+# (needs static compression libraries, which Debian
+# and Ubuntu ship and Fedora, Rocky and RHEL do not)
+# -j N build with N jobs (default: all cores)
+#
+set -euo pipefail
+
+DEREKO2VEC_SRC=$(cd "$(dirname "$0")/.." && pwd)
+PREFIX=/usr/local
+ROCKSDB_PREFIX=$HOME/rocksdb
+ROCKSDB_VERSION=
+STATIC=1
+STATIC_BINARY=0
+SKIP_DEPS=0
+SKIP_TESTS=0
+JOBS=
+
+usage() {
+ sed -n '2,26p' "$0"
+ exit "${1:-0}"
+}
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ --shared) STATIC=0; shift ;;
+ --static-binary) STATIC_BINARY=1; shift ;;
+ --prefix) PREFIX=$2; shift 2 ;;
+ --rocksdb-prefix) ROCKSDB_PREFIX=$2; shift 2 ;;
+ --rocksdb-version) ROCKSDB_VERSION=$2; shift 2 ;;
+ --skip-deps) SKIP_DEPS=1; shift ;;
+ --skip-tests) SKIP_TESTS=1; shift ;;
+ -j) JOBS=$2; shift 2 ;;
+ -h|--help) usage 0 ;;
+ *) echo "Unknown option: $1" >&2; usage 1 ;;
+ esac
+done
+
+if [ -z "$JOBS" ]; then
+ JOBS=$(getconf _NPROCESSORS_ONLN 2> /dev/null || sysctl -n hw.ncpu)
+fi
+export CMAKE_BUILD_PARALLEL_LEVEL=$JOBS
+
+step() { printf '\n=== %s ===\n' "$*"; }
+
+# sudo only where the target is not writable
+if [ -w "$PREFIX" ] || { [ ! -e "$PREFIX" ] && [ -w "$(dirname "$PREFIX")" ]; }; then
+ SUDO=
+else
+ SUDO=sudo
+fi
+
+# The rocksdb and the compression libraries of homebrew are below
+# /opt/homebrew on Apple Silicon, which is not searched by default
+BREW_PREFIX=
+if command -v brew > /dev/null; then
+ BREW_PREFIX=$(brew --prefix)
+fi
+
+# --- 1. the packages of the distribution -------------------------------------
+if [ $SKIP_DEPS = 0 ]; then
+ step "Installing the packages of the distribution"
+ if command -v dnf > /dev/null; then
+ $SUDO dnf install -y cmake gcc-c++ git make pkgconf-pkg-config \
+ snappy-devel zlib-devel bzip2-devel lz4-devel libzstd-devel
+ # Rocky Linux and RHEL have no rocksdb-devel everywhere, that is what
+ # step 2 is for
+ $SUDO dnf install -y rocksdb-devel || true
+ elif command -v apt-get > /dev/null; then
+ $SUDO apt-get update
+ $SUDO apt-get install -y cmake g++ git make pkg-config \
+ librocksdb-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+ elif command -v brew > /dev/null; then
+ brew install cmake rocksdb snappy zlib bzip2 lz4 zstd
+ else
+ echo "No dnf, apt-get or brew - install cmake, a C++ compiler, git, rocksdb" >&2
+ echo "and the devel packages of snappy, zlib, bzip2, lz4 and zstd by hand." >&2
+ fi
+fi
+
+# --- 2. rocksdb --------------------------------------------------------------
+static_rocksdb_of_the_distribution() {
+ local f
+ for f in /usr/lib64/librocksdb.a /usr/lib/librocksdb.a /usr/lib/*/librocksdb.a; do
+ [ -e "$f" ] && return 0
+ done
+ if command -v brew > /dev/null && brew list --versions rocksdb > /dev/null 2>&1; then
+ [ -e "$(brew --prefix rocksdb)/lib/librocksdb.a" ] && return 0
+ fi
+ return 1
+}
+
+DISTRO_ROCKSDB=
+if { command -v rpm > /dev/null && rpm -q rocksdb > /dev/null 2>&1; } ||
+ { command -v dpkg-query > /dev/null && dpkg-query -W librocksdb-dev > /dev/null 2>&1; } ||
+ { command -v brew > /dev/null && brew list --versions rocksdb > /dev/null 2>&1; }; then
+ DISTRO_ROCKSDB=1
+fi
+
+ROCKSDB_PREFIX_USED=
+ROCKSDB_SHARED_BUILT=0
+if [ -n "$DISTRO_ROCKSDB" ] && [ -z "$ROCKSDB_VERSION" ]; then
+ if [ $STATIC = 1 ] && ! static_rocksdb_of_the_distribution; then
+ step "rocksdb: building the static library, same version as the distribution"
+ "$DEREKO2VEC_SRC/scripts/build-rocksdb.sh" --static-only -p "$ROCKSDB_PREFIX"
+ ROCKSDB_PREFIX_USED=$ROCKSDB_PREFIX
+ else
+ step "rocksdb: the one of the distribution is used"
+ fi
+else
+ if [ -z "$DISTRO_ROCKSDB" ]; then
+ step "rocksdb: no rocksdb package - building the shared and the static library"
+ else
+ step "rocksdb: building version $ROCKSDB_VERSION as asked"
+ fi
+ "$DEREKO2VEC_SRC/scripts/build-rocksdb.sh" -p "$ROCKSDB_PREFIX" ${ROCKSDB_VERSION:+-v "$ROCKSDB_VERSION"}
+ ROCKSDB_PREFIX_USED=$ROCKSDB_PREFIX
+ ROCKSDB_SHARED_BUILT=1
+fi
+
+# Programs that link the shared rocksdb of the prefix have to find it. Only a
+# system-wide install can register the directory; for one below $HOME the
+# programs carry the rpath of the libraries they were linked against.
+if [ $ROCKSDB_SHARED_BUILT = 1 ] && [ "$(uname)" = Linux ] && [ -n "$SUDO" ]; then
+ RDB_LIBDIR=$(dirname "$(find "$ROCKSDB_PREFIX_USED" -name 'librocksdb.so' | head -1)")
+ echo "$RDB_LIBDIR" | $SUDO tee /etc/ld.so.conf.d/rocksdb-prefix.conf > /dev/null
+ $SUDO ldconfig
+fi
+
+# --- 3. collocatordb ---------------------------------------------------------
+step "collocatordb"
+WORK=$(mktemp -d)
+trap 'rc=$?; if [ $rc = 0 ]; then rm -rf "$WORK"; else echo "Keeping $WORK for a look at what went wrong" >&2; fi' EXIT
+git clone "https://korap.ids-mannheim.de/gerrit/ids-kl/collocatordb" "$WORK/collocatordb"
+CDB_PREFIXES=$ROCKSDB_PREFIX_USED
+if [ -n "$BREW_PREFIX" ]; then
+ CDB_PREFIXES=${CDB_PREFIXES:+$CDB_PREFIXES;}$BREW_PREFIX
+fi
+cmake -S "$WORK/collocatordb" -B "$WORK/collocatordb/build" \
+ -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+ ${CDB_PREFIXES:+-DCMAKE_PREFIX_PATH=$CDB_PREFIXES}
+cmake --build "$WORK/collocatordb/build" -j "$JOBS"
+$SUDO cmake --install "$WORK/collocatordb/build"
+if [ "$(uname)" = Linux ] && [ -n "$SUDO" ]; then
+ $SUDO ldconfig
+fi
+if [ $SKIP_TESTS = 0 ]; then
+ ctest --test-dir "$WORK/collocatordb/build" --output-on-failure
+fi
+
+# --- 4. dereko2vec -----------------------------------------------------------
+step "dereko2vec"
+# A stale build directory would silently keep the collocatordb and rocksdb it
+# was once configured with, and an install manifest of a root install breaks a
+# later install without sudo. Start over, the build takes a minute.
+rm -rf "$DEREKO2VEC_SRC/build"
+CMAKE_PREFIXES=$PREFIX
+if [ -n "$ROCKSDB_PREFIX_USED" ]; then
+ CMAKE_PREFIXES="$CMAKE_PREFIXES;$ROCKSDB_PREFIX_USED"
+fi
+if [ -n "$BREW_PREFIX" ]; then
+ CMAKE_PREFIXES="$CMAKE_PREFIXES;$BREW_PREFIX"
+fi
+STATIC_FLAG=
+if [ $STATIC_BINARY = 1 ]; then
+ STATIC_FLAG=-DSTATIC_DEREKO2VEC=ON
+elif [ $STATIC = 1 ]; then
+ STATIC_FLAG=-DSTATIC_ROCKSDB=ON
+fi
+cmake -S "$DEREKO2VEC_SRC" -B "$DEREKO2VEC_SRC/build" \
+ -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+ -DCMAKE_PREFIX_PATH="$CMAKE_PREFIXES" \
+ $STATIC_FLAG
+cmake --build "$DEREKO2VEC_SRC/build" -j "$JOBS"
+if [ $SKIP_TESTS = 0 ]; then
+ ctest --test-dir "$DEREKO2VEC_SRC/build" --output-on-failure
+fi
+$SUDO cmake --install "$DEREKO2VEC_SRC/build"
+
+step "Done"
+echo "dereko2vec and vecs2mmap are in $PREFIX/bin"