Install everything with one script, and use it in the CI
scripts/install.sh runs the whole installation: the packages of the
distribution, rocksdb where there is none or none with the static
library, collocatordb, and dereko2vec with the fast static link, tested
and installed below /usr/local. Options: --shared, --skip-deps,
--skip-tests, --static-binary, --prefix, --rocksdb-prefix,
--rocksdb-version, -j.
The GitHub workflow (Ubuntu, MacOS) and the GitLab CI (Debian, with the
completely static binary as the artifact) now call it instead of
repeating the steps.
build-rocksdb.sh passes the homebrew prefix to cmake on MacOS, so that
the compression libraries are found.
Change-Id: Idbe7d7e598001d91aef8aaba045ac1ace61bf92a
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3bfddc8..6f634da 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -17,44 +17,27 @@
steps:
- uses: actions/checkout@v4
- - name: Install dependencies (Linux)
+ # Deps, collocatordb and dereko2vec, with rocksdb and collocatordb
+ # linked statically. The distribution brings the static rocksdb on both,
+ # so no rocksdb is built.
+ - name: Install everything (Linux)
if: runner.os == 'Linux'
- run: |
- sudo apt-get update
- sudo apt-get install -y cmake librocksdb-dev libgflags-dev libsnappy-dev \
- zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+ run: scripts/install.sh
- - name: Install dependencies (macOS)
+ # The count based part of the test traps on MacOS, while it runs on
+ # Linux. So the tests are a step of their own there, with a backtrace
+ # below when they fail, instead of failing the installation.
+ - name: Install everything (macOS)
if: runner.os == 'macOS'
- run: brew install cmake rocksdb snappy lz4 zstd gflags
+ run: scripts/install.sh --skip-tests
- - name: Build and install collocatordb
- run: |
- EXTRA=""
- [ "$RUNNER_OS" = "macOS" ] && EXTRA="-DCMAKE_PREFIX_PATH=$(brew --prefix)"
- git clone --depth 1 https://korap.ids-mannheim.de/gerrit/ids-kl/collocatordb
- cmake -S collocatordb -B collocatordb/build -DCMAKE_INSTALL_PREFIX=/usr/local $EXTRA
- cmake --build collocatordb/build -j
- sudo cmake --install collocatordb/build
- if [ "$RUNNER_OS" = "Linux" ]; then sudo ldconfig; fi
-
- # The build directory has to be build/ inside the sources, the test looks
- # for the binary relative to it.
- - name: Configure
- run: |
- EXTRA=""
- [ "$RUNNER_OS" = "macOS" ] && EXTRA="-DCMAKE_PREFIX_PATH=$(brew --prefix)"
- cmake -S . -B build -DCMAKE_INSTALL_PREFIX=/usr/local $EXTRA
-
- - name: Build
- run: cmake --build build -j
-
- - name: Test
+ - name: Test (macOS)
+ if: runner.os == 'macOS'
run: ctest --test-dir build --output-on-failure
- # The count based part of the test traps on MacOS, while it runs on Linux
- # with rocksdb 8.9, 10.2 and 11.0. Without a backtrace there is nothing to
- # go by, so produce one.
+ # The count based part of the test traps on MacOS, while it runs on
+ # Linux with rocksdb 8.9, 10.2 and 11.0. Without a backtrace there is
+ # nothing to go by, so produce one.
- name: Backtrace of the crash (macOS)
if: failure() && runner.os == 'macOS'
run: |
@@ -79,8 +62,8 @@
echo "written by vecs2mmap:"
ls -l /tmp/m.vecs.vecs /tmp/m.vecs.words
- - name: Install
+ # install.sh has already installed; make sure the installed binaries run
+ - name: Smoke test of the installed binaries
run: |
- sudo cmake --install build
dereko2vec 2>&1 | head -1
vecs2mmap -h | head -1
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5ea23b7..f77d7e4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,33 +32,18 @@
- start_section install_linux_packages "Installing missing Linux packages"
- mkdir -pv $APT_CACHE_DIR ccache
- apt-get update && apt-get -o dir::cache::archives="$APT_CACHE_DIR" -y install ccache cmake librocksdb-dev libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev libomp-dev
- # ccache only makes the build faster, it must never be able to make it fail,
- # so everything that uses it is guarded and CCACHE_OPTS stays empty without it
- - if CCACHE=$(command -v ccache); then ln -sf "$CCACHE" /usr/local/sbin/gcc && ln -sf "$CCACHE" /usr/local/sbin/g++ && CCACHE_OPTS="-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"; else echo "ccache is not available - building without it"; CCACHE_OPTS=""; fi
+ # ccache only makes the build faster, it must never be able to make it
+ # fail, so it is wired in through compiler symlinks only when it is there
+ - if CCACHE=$(command -v ccache); then ln -sf "$CCACHE" /usr/local/sbin/gcc && ln -sf "$CCACHE" /usr/local/sbin/g++; else echo "ccache is not available - building without it"; fi
- end_section install_linux_packages
- - start_section install_collocatordb "Building and installing collocatordb"
- - git clone "https://korap.ids-mannheim.de/gerrit/ids-kl/collocatordb"
- - cd collocatordb
- - mkdir -p build
- - cd build
- - cmake $CCACHE_OPTS ..
- - make
- - command -v ccache > /dev/null && ccache --show-stats || true
- - make install # && ctest --extra-verbose
- - ldconfig
- - end_section install_collocatordb
- - cd ../..
-
script:
+ # Debian ships the static rocksdb in librocksdb-dev, so install.sh builds
+ # no rocksdb here. --static-binary keeps the artifact a completely static
+ # binary. ccache works through the compiler symlinks set up above.
- start_section build_dereko2vec "Building and testing dereko2vec"
- - pwd
- - mkdir build
- - cd build
- - cmake -DSTATIC_DEREKO2VEC=ON $CCACHE_OPTS ..
- - make || (cmake $CCACHE_OPTS .. && make)
+ - scripts/install.sh --skip-deps --static-binary
- command -v ccache > /dev/null && ccache --show-stats || true
- - ctest --extra-verbose
- end_section build_dereko2vec
artifacts:
diff --git a/README.md b/README.md
index bf06a9f..9cb8e5b 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,23 @@
## Installation
+### The short way
+
+```bash
+scripts/install.sh
+```
+
+installs the packages of the distribution, builds rocksdb where there is none
+or none with the static library, then collocatordb, and then dereko2vec with
+the fast static link, testing and installing everything below `/usr/local`.
+Works on Fedora, Rocky Linux 9 and 10, RHEL, Debian, Ubuntu and MacOS. See
+`scripts/install.sh --help` for the options, e.g. `--shared` for linking
+rocksdb and collocatordb shared, or `--prefix` for another install location.
+
+### The four steps, one at a time
+
+What the script runs, for doing it by hand or where it does not reach:
+
### 1. Prerequisites
* on Fedora:
diff --git a/scripts/build-rocksdb.sh b/scripts/build-rocksdb.sh
index 36e4565..cc2cd1f 100755
--- a/scripts/build-rocksdb.sh
+++ b/scripts/build-rocksdb.sh
@@ -107,6 +107,12 @@
LIBDIR=lib
fi
+# So that the compression libraries of homebrew are found on MacOS
+BREW_PREFIX=
+if command -v brew > /dev/null; then
+ BREW_PREFIX="-DCMAKE_PREFIX_PATH=$(brew --prefix)"
+fi
+
# WITH_GFLAGS and WITH_LIBURING are off so that the static library does not
# drag them into everything that links it. FAIL_ON_WARNINGS off so that a
# newer compiler does not turn rocksdb's own warnings into errors. And
@@ -124,8 +130,9 @@
-DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON -DWITH_ZSTD=ON -DWITH_BZ2=ON \
-DROCKSDB_BUILD_SHARED=$([ $STATIC_ONLY = 1 ] && echo OFF || echo ON) \
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
- -DCMAKE_INSTALL_LIBDIR="$LIBDIR"
-cmake --build "$SOURCE_DIR/build" -j "$(getconf _NPROCESSORS_ONLN 2> /dev/null || sysctl -n hw.ncpu)"
+ -DCMAKE_INSTALL_LIBDIR="$LIBDIR" \
+ $BREW_PREFIX
+cmake --build "$SOURCE_DIR/build" -j "${CMAKE_BUILD_PARALLEL_LEVEL:-$(getconf _NPROCESSORS_ONLN 2> /dev/null || sysctl -n hw.ncpu)}"
cmake --install "$SOURCE_DIR/build"
# --- what to do with it ------------------------------------------------------
diff --git a/scripts/install.sh b/scripts/install.sh
new file mode 100755
index 0000000..bb076ea
--- /dev/null
+++ b/scripts/install.sh
@@ -0,0 +1,201 @@
+#!/usr/bin/env bash
+#
+# Installs dereko2vec and everything it needs, in one go:
+#
+# 1. the packages of the distribution (dnf, apt-get or brew)
+# 2. rocksdb, but only where the distribution has none or none with the
+# static library (scripts/build-rocksdb.sh)
+# 3. collocatordb
+# 4. dereko2vec, with the fast static link of rocksdb and collocatordb
+#
+# Works on Fedora, Rocky Linux, RHEL, Debian, Ubuntu and MacOS.
+#
+# Options:
+# --shared link rocksdb and collocatordb shared (default:
+# static, which is about 22% faster when indexing)
+# --prefix DIR install there instead of /usr/local
+# --rocksdb-prefix DIR build rocksdb into DIR instead of $HOME/rocksdb
+# --rocksdb-version V build rocksdb version V instead of the one of the
+# distribution
+# --skip-deps do not install the packages of the distribution
+# --skip-tests do not run ctest
+# --static-binary build dereko2vec as a completely static binary
+# (needs static compression libraries, which Debian
+# and Ubuntu ship and Fedora, Rocky and RHEL do not)
+# -j N build with N jobs (default: all cores)
+#
+set -euo pipefail
+
+DEREKO2VEC_SRC=$(cd "$(dirname "$0")/.." && pwd)
+PREFIX=/usr/local
+ROCKSDB_PREFIX=$HOME/rocksdb
+ROCKSDB_VERSION=
+STATIC=1
+STATIC_BINARY=0
+SKIP_DEPS=0
+SKIP_TESTS=0
+JOBS=
+
+usage() {
+ sed -n '2,26p' "$0"
+ exit "${1:-0}"
+}
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ --shared) STATIC=0; shift ;;
+ --static-binary) STATIC_BINARY=1; shift ;;
+ --prefix) PREFIX=$2; shift 2 ;;
+ --rocksdb-prefix) ROCKSDB_PREFIX=$2; shift 2 ;;
+ --rocksdb-version) ROCKSDB_VERSION=$2; shift 2 ;;
+ --skip-deps) SKIP_DEPS=1; shift ;;
+ --skip-tests) SKIP_TESTS=1; shift ;;
+ -j) JOBS=$2; shift 2 ;;
+ -h|--help) usage 0 ;;
+ *) echo "Unknown option: $1" >&2; usage 1 ;;
+ esac
+done
+
+if [ -z "$JOBS" ]; then
+ JOBS=$(getconf _NPROCESSORS_ONLN 2> /dev/null || sysctl -n hw.ncpu)
+fi
+export CMAKE_BUILD_PARALLEL_LEVEL=$JOBS
+
+step() { printf '\n=== %s ===\n' "$*"; }
+
+# sudo only where the target is not writable
+if [ -w "$PREFIX" ] || { [ ! -e "$PREFIX" ] && [ -w "$(dirname "$PREFIX")" ]; }; then
+ SUDO=
+else
+ SUDO=sudo
+fi
+
+# The rocksdb and the compression libraries of homebrew are below
+# /opt/homebrew on Apple Silicon, which is not searched by default
+BREW_PREFIX=
+if command -v brew > /dev/null; then
+ BREW_PREFIX=$(brew --prefix)
+fi
+
+# --- 1. the packages of the distribution -------------------------------------
+if [ $SKIP_DEPS = 0 ]; then
+ step "Installing the packages of the distribution"
+ if command -v dnf > /dev/null; then
+ $SUDO dnf install -y cmake gcc-c++ git make pkgconf-pkg-config \
+ snappy-devel zlib-devel bzip2-devel lz4-devel libzstd-devel
+ # Rocky Linux and RHEL have no rocksdb-devel everywhere, that is what
+ # step 2 is for
+ $SUDO dnf install -y rocksdb-devel || true
+ elif command -v apt-get > /dev/null; then
+ $SUDO apt-get update
+ $SUDO apt-get install -y cmake g++ git make pkg-config \
+ librocksdb-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+ elif command -v brew > /dev/null; then
+ brew install cmake rocksdb snappy zlib bzip2 lz4 zstd
+ else
+ echo "No dnf, apt-get or brew - install cmake, a C++ compiler, git, rocksdb" >&2
+ echo "and the devel packages of snappy, zlib, bzip2, lz4 and zstd by hand." >&2
+ fi
+fi
+
+# --- 2. rocksdb --------------------------------------------------------------
+static_rocksdb_of_the_distribution() {
+ local f
+ for f in /usr/lib64/librocksdb.a /usr/lib/librocksdb.a /usr/lib/*/librocksdb.a; do
+ [ -e "$f" ] && return 0
+ done
+ if command -v brew > /dev/null && brew list --versions rocksdb > /dev/null 2>&1; then
+ [ -e "$(brew --prefix rocksdb)/lib/librocksdb.a" ] && return 0
+ fi
+ return 1
+}
+
+DISTRO_ROCKSDB=
+if { command -v rpm > /dev/null && rpm -q rocksdb > /dev/null 2>&1; } ||
+ { command -v dpkg-query > /dev/null && dpkg-query -W librocksdb-dev > /dev/null 2>&1; } ||
+ { command -v brew > /dev/null && brew list --versions rocksdb > /dev/null 2>&1; }; then
+ DISTRO_ROCKSDB=1
+fi
+
+ROCKSDB_PREFIX_USED=
+ROCKSDB_SHARED_BUILT=0
+if [ -n "$DISTRO_ROCKSDB" ] && [ -z "$ROCKSDB_VERSION" ]; then
+ if [ $STATIC = 1 ] && ! static_rocksdb_of_the_distribution; then
+ step "rocksdb: building the static library, same version as the distribution"
+ "$DEREKO2VEC_SRC/scripts/build-rocksdb.sh" --static-only -p "$ROCKSDB_PREFIX"
+ ROCKSDB_PREFIX_USED=$ROCKSDB_PREFIX
+ else
+ step "rocksdb: the one of the distribution is used"
+ fi
+else
+ if [ -z "$DISTRO_ROCKSDB" ]; then
+ step "rocksdb: no rocksdb package - building the shared and the static library"
+ else
+ step "rocksdb: building version $ROCKSDB_VERSION as asked"
+ fi
+ "$DEREKO2VEC_SRC/scripts/build-rocksdb.sh" -p "$ROCKSDB_PREFIX" ${ROCKSDB_VERSION:+-v "$ROCKSDB_VERSION"}
+ ROCKSDB_PREFIX_USED=$ROCKSDB_PREFIX
+ ROCKSDB_SHARED_BUILT=1
+fi
+
+# Programs that link the shared rocksdb of the prefix have to find it. Only a
+# system-wide install can register the directory; for one below $HOME the
+# programs carry the rpath of the libraries they were linked against.
+if [ $ROCKSDB_SHARED_BUILT = 1 ] && [ "$(uname)" = Linux ] && [ -n "$SUDO" ]; then
+ RDB_LIBDIR=$(dirname "$(find "$ROCKSDB_PREFIX_USED" -name 'librocksdb.so' | head -1)")
+ echo "$RDB_LIBDIR" | $SUDO tee /etc/ld.so.conf.d/rocksdb-prefix.conf > /dev/null
+ $SUDO ldconfig
+fi
+
+# --- 3. collocatordb ---------------------------------------------------------
+step "collocatordb"
+WORK=$(mktemp -d)
+trap 'rc=$?; if [ $rc = 0 ]; then rm -rf "$WORK"; else echo "Keeping $WORK for a look at what went wrong" >&2; fi' EXIT
+git clone "https://korap.ids-mannheim.de/gerrit/ids-kl/collocatordb" "$WORK/collocatordb"
+CDB_PREFIXES=$ROCKSDB_PREFIX_USED
+if [ -n "$BREW_PREFIX" ]; then
+ CDB_PREFIXES=${CDB_PREFIXES:+$CDB_PREFIXES;}$BREW_PREFIX
+fi
+cmake -S "$WORK/collocatordb" -B "$WORK/collocatordb/build" \
+ -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+ ${CDB_PREFIXES:+-DCMAKE_PREFIX_PATH=$CDB_PREFIXES}
+cmake --build "$WORK/collocatordb/build" -j "$JOBS"
+$SUDO cmake --install "$WORK/collocatordb/build"
+if [ "$(uname)" = Linux ] && [ -n "$SUDO" ]; then
+ $SUDO ldconfig
+fi
+if [ $SKIP_TESTS = 0 ]; then
+ ctest --test-dir "$WORK/collocatordb/build" --output-on-failure
+fi
+
+# --- 4. dereko2vec -----------------------------------------------------------
+step "dereko2vec"
+# A stale build directory would silently keep the collocatordb and rocksdb it
+# was once configured with, and an install manifest of a root install breaks a
+# later install without sudo. Start over, the build takes a minute.
+rm -rf "$DEREKO2VEC_SRC/build"
+CMAKE_PREFIXES=$PREFIX
+if [ -n "$ROCKSDB_PREFIX_USED" ]; then
+ CMAKE_PREFIXES="$CMAKE_PREFIXES;$ROCKSDB_PREFIX_USED"
+fi
+if [ -n "$BREW_PREFIX" ]; then
+ CMAKE_PREFIXES="$CMAKE_PREFIXES;$BREW_PREFIX"
+fi
+STATIC_FLAG=
+if [ $STATIC_BINARY = 1 ]; then
+ STATIC_FLAG=-DSTATIC_DEREKO2VEC=ON
+elif [ $STATIC = 1 ]; then
+ STATIC_FLAG=-DSTATIC_ROCKSDB=ON
+fi
+cmake -S "$DEREKO2VEC_SRC" -B "$DEREKO2VEC_SRC/build" \
+ -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+ -DCMAKE_PREFIX_PATH="$CMAKE_PREFIXES" \
+ $STATIC_FLAG
+cmake --build "$DEREKO2VEC_SRC/build" -j "$JOBS"
+if [ $SKIP_TESTS = 0 ]; then
+ ctest --test-dir "$DEREKO2VEC_SRC/build" --output-on-failure
+fi
+$SUDO cmake --install "$DEREKO2VEC_SRC/build"
+
+step "Done"
+echo "dereko2vec and vecs2mmap are in $PREFIX/bin"