| #!/usr/bin/env bash |
| # |
| # Installs dereko2vec and everything it needs, in one go: |
| # |
| # 1. the packages of the distribution (dnf, apt-get or brew) |
| # 2. rocksdb, but only where the distribution has none or none with the |
| # static library (scripts/build-rocksdb.sh) |
| # 3. collocatordb |
| # 4. dereko2vec, with the fast static link of rocksdb and collocatordb |
| # |
| # Works on Fedora, Rocky Linux, RHEL, Debian, Ubuntu and MacOS. |
| # |
| # Options: |
| # --shared link rocksdb and collocatordb shared (default: |
| # static, which is about 22% faster when indexing) |
| # --prefix DIR install there instead of /usr/local |
| # --rocksdb-prefix DIR build rocksdb into DIR instead of $HOME/rocksdb |
| # --rocksdb-version V build rocksdb version V instead of the one of the |
| # distribution |
| # --skip-deps do not install the packages of the distribution |
| # --skip-tests do not run ctest |
| # --static-binary build dereko2vec as a completely static binary |
| # (needs static compression libraries, which Debian |
| # and Ubuntu ship and Fedora, Rocky and RHEL do not) |
| # -j N build with N jobs (default: all cores) |
| # |
| set -euo pipefail |
| |
| DEREKO2VEC_SRC=$(cd "$(dirname "$0")/.." && pwd) |
| PREFIX=/usr/local |
| ROCKSDB_PREFIX=$HOME/rocksdb |
| ROCKSDB_VERSION= |
| STATIC=1 |
| STATIC_BINARY=0 |
| SKIP_DEPS=0 |
| SKIP_TESTS=0 |
| JOBS= |
| |
| usage() { |
| sed -n '2,26p' "$0" |
| exit "${1:-0}" |
| } |
| |
| while [ $# -gt 0 ]; do |
| case $1 in |
| --shared) STATIC=0; shift ;; |
| --static-binary) STATIC_BINARY=1; shift ;; |
| --prefix) PREFIX=$2; shift 2 ;; |
| --rocksdb-prefix) ROCKSDB_PREFIX=$2; shift 2 ;; |
| --rocksdb-version) ROCKSDB_VERSION=$2; shift 2 ;; |
| --skip-deps) SKIP_DEPS=1; shift ;; |
| --skip-tests) SKIP_TESTS=1; shift ;; |
| -j) JOBS=$2; shift 2 ;; |
| -h|--help) usage 0 ;; |
| *) echo "Unknown option: $1" >&2; usage 1 ;; |
| esac |
| done |
| |
| if [ -z "$JOBS" ]; then |
| JOBS=$(getconf _NPROCESSORS_ONLN 2> /dev/null || sysctl -n hw.ncpu) |
| fi |
| export CMAKE_BUILD_PARALLEL_LEVEL=$JOBS |
| |
| step() { printf '\n=== %s ===\n' "$*"; } |
| |
| # sudo only where the target is not writable |
| if [ -w "$PREFIX" ] || { [ ! -e "$PREFIX" ] && [ -w "$(dirname "$PREFIX")" ]; }; then |
| SUDO= |
| else |
| SUDO=sudo |
| fi |
| |
| # The rocksdb and the compression libraries of homebrew are below |
| # /opt/homebrew on Apple Silicon, which is not searched by default |
| BREW_PREFIX= |
| if command -v brew > /dev/null; then |
| BREW_PREFIX=$(brew --prefix) |
| fi |
| |
| # --- 1. the packages of the distribution ------------------------------------- |
| if [ $SKIP_DEPS = 0 ]; then |
| step "Installing the packages of the distribution" |
| if command -v dnf > /dev/null; then |
| $SUDO dnf install -y cmake gcc-c++ git make pkgconf-pkg-config \ |
| snappy-devel zlib-devel bzip2-devel lz4-devel libzstd-devel |
| # Rocky Linux and RHEL have no rocksdb-devel everywhere, that is what |
| # step 2 is for |
| $SUDO dnf install -y rocksdb-devel || true |
| elif command -v apt-get > /dev/null; then |
| $SUDO apt-get update |
| $SUDO apt-get install -y cmake g++ git make pkg-config \ |
| librocksdb-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev |
| elif command -v brew > /dev/null; then |
| brew install cmake rocksdb snappy zlib bzip2 lz4 zstd |
| else |
| echo "No dnf, apt-get or brew - install cmake, a C++ compiler, git, rocksdb" >&2 |
| echo "and the devel packages of snappy, zlib, bzip2, lz4 and zstd by hand." >&2 |
| fi |
| fi |
| |
| # --- 2. rocksdb -------------------------------------------------------------- |
| static_rocksdb_of_the_distribution() { |
| local f |
| for f in /usr/lib64/librocksdb.a /usr/lib/librocksdb.a /usr/lib/*/librocksdb.a; do |
| [ -e "$f" ] && return 0 |
| done |
| if command -v brew > /dev/null && brew list --versions rocksdb > /dev/null 2>&1; then |
| [ -e "$(brew --prefix rocksdb)/lib/librocksdb.a" ] && return 0 |
| fi |
| return 1 |
| } |
| |
| DISTRO_ROCKSDB= |
| if { command -v rpm > /dev/null && rpm -q rocksdb > /dev/null 2>&1; } || |
| { command -v dpkg-query > /dev/null && dpkg-query -W librocksdb-dev > /dev/null 2>&1; } || |
| { command -v brew > /dev/null && brew list --versions rocksdb > /dev/null 2>&1; }; then |
| DISTRO_ROCKSDB=1 |
| fi |
| |
| ROCKSDB_PREFIX_USED= |
| ROCKSDB_SHARED_BUILT=0 |
| if [ -n "$DISTRO_ROCKSDB" ] && [ -z "$ROCKSDB_VERSION" ]; then |
| if [ $STATIC = 1 ] && ! static_rocksdb_of_the_distribution; then |
| step "rocksdb: building the static library, same version as the distribution" |
| "$DEREKO2VEC_SRC/scripts/build-rocksdb.sh" --static-only -p "$ROCKSDB_PREFIX" |
| ROCKSDB_PREFIX_USED=$ROCKSDB_PREFIX |
| else |
| step "rocksdb: the one of the distribution is used" |
| fi |
| else |
| if [ -z "$DISTRO_ROCKSDB" ]; then |
| step "rocksdb: no rocksdb package - building the shared and the static library" |
| else |
| step "rocksdb: building version $ROCKSDB_VERSION as asked" |
| fi |
| "$DEREKO2VEC_SRC/scripts/build-rocksdb.sh" -p "$ROCKSDB_PREFIX" ${ROCKSDB_VERSION:+-v "$ROCKSDB_VERSION"} |
| ROCKSDB_PREFIX_USED=$ROCKSDB_PREFIX |
| ROCKSDB_SHARED_BUILT=1 |
| fi |
| |
| # Programs that link the shared rocksdb of the prefix have to find it. Only a |
| # system-wide install can register the directory; for one below $HOME the |
| # programs carry the rpath of the libraries they were linked against. |
| if [ $ROCKSDB_SHARED_BUILT = 1 ] && [ "$(uname)" = Linux ] && [ -n "$SUDO" ]; then |
| RDB_LIBDIR=$(dirname "$(find "$ROCKSDB_PREFIX_USED" -name 'librocksdb.so' | head -1)") |
| echo "$RDB_LIBDIR" | $SUDO tee /etc/ld.so.conf.d/rocksdb-prefix.conf > /dev/null |
| $SUDO ldconfig |
| fi |
| |
| # --- 3. collocatordb --------------------------------------------------------- |
| step "collocatordb" |
| WORK=$(mktemp -d) |
| trap 'rc=$?; if [ $rc = 0 ]; then rm -rf "$WORK"; else echo "Keeping $WORK for a look at what went wrong" >&2; fi' EXIT |
| git clone "https://korap.ids-mannheim.de/gerrit/ids-kl/collocatordb" "$WORK/collocatordb" |
| CDB_PREFIXES=$ROCKSDB_PREFIX_USED |
| if [ -n "$BREW_PREFIX" ]; then |
| CDB_PREFIXES=${CDB_PREFIXES:+$CDB_PREFIXES;}$BREW_PREFIX |
| fi |
| cmake -S "$WORK/collocatordb" -B "$WORK/collocatordb/build" \ |
| -DCMAKE_INSTALL_PREFIX="$PREFIX" \ |
| ${CDB_PREFIXES:+-DCMAKE_PREFIX_PATH=$CDB_PREFIXES} |
| cmake --build "$WORK/collocatordb/build" -j "$JOBS" |
| $SUDO cmake --install "$WORK/collocatordb/build" |
| if [ "$(uname)" = Linux ] && [ -n "$SUDO" ]; then |
| $SUDO ldconfig |
| fi |
| if [ $SKIP_TESTS = 0 ]; then |
| ctest --test-dir "$WORK/collocatordb/build" --output-on-failure |
| fi |
| |
| # --- 4. dereko2vec ----------------------------------------------------------- |
| step "dereko2vec" |
| # A stale build directory would silently keep the collocatordb and rocksdb it |
| # was once configured with, and an install manifest of a root install breaks a |
| # later install without sudo. Start over, the build takes a minute. |
| rm -rf "$DEREKO2VEC_SRC/build" |
| CMAKE_PREFIXES=$PREFIX |
| if [ -n "$ROCKSDB_PREFIX_USED" ]; then |
| CMAKE_PREFIXES="$CMAKE_PREFIXES;$ROCKSDB_PREFIX_USED" |
| fi |
| if [ -n "$BREW_PREFIX" ]; then |
| CMAKE_PREFIXES="$CMAKE_PREFIXES;$BREW_PREFIX" |
| fi |
| STATIC_FLAG= |
| if [ $STATIC_BINARY = 1 ]; then |
| STATIC_FLAG=-DSTATIC_DEREKO2VEC=ON |
| elif [ $STATIC = 1 ]; then |
| STATIC_FLAG=-DSTATIC_ROCKSDB=ON |
| fi |
| cmake -S "$DEREKO2VEC_SRC" -B "$DEREKO2VEC_SRC/build" \ |
| -DCMAKE_INSTALL_PREFIX="$PREFIX" \ |
| -DCMAKE_PREFIX_PATH="$CMAKE_PREFIXES" \ |
| $STATIC_FLAG |
| cmake --build "$DEREKO2VEC_SRC/build" -j "$JOBS" |
| if [ $SKIP_TESTS = 0 ]; then |
| ctest --test-dir "$DEREKO2VEC_SRC/build" --output-on-failure |
| fi |
| $SUDO cmake --install "$DEREKO2VEC_SRC/build" |
| |
| step "Done" |
| echo "dereko2vec and vecs2mmap are in $PREFIX/bin" |