.gitlab-ci.yml

stages:
  - test
  - build
  - deploy

variables:
  VID: $VID
  DEBIAN_FRONTEND: noninteractive
  APT_CACHE_DIR: apt-cache
  SRC_DIR: test/resources/DNB
  YEARS: "13 18"
  SAXON_HOME: "$CI_PROJECT_DIR/lib"

build-and-test-i5:
  stage: test
  image: rocker/verse
  artifacts:
    paths:
      - "target/dnb??.i5.xml"
      - "target/*.zip"
  cache:
    - key: DNB4KorAP
      paths:
          - apt-cache/

  before_script:
    - source `find .. -name section_helper.sh`
    - mkdir -p "$SAXON_HOME"
    - start_section saxon_license_debug "Preparing Saxon license"
    - echo "SAXON_HOME=$SAXON_HOME"
    - echo "SAXON_LICENSE=$SAXON_LICENSE"
    - ls -l "$SAXON_HOME" || true
    - ls -l "$SAXON_LICENSE" || true
    - test -r "$SAXON_LICENSE" && echo "SAXON_LICENSE readable: yes" || echo "SAXON_LICENSE readable: no"
    - |
      if [ -r "$SAXON_LICENSE" ]; then
        cp -v "$SAXON_LICENSE" "$SAXON_HOME/saxon-license.lic" && ls -l "$SAXON_HOME/saxon-license.lic"
      else
        echo "No readable SAXON_LICENSE file variable present"
      fi
    - end_section saxon_license_debug
    - start_section install_linux_packages "Installing missing Linux packages"
    - apt-get -o dir::cache::archives="$APT_CACHE_DIR" update
    - apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y libxml2-utils curl openjdk-17-jre-headless xmlstarlet
    - end_section install_linux_packages
    - start_section install_models "Installing models"
    - make models/dereko_domains_s.classifier
    - end_section install_models

  script:
    - start_section checking_i5 "Building and testing I5 files"
    - make -j $(nproc) test
    - end_section checking_i5


build-index:
  image: perl:5.38
  services:
    - docker:dind
  stage: build
  rules:
    - if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_JOB_MANUAL == "true"'
  variables:
      VID: $CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA
      PERL_LOCAL_LIB_ROOT: ./perl5
      PERL5LIB: ./perl5/lib/perl5
  cache:
    - key: DNB4KorAP
      paths:
          - $PERL_LOCAL_LIB_ROOT
          - perl5/
          - apt-cache/
    - key:
        files:
          - target/dnb18.i5.xml
      paths:
          - target/dnb18.zip
          - target/dnb18.tree_tagger.zip
          - target/dnb18.marmot-malt.zip
          - target/dnb18.spacy.zip
          - target/dnb18.krill.tar

  before_script:
    - source `find .. -name section_helper.sh`
    - mkdir -p "$SAXON_HOME"
    - start_section saxon_license_debug "Preparing Saxon license"
    - echo "SAXON_HOME=$SAXON_HOME"
    - echo "SAXON_LICENSE=$SAXON_LICENSE"
    - ls -l "$SAXON_HOME" || true
    - ls -l "$SAXON_LICENSE" || true
    - test -r "$SAXON_LICENSE" && echo "SAXON_LICENSE readable: yes" || echo "SAXON_LICENSE readable: no"
    - |
      if [ -r "$SAXON_LICENSE" ]; then
        cp -v "$SAXON_LICENSE" "$SAXON_HOME/saxon-license.lic" && ls -l "$SAXON_HOME/saxon-license.lic"
      else
        echo "No readable SAXON_LICENSE file variable present"
      fi
    - end_section saxon_license_debug
    - start_section install_linux_packages "Installing missing Linux packages"
    - mkdir -pv $APT_CACHE_DIR
    - apt-get -o dir::cache::archives="$APT_CACHE_DIR" update
    - apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y rsync pv jq curl openjdk-17-jre-headless docker.io
    - end_section install_linux_packages

    - start_section install_tree_tagger "Installing TreeTagger"
    - docker image inspect korap/conllu2treetagger:latest >/dev/null 2>&1 || curl -Ls 'https://gitlab.ids-mannheim.de/KorAP/CoNLL-U-Treetagger/-/jobs/artifacts/master/raw/conllu2treetagger.xz?job=build-docker-image' | docker load
    - end_section install_tree_tagger

    - start_section install_spacy "Installing spaCy"
    - docker image inspect korap/conllu2spacy:latest >/dev/null 2>&1 || curl -Ls https://corpora.ids-mannheim.de/tools/conllu2spacy.tar.xz | docker load
    - end_section install_spacy

    - start_section install_perl_packages "Installing missing Perl packages"
    - curl -L https://cpanmin.us | perl - App::cpanminus
    - cpanm -n -l $PERL_LOCAL_LIB_ROOT File::ShareDir::Install https://github.com/KorAP/KorAP-XML-TEI.git
    - cpanm -n -l $PERL_LOCAL_LIB_ROOT https://github.com/KorAP/KorAP-XML-Krill.git
    - cpanm -n -l $PERL_LOCAL_LIB_ROOT https://github.com/KorAP/KorAP-XML-CoNLL-U.git
    - end_section install_perl_packages

  script:
    - export PATH=$PERL_LOCAL_LIB_ROOT/bin:$PATH
    - start_section building_index "Building index"
    - touch target/*.zip # ignore timestamps in make
    - sleep 1
    - touch target/*.*.zip
    - MAX_THREADS=2 make -j $(nproc) target/dnb.index.tar.xz
    - end_section building_krill
  artifacts:
    paths:
      - target/*.zip
      - target/dnb18.index.tar.xz

deploy:
  stage: deploy
  dependencies:
    - "build-index"
  rules:
    - if: '$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_JOB_MANUAL == "true"'
  image: rocker/verse
  before_script:
    - source `find .. -name section_helper.sh`
    - mkdir -p "$SAXON_HOME"
    - start_section saxon_license_debug "Preparing Saxon license"
    - echo "SAXON_HOME=$SAXON_HOME"
    - echo "SAXON_LICENSE=$SAXON_LICENSE"
    - ls -l "$SAXON_HOME" || true
    - ls -l "$SAXON_LICENSE" || true
    - test -r "$SAXON_LICENSE" && echo "SAXON_LICENSE readable: yes" || echo "SAXON_LICENSE readable: no"
    - |
      if [ -r "$SAXON_LICENSE" ]; then
        cp -v "$SAXON_LICENSE" "$SAXON_HOME/saxon-license.lic" && ls -l "$SAXON_HOME/saxon-license.lic"
      else
        echo "No readable SAXON_LICENSE file variable present"
      fi
    - end_section saxon_license_debug
    - start_section setup_ssh "Setting up SSH"
    - apt-get update
    - apt-get install  -y rsync openssh-client
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh
    - eval $(ssh-agent -s)
    - chmod 400 $SSH_PRIVATE_KEY
    - ssh-add $SSH_PRIVATE_KEY
    - end_section setup_ssh
  script:
    - start_section korapxmlu "Uploading index to KorAP4DNB instance"
    - touch target/dnb18.index.tar.xz
    - make deploy
    - end_section deploy