CI: modernize the docker slimming
Follows what Kalamar does. slimtoolkit became mintoolkit, and the invocation
had to change anyway: the tool has no --in-container option anymore, so the
job would have stopped at an unknown flag. The download of the slim binary
goes away with it, it was never used, the containerized tool was called
instead.
FF_NETWORK_PER_BUILD and the dns option of the dind service let the mint
container reach the container it probes, as in Kalamar.
The probe waited 10 seconds for the server, which is not enough: the image
compiles its Inline C at startup and then loads the model, and only answers
after 21 seconds, measured with the image of this commit. It waits 60 now.
The image tarball carries the version in its name, as in Kalamar, and
upload-docker was adapted accordingly. It only runs for v tags now and has to
be started by hand. Before it also ran for every master commit, where
CI_COMMIT_TAG is empty and it would have pushed a tag-less image.
The .dockerignore did not catch .gitlab-ci-local, it was written with a dot
instead of a hyphen. Everybody who runs gitlab-ci-local and then builds the
image locally packed the whole CI scratch directory into it, 549 MB in my
case, including job logs, which contain the CI variables. Release tarballs are
ignored now, too. This does not affect images built in CI, whose checkout is
clean.
Verified locally by running the whole chain: the large image builds, mint
minifies it from 1.3 GB to 905 MB with the http probe answering 200, and the
image from Dockerfile.remove-example-data serves neighbours, collocators and
getVecsByRanks correctly with the compilers removed.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Change-Id: I0014c8d2e4f4cee667b0d1c5016f037c9aa44743
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b4ada36..cfee89e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -95,8 +95,12 @@
build-docker:
image: docker:latest
+ variables:
+ # the mint container has to reach the container it probes
+ FF_NETWORK_PER_BUILD: "true"
services:
- - docker:dind
+ - name: docker:dind
+ command: [--dns=127.0.0.11]
rules:
- if: $CI_COMMIT_TAG =~ /.+/
variables:
@@ -109,34 +113,41 @@
- apk update
- apk add --no-cache git
script:
+ # the http probe needs a model to serve, so keep the example models in the
+ # large image and remove them further down with Dockerfile.remove-example-data
- sed -i -e "s/RUN rm -rf example-models//" Dockerfile
- docker build -f Dockerfile -t idscorpuslinguistics/derekovecs:$VID-large .
- - wget https://github.com/slimtoolkit/slim/releases/latest/download/dist_linux.tar.gz
- - tar zxf dist_linux.tar.gz
- - export PATH=$(pwd)/dist_linux:$PATH
- - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock dslim/slim --in-container build --http-probe-start-wait 10 --include-workdir --include-path /usr/local --env MOJO_CONFIG=/derekovecs/example-models/example-docker.conf --tag idscorpuslinguistics/derekovecs:tmp idscorpuslinguistics/derekovecs:$VID-large
+ - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock mintoolkit/mint --crt-api-version 1.46 build --http-probe=true --http-probe-start-wait 60 --include-workdir=true --include-path=/usr/local --env MOJO_CONFIG=/derekovecs/example-models/example-docker.conf --tag idscorpuslinguistics/derekovecs:tmp idscorpuslinguistics/derekovecs:$VID-large || true
- docker build -f Dockerfile.remove-example-data --tag idscorpuslinguistics/derekovecs:latest --tag idscorpuslinguistics/derekovecs:$VID .
- - docker save idscorpuslinguistics/derekovecs:$VID | xz -T0 -M16G -9 > derekovecs.tar.xz
+ - ARTIFACT=derekovecs-${VID}.tar.xz
+ - docker save idscorpuslinguistics/derekovecs:$VID | xz -T0 -M16G -9 > "$ARTIFACT"
artifacts:
paths:
- - derekovecs.tar.xz
+ - derekovecs-*.tar.xz
upload-docker:
- only:
- - master
- - tags
- variables:
- VID: $CI_COMMIT_TAG
stage: deploy
image: docker:latest
services:
- - docker:dind
- script:
- - unxz -c derekovecs.tar.xz | docker load
- - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin
- - docker push idscorpuslinguistics/derekovecs:$VID
- - docker push idscorpuslinguistics/derekovecs
+ - name: docker:dind
+ command: [--dns=127.0.0.11]
+ needs:
+ - job: build-docker
+ artifacts: true
dependencies:
- build-docker
+ rules:
+ - if: $CI_COMMIT_TAG =~ /^v.+/
+ when: manual
+ - when: never
+ script:
+ - apk update
+ - apk add --no-cache xz
+ - ARTIFACT=derekovecs-${CI_COMMIT_TAG}.tar.xz
+ - xz -d -c "$ARTIFACT" | docker load
+ - echo "$DOCKER_HUB_PASSWORD" | docker login -u "$DOCKER_HUB_USERNAME" --password-stdin
+ - docker tag idscorpuslinguistics/derekovecs:$CI_COMMIT_TAG idscorpuslinguistics/derekovecs:latest
+ - docker push idscorpuslinguistics/derekovecs:$CI_COMMIT_TAG
+ - docker push idscorpuslinguistics/derekovecs:latest
environment:
name: production