From 3b4d5bb5a11a210568c2bef87cd06b4142f839e9 Mon Sep 17 00:00:00 2001
From: George Sedov <george.sedov@desy.de>
Date: Tue, 8 Feb 2022 19:33:31 +0100
Subject: [PATCH] build docker for services
---
.gitlab-ci.yml | 41 ++++++++++++++++++++++------
receiver/docker/Dockerfile | 10 +++----
receiver/docker/install_libfabric.sh | 14 ----------
receiver/docker/install_rdkafka.sh | 15 ----------
4 files changed, 37 insertions(+), 43 deletions(-)
delete mode 100755 receiver/docker/install_libfabric.sh
delete mode 100755 receiver/docker/install_rdkafka.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index beaa355dc..357265880 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,7 +25,7 @@ build-services-linux-debug:
tags:
- kubernetes-executor
rules:
- - when: always
+ - when: never
artifacts:
paths:
- build
@@ -52,7 +52,7 @@ test-services-linux-debug:
tags:
- kubernetes-executor
rules:
- - when: always
+ - when: never
dependencies:
- build-services-linux-debug
artifacts:
@@ -69,8 +69,12 @@ build-services-linux-release:
script:
- mkdir $CI_PROJECT_DIR/build
- cd $CI_PROJECT_DIR/build
- - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_EXAMPLES=ON -DBUILD_CONSUMER_TOOLS=ON -DBUILD_BROKER=ON -DBUILD_INTEGRATION_TESTS=ON -DENABLE_LIBFABRIC=ON -DBUILD_CPP_DOCS=ON -DBUILD_EVENT_MONITOR_PRODUCER=ON -DSTOP_BUILD_ON_WARNINGS=ON -DENABLE_NEW_RECEIVER_MONITORING=ON ..
+ - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_EXAMPLES=ON -DBUILD_CONSUMER_TOOLS=ON -DBUILD_BROKER=ON -DBUILD_INTEGRATION_TESTS=ON -DENABLE_LIBFABRIC=ON -DBUILD_CPP_DOCS=ON -DBUILD_EVENT_MONITOR_PRODUCER=ON -DSTOP_BUILD_ON_WARNINGS=ON -DENABLE_NEW_RECEIVER_MONITORING=ON ..
- make -j 4
+ # special treatment for receiver dependencies.
+ - KAFKA_LIBS=$(dpkg -L librdkafka-dev | grep "\.so")
+ - FABRIC_LIBS=$(dpkg -L libfabric-dev | grep "\.so")
+ - printf "${KAFKA_LIBS}\n${FABRIC_LIBS}" > $CI_PROJECT_DIR/build/receiver/docker/dependencies
tags:
- kubernetes-executor
rules:
@@ -79,10 +83,29 @@ build-services-linux-release:
paths:
- build
exclude:
- - build/**/CMakeFiles/**/*.o
- - build/**/CMakeFiles/**/*.gcno
- - build/**/CMakeFiles/**/*.gcda
- - build/**/CMakeFiles/**/*.objlib
- - build/**/CMakeFiles/**/*.includecache
+ - build/**/CMakeFiles/**/*
- build/**/*.a
- expire_in: 3 hrs
\ No newline at end of file
+ expire_in: 3 hrs
+
+build-services-docker:
+ image:
+ name: gcr.io/kaniko-project/executor:latest
+ entrypoint: [""]
+ stage: deploy
+ script:
+ - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+ - services="broker authorizer discovery receiver file-transfer"
+ - >
+ for service in $services
+ do
+ service_dir=${service//"-"/"_"}
+ # read the dependency list if exists
+ [[ -f $CI_PROJECT_DIR/${service_dir}/docker/dependencies ]] && DEPENDENCY_LIBS=$(<$CI_PROJECT_DIR/${service_dir}/docker/dependencies) || DEPENDENCY_LIBS=''
+ IFS='' /kaniko/executor --cache=false --context $CI_PROJECT_DIR/${service_dir}/docker --dockerfile $CI_PROJECT_DIR/${service_dir}/docker/Dockerfile --destination $CI_REGISTRY_IMAGE/asapo-dev-${service}:${CI_COMMIT_SHORT_SHA} --build-args "CI_REGISTRY_IMAGE='${CI_REGISTRY_IMAGE}'" --build-args "DEPENDENCY_LIBS='${DEPENDENCY_LIBS}'"
+ done
+ tags:
+ - kubernetes-executor
+ rules:
+ - when: always
+ dependencies:
+ - build-services-linux-release
diff --git a/receiver/docker/Dockerfile b/receiver/docker/Dockerfile
index 00df9b405..d9ef27501 100644
--- a/receiver/docker/Dockerfile
+++ b/receiver/docker/Dockerfile
@@ -1,7 +1,7 @@
-FROM ubuntu:18.04
-ADD receiver /
-ADD install_libfabric.sh install_libfabric.sh
-ADD install_rdkafka.sh install_rdkakfa.sh
-RUN apt update && ./install_libfabric.sh && ./install_rdkakfa.sh
+FROM $CI_REGISTRY_IMAGE/asapo-services-linux-build-env:latest
+COPY $DEPENDENCY_LIBS /
+
+FROM busybox:glibc
+COPY receiver /
CMD ["/receiver","/var/lib/receiver/config.json"]
diff --git a/receiver/docker/install_libfabric.sh b/receiver/docker/install_libfabric.sh
deleted file mode 100755
index c2842b87d..000000000
--- a/receiver/docker/install_libfabric.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-apt install -y wget autoconf libtool make librdmacm-dev rdma-core
-wget https://github.com/ofiwg/libfabric/archive/v1.11.0.tar.gz
-tar xzf v1.11.0.tar.gz
-cd libfabric-1.11.0
-./autogen.sh
-./configure
-make
-make install
-ldconfig
-cd -
-rm -rf libfabric-1.11.0
-rm v1.11.0.tar.gz
diff --git a/receiver/docker/install_rdkafka.sh b/receiver/docker/install_rdkafka.sh
deleted file mode 100755
index e1bb3c207..000000000
--- a/receiver/docker/install_rdkafka.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-apt install -y wget build-essential autoconf libtool make zlib1g-dev libzstd-dev
-
-wget https://github.com/edenhill/librdkafka/archive/refs/tags/v1.8.2.tar.gz
-tar xzf v1.8.2.tar.gz
-cd librdkafka-1.8.2
-
-./configure --enable-zlib --enable-zstd --disable-lz4 --disable-lz4-ext --disable-ssl --disable-gssapi --disable-sasl
-make -j 4
-make install
-ldconfig
-cd -
-rm -rf librdkafka-1.8.2
-rm v1.8.2.tar.gz
\ No newline at end of file
--
GitLab