diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..16e929675b49380e31919993b3114c251f5c1dc2 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,6 @@ +# Correct build order + +## For Cluster + +1. `nomad_consul_docker` for `$REPO/asapo-nomad-cluster` +2. `asapo_services` for `$REPO/asapo-cluster` diff --git a/deploy/_docker_vars.sh b/deploy/_docker_vars.sh new file mode 100755 index 0000000000000000000000000000000000000000..a501eb827bdf4df10de04b19c204208f1d619c0a --- /dev/null +++ b/deploy/_docker_vars.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +if [[ -z "$ASAPO_DOCKER_REPOSITORY" ]]; then + ASAPO_DOCKER_REPOSITORY=yakser +fi + +if [[ -z "$ASAPO_DOCKER_DO_PUSH" ]]; then + ASAPO_DOCKER_DO_PUSH=YES +fi diff --git a/deploy/asapo_orchestration_docker/Dockerfile b/deploy/asapo_orchestration_docker/Dockerfile.template similarity index 64% rename from deploy/asapo_orchestration_docker/Dockerfile rename to deploy/asapo_orchestration_docker/Dockerfile.template index 87e56865ceaab1f03cedc23c06b4ef18a52b0e2b..a3edfe710adba7d53f5b24e6d80f9e100c046364 100644 --- a/deploy/asapo_orchestration_docker/Dockerfile +++ b/deploy/asapo_orchestration_docker/Dockerfile.template @@ -1,4 +1,4 @@ -FROM yakser/asapo-nomad-cluster +FROM $ASAPO_DOCKER_REPOSITORY/asapo-nomad-cluster MAINTAINER DESY IT diff --git a/deploy/asapo_orchestration_docker/build_image.sh b/deploy/asapo_orchestration_docker/build_image.sh index f375d647bd9799f5a229c9d45ebeefef4cca1398..abf4b08ea26fab898ab014a9e72dbeeb377b562d 100755 --- a/deploy/asapo_orchestration_docker/build_image.sh +++ b/deploy/asapo_orchestration_docker/build_image.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash -docker build -t yakser/asapo-orc . -docker push yakser/asapo-orc +. ../_docker_vars.sh + +sed "s/\$ASAPO_DOCKER_REPOSITORY/$ASAPO_DOCKER_REPOSITORY/" Dockerfile.template | docker build -t $ASAPO_DOCKER_REPOSITORY/asapo-orc . -f - + +if [ $ASAPO_DOCKER_DO_PUSH = "YES" ]; then + docker push $ASAPO_DOCKER_REPOSITORY/asapo-orc +fi diff --git a/deploy/asapo_services/Dockerfile b/deploy/asapo_services/Dockerfile.template similarity index 72% rename from deploy/asapo_services/Dockerfile rename to deploy/asapo_services/Dockerfile.template index 1ea67d704f343044c29e1c18d1dbf08ff9ac4792..8a01f49f14073b06cf2afa2bed4983776fcb90f7 100644 --- a/deploy/asapo_services/Dockerfile +++ b/deploy/asapo_services/Dockerfile.template @@ -1,4 +1,4 @@ -FROM yakser/asapo-nomad-cluster +FROM $ASAPO_DOCKER_REPOSITORY/asapo-nomad-cluster MAINTAINER DESY IT diff --git a/deploy/asapo_services/asap3.tfvars b/deploy/asapo_services/asap3.tfvars index c3c5434b8828865ead3f552ff91240392cedabe8..4734c46e645af4132673e235c5f0f5882df0cb7b 100644 --- a/deploy/asapo_services/asap3.tfvars +++ b/deploy/asapo_services/asap3.tfvars @@ -1,11 +1,11 @@ elk_logs = true +asapo_docker_repository = "cpatzke" asapo_imagename_suffix = "" asapo_image_tag = "" influxdb_version="1.8.4" - service_dir="/gpfs/asapo/shared/service_dir" online_dir="/beamline" offline_dir="/asap3" @@ -31,5 +31,3 @@ discovery_total_memory_size = 512 n_receivers = 1 n_brokers = 1 n_fts = 1 - - diff --git a/deploy/asapo_services/build_image.sh b/deploy/asapo_services/build_image.sh index 69b3e638d370540da56478932995c0ca570397a3..25c8383583d6b65f63bf04aed7917fae46ad036b 100755 --- a/deploy/asapo_services/build_image.sh +++ b/deploy/asapo_services/build_image.sh @@ -1,4 +1,11 @@ #!/usr/bin/env bash -docker build -t yakser/asapo-cluster . -docker push yakser/asapo-cluster +set -e + +. ../_docker_vars.sh + +sed "s/\$ASAPO_DOCKER_REPOSITORY/$ASAPO_DOCKER_REPOSITORY/" Dockerfile.template | docker build -t $ASAPO_DOCKER_REPOSITORY/asapo-cluster . -f - + +if [ $ASAPO_DOCKER_DO_PUSH = "YES" ]; then + docker push $ASAPO_DOCKER_REPOSITORY/asapo-cluster +fi diff --git a/deploy/asapo_services/run.sh b/deploy/asapo_services/run.sh index 0a4f38fc588f203f46a55db165818d9a4645406b..c215580a2de41f0e5729e197d22a501f8724c080 100755 --- a/deploy/asapo_services/run.sh +++ b/deploy/asapo_services/run.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +set -e + +. ../_docker_vars.sh + NOMAD_ALLOC_HOST_SHARED=/var/tmp/asapo/container_host_shared/nomad_alloc SERVICE_DATA_CLUSTER_SHARED=/var/tmp/asapo/asapo_cluster_shared/service_data DATA_GLOBAL_SHARED=/var/tmp/asapo/global_shared/data @@ -18,17 +22,19 @@ ASAPO_USER=`id -u`:`id -g` ASAPO_VAR_FILE=`pwd`/asapo_overwrite_vars.tfvars ACL_ENABLED=true +echo creating $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED cd $SERVICE_DATA_CLUSTER_SHARED -mkdir esdatadir fluentd grafana influxdb mongodb +mkdir -p esdatadir fluentd grafana influxdb mongodb chmod 777 * mmc=`cat /proc/sys/vm/max_map_count` if (( mmc < 262144 )); then echo increase max_map_count - needed for elasticsearch +echo "Just run 'sudo sysctl -w vm.max_map_count=262144' to temporarily set it (until next restart)" exit 1 fi @@ -58,5 +64,5 @@ docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \ -e ACL_ENABLED=$ACL_ENABLED \ -e SERVER_ADRESSES=$SERVER_ADRESSES \ -e N_SERVERS=$N_SERVERS \ ---name asapo --net=host -d yakser/asapo-cluster +--name asapo --net host -d $ASAPO_DOCKER_REPOSITORY/asapo-cluster diff --git a/deploy/asapo_services/run_maxwell.sh b/deploy/asapo_services/run_maxwell.sh index d2a74ed1f22c7cd92adf9e18ce672383dc5f7bd4..b9ec305fe877a4ae3869c1961ce2239ac4eca6d5 100755 --- a/deploy/asapo_services/run_maxwell.sh +++ b/deploy/asapo_services/run_maxwell.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash -IMAGE=yakser/asapo-cluster:20.06.3 +set -e + +. ../_docker_vars.sh + +IMAGE=$ASAPO_DOCKER_REPOSITORY/asapo-cluster:20.06.3 #folders NOMAD_ALLOC_HOST_SHARED=/var/tmp/asapo/container_host_shared/nomad_alloc diff --git a/deploy/asapo_services/scripts/asapo-brokers.nmd.tpl b/deploy/asapo_services/scripts/asapo-brokers.nmd.tpl index 14f5eaf5baf9587059c4b644d3dbfea8a29eb137..375620fec4e96da4fa44fdcc1439570e48289ed2 100644 --- a/deploy/asapo_services/scripts/asapo-brokers.nmd.tpl +++ b/deploy/asapo_services/scripts/asapo-brokers.nmd.tpl @@ -29,7 +29,7 @@ job "asapo-brokers" { network_mode = "host" security_opt = ["no-new-privileges"] userns_mode = "host" - image = "yakser/asapo-broker${image_suffix}" + image = "${docker_repository}/asapo-broker${image_suffix}" force_pull = ${force_pull_images} volumes = ["local/config.json:/var/lib/broker/config.json"] %{ if ! nomad_logs } diff --git a/deploy/asapo_services/scripts/asapo-fts.nmd.tpl b/deploy/asapo_services/scripts/asapo-fts.nmd.tpl index 5f4d98278ccb419e54ce9465895e47ea22c92bbb..29bb0def18a719c798655208ccd9d7d9b0ac8e41 100644 --- a/deploy/asapo_services/scripts/asapo-fts.nmd.tpl +++ b/deploy/asapo_services/scripts/asapo-fts.nmd.tpl @@ -29,7 +29,7 @@ job "asapo-file-transfer" { network_mode = "host" security_opt = ["no-new-privileges"] userns_mode = "host" - image = "yakser/asapo-file-transfer${image_suffix}" + image = "${docker_repository}/asapo-file-transfer${image_suffix}" force_pull = ${force_pull_images} volumes = ["local/config.json:/var/lib/file_transfer/config.json", "${offline_dir}:${offline_dir}", diff --git a/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl b/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl index 2096021f5cc99da4edcfbecafaf5c4a2343855d3..04de3245f464690ed984341a5423adc961dbc140 100644 --- a/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl +++ b/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl @@ -34,7 +34,7 @@ job "asapo-receivers" { security_opt = ["no-new-privileges"] userns_mode = "host" privileged = true - image = "yakser/asapo-receiver${image_suffix}" + image = "${docker_repository}/asapo-receiver${image_suffix}" force_pull = ${force_pull_images} volumes = ["local/config.json:/var/lib/receiver/config.json", "${offline_dir}:${offline_dir}", diff --git a/deploy/asapo_services/scripts/asapo-services.nmd.tpl b/deploy/asapo_services/scripts/asapo-services.nmd.tpl index 9cb6298aa08ab6edf9b2f3897c21ead5a7dd9514..477b03c19e321f3170af96ac984a040f48c2ec05 100644 --- a/deploy/asapo_services/scripts/asapo-services.nmd.tpl +++ b/deploy/asapo_services/scripts/asapo-services.nmd.tpl @@ -18,7 +18,7 @@ job "asapo-services" { network_mode = "host" security_opt = ["no-new-privileges"] userns_mode = "host" - image = "yakser/asapo-authorizer${image_suffix}" + image = "${docker_repository}/asapo-authorizer${image_suffix}" force_pull = ${force_pull_images} volumes = ["local/config.json:/var/lib/authorizer/config.json", "${offline_dir}:${offline_dir}", @@ -95,7 +95,7 @@ job "asapo-services" { network_mode = "host" security_opt = ["no-new-privileges"] userns_mode = "host" - image = "yakser/asapo-discovery${image_suffix}" + image = "${docker_repository}/asapo-discovery${image_suffix}" force_pull = ${force_pull_images} volumes = ["local/config.json:/var/lib/discovery/config.json"] %{ if ! nomad_logs } diff --git a/deploy/asapo_services/scripts/templates.tf b/deploy/asapo_services/scripts/templates.tf index e5dae73a5b2e39f2550df7de744e1142ed60d962..30789475e5e7f5a1738860dd4697ea4f79013fe8 100644 --- a/deploy/asapo_services/scripts/templates.tf +++ b/deploy/asapo_services/scripts/templates.tf @@ -22,6 +22,7 @@ data "template_file" "asapo_services" { scripts_dir = "${var.job_scripts_dir}" online_dir = "${var.online_dir}" offline_dir = "${var.offline_dir}" + docker_repository = "${var.asapo_docker_repository}" image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}" nomad_logs = "${var.nomad_logs}" authorizer_total_memory_size = "${var.authorizer_total_memory_size}" @@ -39,6 +40,7 @@ data "template_file" "asapo_receivers" { scripts_dir = "${var.job_scripts_dir}" online_dir = "${var.online_dir}" offline_dir = "${var.offline_dir}" + docker_repository = "${var.asapo_docker_repository}" image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}" nomad_logs = "${var.nomad_logs}" receiver_total_memory_size = "${var.receiver_total_memory_size}" @@ -57,6 +59,7 @@ data "template_file" "asapo_brokers" { template = "${file("${var.job_scripts_dir}/asapo-brokers.nmd.tpl")}" vars = { scripts_dir = "${var.job_scripts_dir}" + docker_repository = "${var.asapo_docker_repository}" image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}" nomad_logs = "${var.nomad_logs}" asapo_user = "${var.asapo_user}" @@ -73,6 +76,7 @@ data "template_file" "asapo_fts" { scripts_dir = "${var.job_scripts_dir}" online_dir = "${var.online_dir}" offline_dir = "${var.offline_dir}" + docker_repository = "${var.asapo_docker_repository}" image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}" nomad_logs = "${var.nomad_logs}" asapo_user = "${var.asapo_user}" diff --git a/deploy/build_env/centos/build_el7.sh b/deploy/build_env/centos/build_el7.sh index b64109e08d9762d4d51de0b406290c1ebb1b8c7a..9784d78c0b0e1e8a6bf6eb364f15a94810090c38 100755 --- a/deploy/build_env/centos/build_el7.sh +++ b/deploy/build_env/centos/build_el7.sh @@ -2,5 +2,10 @@ set -e -docker build -t yakser/asapo-env:centos7.9.2009 -f Dockerfile.7.9.2009 . -docker push yakser/asapo-env:centos7.9.2009 +. ../../_docker_vars.sh + +docker build -t $ASAPO_DOCKER_REPOSITORY/asapo-env:centos7.9.2009 -f Dockerfile.7.9.2009 . + +if [ $ASAPO_DOCKER_DO_PUSH = "YES" ]; then + docker push $ASAPO_DOCKER_REPOSITORY/asapo-env:centos7.9.2009 +fi diff --git a/deploy/build_env/centos/build_el8.sh b/deploy/build_env/centos/build_el8.sh index 1f273ad2a6653ca2aa7aa7984e32267ff8a0dc63..cd2277bc8ac4acf3cefff66559e7dd92b726bd28 100755 --- a/deploy/build_env/centos/build_el8.sh +++ b/deploy/build_env/centos/build_el8.sh @@ -2,5 +2,10 @@ set -e -docker build -t yakser/asapo-env:centos8.3.2011 -f Dockerfile.8.3.2011 . -docker push yakser/asapo-env:centos8.3.2011 +. ../../_docker_vars.sh + +docker build -t $ASAPO_DOCKER_REPOSITORY/asapo-env:centos8.3.2011 -f Dockerfile.8.3.2011 . + +if [ $ASAPO_DOCKER_DO_PUSH = "YES" ]; then + docker push $ASAPO_DOCKER_REPOSITORY/asapo-env:centos8.3.2011 +fi diff --git a/deploy/build_env/debians/build_images.sh b/deploy/build_env/debians/build_images.sh index 9ed79e5de2f3488e2034463d4e8f538337d92684..179b8d57161324f556aa478756d8b37306c6d951 100755 --- a/deploy/build_env/debians/build_images.sh +++ b/deploy/build_env/debians/build_images.sh @@ -1,10 +1,16 @@ #!/usr/bin/env bash + set -e +. ../../_docker_vars.sh + vers="ubuntu18.04 ubuntu16.04 debian9.13 debian10.7" for ver in $vers do - docker build -t yakser/asapo-env:${ver} -f Dockerfile_${ver} --build-arg OS=${ver} . - docker push yakser/asapo-env:${ver} + docker build -t $ASAPO_DOCKER_REPOSITORY/asapo-env:${ver} -f Dockerfile_${ver} --build-arg OS=${ver} . + + if [ $ASAPO_DOCKER_DO_PUSH = "YES" ]; then + docker push $ASAPO_DOCKER_REPOSITORY/asapo-env:${ver} + fi done diff --git a/deploy/build_env/manylinux2010/build.sh b/deploy/build_env/manylinux2010/build.sh index 0fee6f1faebfef1a0776629a1db85c316d46a31b..c800bbace4d2122a3aef0a91e3505121b90fb554 100755 --- a/deploy/build_env/manylinux2010/build.sh +++ b/deploy/build_env/manylinux2010/build.sh @@ -9,6 +9,7 @@ numpy_versions[cp36m]=1.12.1 numpy_versions[cp37m]=1.14.5 numpy_versions[cp38]=1.17.3 + for python_path in /opt/python/cp{27,35,36,37,38}*; do python_version=$(basename $python_path) python_version=${python_version#*-} @@ -17,6 +18,7 @@ for python_path in /opt/python/cp{27,35,36,37,38}*; do numpy_version=${numpy_versions[$python_version]} echo "building wheel for python_version=$python_version with numpy_version=$numpy_version" + mkdir -p /asapo/build cd /asapo/build cmake -DENABLE_LIBFABRIC=on \ -DCMAKE_BUILD_TYPE="Release" \ @@ -42,4 +44,4 @@ cd /asapo/build/consumer \ cd /asapo/build/producer \ && for wheel in wheelhouse/asapo_producer*.whl; do auditwheel repair $wheel --plat manylinux2010_x86_64 -w /asapo/build/wheelhouse - done \ No newline at end of file + done diff --git a/deploy/build_env/manylinux2010/build_image.sh b/deploy/build_env/manylinux2010/build_image.sh index 9a2bfe88613fdfd7b4a467ca527a78289a83ab52..d82efba17a387383ba08757b45a114bdf420f1a3 100755 --- a/deploy/build_env/manylinux2010/build_image.sh +++ b/deploy/build_env/manylinux2010/build_image.sh @@ -3,8 +3,9 @@ #docker build -t yakser/asapo-env:manylinux2010_ . #./docker-squash yakser/asapo-env:manylinux2010_ -t yakser/asapo-env:manylinux2010 -docker build -t yakser/asapo-env:manylinux2010 . -docker push yakser/asapo-env:manylinux2010 - - +. ../../_docker_vars.sh +docker build -t $ASAPO_DOCKER_REPOSITORY/asapo-env:manylinux2010 . +if [ $ASAPO_DOCKER_DO_PUSH = "YES" ]; then + docker push $ASAPO_DOCKER_REPOSITORY/asapo-env:manylinux2010 +fi diff --git a/deploy/build_env/manylinux2010/docker-squash b/deploy/build_env/manylinux2010/docker-squash index 22433497d01c838a7981dbff32f7ccef785a09d5..439a4e949e39fd1c06261d737139d17ac2184a1e 100755 --- a/deploy/build_env/manylinux2010/docker-squash +++ b/deploy/build_env/manylinux2010/docker-squash @@ -7,5 +7,6 @@ import sys from docker_squash.cli import run if __name__ == '__main__': + print("Hey Sergey, use the new --squash option in docker build") sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) sys.exit(run()) diff --git a/deploy/build_env/site/build_image.sh b/deploy/build_env/site/build_image.sh index 51ff855c17c518e1de76ab48a0cff2e1df9a86c3..51004244fb159c27b9182adf542bb1c84eae36c3 100755 --- a/deploy/build_env/site/build_image.sh +++ b/deploy/build_env/site/build_image.sh @@ -1,3 +1,8 @@ #!/usr/bin/env bash -docker build -t yakser/asapo-site-env -f Dockerfile . -docker push yakser/asapo-site-env \ No newline at end of file + +. ../../_docker_vars.sh + +docker build -t $ASAPO_DOCKER_REPOSITORY/asapo-site-env -f Dockerfile . +if [ $ASAPO_DOCKER_DO_PUSH = "YES" ]; then + docker push $ASAPO_DOCKER_REPOSITORY/asapo-site-env +fi diff --git a/deploy/nomad_consul_docker/build_image.sh b/deploy/nomad_consul_docker/build_image.sh index 227ac362f2d64b67ef80888f135065b2a16c3850..7b67d1ea3fad8226d4c5428498e26853f1d43322 100755 --- a/deploy/nomad_consul_docker/build_image.sh +++ b/deploy/nomad_consul_docker/build_image.sh @@ -1,4 +1,10 @@ #!/usr/bin/env bash -docker build -t yakser/asapo-nomad-cluster . -docker push yakser/asapo-nomad-cluster +set -e + +. ../_docker_vars.sh + +docker build -t $ASAPO_DOCKER_REPOSITORY/asapo-nomad-cluster . +if [ $ASAPO_DOCKER_DO_PUSH = "YES" ]; then + docker push $ASAPO_DOCKER_REPOSITORY/asapo-nomad-cluster +fi diff --git a/deploy/nomad_consul_docker/run.sh b/deploy/nomad_consul_docker/run.sh index 0c5bedbb62907ea8f20f814c1794e233bd8cdd3f..9bdbb41f602d87f822e376cf2c96bb41a3dc478a 100755 --- a/deploy/nomad_consul_docker/run.sh +++ b/deploy/nomad_consul_docker/run.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +set -e + +. ../_docker_vars.sh + NOMAD_ALLOC_HOST_SHARED=/tmp/asapo/container_host_shared/nomad_alloc ASAPO_USER=`id -u`:`id -g` @@ -13,5 +17,4 @@ docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \ -v /var/lib/docker:/var/lib/docker \ -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \ -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \ - --name asapo --net=host -d yakser/asapo-nomad-cluster - + --name asapo --net=host -d $ASAPO_DOCKER_REPOSITORY/asapo-nomad-cluster