From a0e2f18864c7196e19e487aefc8ea911ae725c68 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Wed, 9 Oct 2019 11:02:09 +0200
Subject: [PATCH] more work on deployment
---
deploy/docker/cluster/consul.hcl.tpl | 2 +-
deploy/docker/cluster/nomad.hcl.tpl | 5 +++-
deploy/docker/cluster/orchestr_config.py | 5 ++++
deploy/docker/cluster/run_maxwell.sh | 24 ++++++++++++-------
.../cluster/scripts/asapo-brokers.nmd.tpl | 5 ++++
.../cluster/scripts/asapo-logging.nmd.tpl | 6 +++++
.../cluster/scripts/asapo-mongo.nmd.tpl | 6 ++++-
.../cluster/scripts/asapo-perfmetrics.nmd.tpl | 5 ++++
.../cluster/scripts/asapo-receivers.nmd.tpl | 5 ++++
.../cluster/scripts/asapo-services.nmd.tpl | 5 ++++
10 files changed, 57 insertions(+), 11 deletions(-)
diff --git a/deploy/docker/cluster/consul.hcl.tpl b/deploy/docker/cluster/consul.hcl.tpl
index d87a2c9dc..181fd8fe7 100644
--- a/deploy/docker/cluster/consul.hcl.tpl
+++ b/deploy/docker/cluster/consul.hcl.tpl
@@ -23,7 +23,7 @@ node_meta = {
}
server = $is_server
-bootstrap_expect = $n_servers
+$bootstrap_expect_string
rejoin_after_leave = true
retry_join = $server_adresses
diff --git a/deploy/docker/cluster/nomad.hcl.tpl b/deploy/docker/cluster/nomad.hcl.tpl
index 0b33eaaa1..ce4ce034d 100644
--- a/deploy/docker/cluster/nomad.hcl.tpl
+++ b/deploy/docker/cluster/nomad.hcl.tpl
@@ -10,7 +10,7 @@ acl {
server {
enabled = $is_server
- bootstrap_expect = $n_servers
+ $bootstrap_expect_string
}
data_dir = "/var/nomad"
@@ -20,6 +20,9 @@ client {
alloc_dir="$nomad_alloc_dir"
}
+meta {
+ "asapo_service" = $is_asapo_lightweight_service_node
+}
plugin "docker" {
config {
diff --git a/deploy/docker/cluster/orchestr_config.py b/deploy/docker/cluster/orchestr_config.py
index 6c40915b7..caccf483e 100644
--- a/deploy/docker/cluster/orchestr_config.py
+++ b/deploy/docker/cluster/orchestr_config.py
@@ -39,9 +39,14 @@ def set_parameters():
d['n_servers']=my_get_env('N_SERVERS',1)
d['server_adresses']=my_get_env('SERVER_ADRESSES','["'+socket.gethostname()+'"]')
d['is_server']=is_server(d['advertise_ip'],d['server_adresses'])
+ if d['is_server']=="true":
+ d['bootstrap_expect_string'] = "bootstrap_expect = "+d['n_servers']
+ else:
+ d['bootstrap_expect_string'] = ""
d['ib_address']=my_get_env('IB_ADDRESS',"none")
d['nomad_alloc_dir']=my_get_env('NOMAD_ALLOC_DIR','')
d['recursors']=my_get_env('RECURSORS','["8.8.8.8"]')
+
return d
def process_file(file_in,file_out):
diff --git a/deploy/docker/cluster/run_maxwell.sh b/deploy/docker/cluster/run_maxwell.sh
index 45031c15a..5abaa3307 100755
--- a/deploy/docker/cluster/run_maxwell.sh
+++ b/deploy/docker/cluster/run_maxwell.sh
@@ -1,9 +1,10 @@
#!/usr/bin/env bash
NOMAD_ALLOC_HOST_SHARED=/tmp/asapo/container_host_shared/nomad_alloc
-SERVICE_DATA_CLUSTER_SHARED=/tmp/asapo/asapo_cluster_shared/service_data
-DATA_GLOBAL_SHARED=/tmp/asapo/global_shared/data
-
+SERVICE_DATA_CLUSTER_SHARED=/home/yakubov/asapo/asapo_cluster_shared/service_data
+DATA_GLOBAL_SHARED=/home/yakubov/asapo/global_shared/data
+MAX_NOMAD_SERVERS=3 # rest are clients
+N_ASAPO_LIGHTWEIGHT_SERVICE_NODES=1 # where to put influx, elk, ... . Rest are receivers, brokers, mongodb
ASAPO_USER=`id -u`:`id -g`
USE_IB=True
@@ -17,8 +18,9 @@ DOCKER_TLS_CERT=/data/netapp/docker/certs/$USER/cert.pem
IB_ADDRESS=`hostname --short`-ib
RECURSORS=["\"131.169.40.200\"",\""131.169.194.200\""]
-SERVER_ADRESSES=`scontrol show hostnames $SLURM_JOB_NODELIST | awk 'BEGIN{printf "[\""} {printf "%s%s",sep,$0; sep=","} END{print "\"]"}'`
-N_SERVERS=$SLURM_JOB_NUM_NODES
+N_SERVERS=$(( $SLURM_JOB_NUM_NODES > $MAX_NOMAD_SERVERS ? $MAX_NOMAD_SERVERS : $SLURM_JOB_NUM_NODES ))
+SERVER_ADRESSES=`scontrol show hostnames $SLURM_JOB_NODELIST | head -$N_SERVERS | awk 'BEGIN{printf "["} {printf "%s\"%s\"",sep,$0; sep=","} END{print "]"}'`
+ASAPO_LIGHTWEIGHT_SERVICE_NODES=`scontrol show hostnames $SLURM_JOB_NODELIST | head -$N_ASAPO_LIGHTWEIGHT_SERVICE_NODES | awk 'BEGIN{printf "["} {printf "%s\"%s\"",sep,$0; sep=","} END{print "]"}'`
mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED
chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED
@@ -33,9 +35,14 @@ if (( mmc < 262144 )); then
# exit 1
fi
-docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \
+docker rm -f asapo
+
+docker pull yakser/asapo-cluster
+
+
+dockerrun --rm \
-u $ASAPO_USER \
- -v /var/lib/docker:/var/lib/docker \
+ -v /scratch/docker/100000.100000:/scratch/docker/100000.100000 \
-v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \
-v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \
-v $DOCKER_TLS_CA:/etc/nomad/ca.pem \
@@ -50,7 +57,8 @@ docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \
-e TF_VAR_asapo_user=$ASAPO_USER \
-e IB_ADDRESS=$IB_ADDRESS \
-e SERVER_ADRESSES=$SERVER_ADRESSES \
+ -e ASAPO_LIGHTWEIGHT_SERVICE_NODES=$ASAPO_LIGHTWEIGHT_SERVICE_NODES \
-e DOCKER_ENDPOINT=$DOCKER_ENDPOINT \
-e N_SERVERS=$N_SERVERS \
- --name asapo --net=host yakser/asapo-cluster
+ --name asapo yakser/asapo-cluster
diff --git a/deploy/docker/cluster/scripts/asapo-brokers.nmd.tpl b/deploy/docker/cluster/scripts/asapo-brokers.nmd.tpl
index 525fcc9df..f0fc6ad7d 100644
--- a/deploy/docker/cluster/scripts/asapo-brokers.nmd.tpl
+++ b/deploy/docker/cluster/scripts/asapo-brokers.nmd.tpl
@@ -1,5 +1,10 @@
job "asapo-brokers" {
datacenters = ["dc1"]
+ affinity {
+ attribute = "${meta.asapo_service}"
+ value = "false"
+ weight = 100
+ }
update {
max_parallel = 1
diff --git a/deploy/docker/cluster/scripts/asapo-logging.nmd.tpl b/deploy/docker/cluster/scripts/asapo-logging.nmd.tpl
index a34c531dd..f56d0e978 100644
--- a/deploy/docker/cluster/scripts/asapo-logging.nmd.tpl
+++ b/deploy/docker/cluster/scripts/asapo-logging.nmd.tpl
@@ -1,5 +1,11 @@
job "asapo-logging" {
datacenters = ["dc1"]
+ affinity {
+ attribute = "${meta.asapo_service}"
+ value = "true"
+ weight = 100
+ }
+
# update {
# max_parallel = 1
diff --git a/deploy/docker/cluster/scripts/asapo-mongo.nmd.tpl b/deploy/docker/cluster/scripts/asapo-mongo.nmd.tpl
index 78050cc19..3ac6e4945 100644
--- a/deploy/docker/cluster/scripts/asapo-mongo.nmd.tpl
+++ b/deploy/docker/cluster/scripts/asapo-mongo.nmd.tpl
@@ -1,6 +1,10 @@
job "asapo-mongo" {
datacenters = ["dc1"]
-
+ affinity {
+ attribute = "${meta.asapo_service}"
+ value = "false"
+ weight = 100
+ }
update {
max_parallel = 1
min_healthy_time = "10s"
diff --git a/deploy/docker/cluster/scripts/asapo-perfmetrics.nmd.tpl b/deploy/docker/cluster/scripts/asapo-perfmetrics.nmd.tpl
index e65405d92..59ea06033 100644
--- a/deploy/docker/cluster/scripts/asapo-perfmetrics.nmd.tpl
+++ b/deploy/docker/cluster/scripts/asapo-perfmetrics.nmd.tpl
@@ -1,5 +1,10 @@
job "asapo-perfmetrics" {
datacenters = ["dc1"]
+ affinity {
+ attribute = "${meta.asapo_service}"
+ value = "true"
+ weight = 100
+ }
# update {
# max_parallel = 1
diff --git a/deploy/docker/cluster/scripts/asapo-receivers.nmd.tpl b/deploy/docker/cluster/scripts/asapo-receivers.nmd.tpl
index 264718d06..ecdeb2252 100644
--- a/deploy/docker/cluster/scripts/asapo-receivers.nmd.tpl
+++ b/deploy/docker/cluster/scripts/asapo-receivers.nmd.tpl
@@ -1,5 +1,10 @@
job "asapo-receivers" {
datacenters = ["dc1"]
+ affinity {
+ attribute = "${meta.asapo_service}"
+ value = "false"
+ weight = 100
+ }
update {
max_parallel = 1
diff --git a/deploy/docker/cluster/scripts/asapo-services.nmd.tpl b/deploy/docker/cluster/scripts/asapo-services.nmd.tpl
index 628deaba4..ed3838639 100644
--- a/deploy/docker/cluster/scripts/asapo-services.nmd.tpl
+++ b/deploy/docker/cluster/scripts/asapo-services.nmd.tpl
@@ -1,5 +1,10 @@
job "asapo-services" {
datacenters = ["dc1"]
+ affinity {
+ attribute = "${meta.asapo_service}"
+ value = "true"
+ weight = 100
+ }
type = "service"
--
GitLab