diff --git a/deploy/docker/cluster/consul.hcl.tpl b/deploy/docker/cluster/consul.hcl.tpl index d87a2c9dc60497cd6db628dbdb1b4d8fa9cd8d86..181fd8fe71e8fc14188089207596797b429cda59 100644 --- a/deploy/docker/cluster/consul.hcl.tpl +++ b/deploy/docker/cluster/consul.hcl.tpl @@ -23,7 +23,7 @@ node_meta = { } server = $is_server -bootstrap_expect = $n_servers +$bootstrap_expect_string rejoin_after_leave = true retry_join = $server_adresses diff --git a/deploy/docker/cluster/nomad.hcl.tpl b/deploy/docker/cluster/nomad.hcl.tpl index 0b33eaaa131ac987a7a4d07025be145b6a94f6db..ce4ce034d97e062601fbdf1860c2ec47bdb41447 100644 --- a/deploy/docker/cluster/nomad.hcl.tpl +++ b/deploy/docker/cluster/nomad.hcl.tpl @@ -10,7 +10,7 @@ acl { server { enabled = $is_server - bootstrap_expect = $n_servers + $bootstrap_expect_string } data_dir = "/var/nomad" @@ -20,6 +20,9 @@ client { alloc_dir="$nomad_alloc_dir" } +meta { + "asapo_service" = $is_asapo_lightweight_service_node +} plugin "docker" { config { diff --git a/deploy/docker/cluster/orchestr_config.py b/deploy/docker/cluster/orchestr_config.py index 6c40915b7770967c46cde551f8714c13cfd6c74b..caccf483ed96d4c7fb56acfacfb43c8b4f5b1bee 100644 --- a/deploy/docker/cluster/orchestr_config.py +++ b/deploy/docker/cluster/orchestr_config.py @@ -39,9 +39,14 @@ def set_parameters(): d['n_servers']=my_get_env('N_SERVERS',1) d['server_adresses']=my_get_env('SERVER_ADRESSES','["'+socket.gethostname()+'"]') d['is_server']=is_server(d['advertise_ip'],d['server_adresses']) + if d['is_server']=="true": + d['bootstrap_expect_string'] = "bootstrap_expect = "+d['n_servers'] + else: + d['bootstrap_expect_string'] = "" d['ib_address']=my_get_env('IB_ADDRESS',"none") d['nomad_alloc_dir']=my_get_env('NOMAD_ALLOC_DIR','') d['recursors']=my_get_env('RECURSORS','["8.8.8.8"]') + return d def process_file(file_in,file_out): diff --git a/deploy/docker/cluster/run_maxwell.sh b/deploy/docker/cluster/run_maxwell.sh index 45031c15aa6dddd722624f7ae6464447c05eb456..5abaa33074a079353edf01ac05136517164feb77 100755 --- a/deploy/docker/cluster/run_maxwell.sh +++ b/deploy/docker/cluster/run_maxwell.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash NOMAD_ALLOC_HOST_SHARED=/tmp/asapo/container_host_shared/nomad_alloc -SERVICE_DATA_CLUSTER_SHARED=/tmp/asapo/asapo_cluster_shared/service_data -DATA_GLOBAL_SHARED=/tmp/asapo/global_shared/data - +SERVICE_DATA_CLUSTER_SHARED=/home/yakubov/asapo/asapo_cluster_shared/service_data +DATA_GLOBAL_SHARED=/home/yakubov/asapo/global_shared/data +MAX_NOMAD_SERVERS=3 # rest are clients +N_ASAPO_LIGHTWEIGHT_SERVICE_NODES=1 # where to put influx, elk, ... . Rest are receivers, brokers, mongodb ASAPO_USER=`id -u`:`id -g` USE_IB=True @@ -17,8 +18,9 @@ DOCKER_TLS_CERT=/data/netapp/docker/certs/$USER/cert.pem IB_ADDRESS=`hostname --short`-ib RECURSORS=["\"131.169.40.200\"",\""131.169.194.200\""] -SERVER_ADRESSES=`scontrol show hostnames $SLURM_JOB_NODELIST | awk 'BEGIN{printf "[\""} {printf "%s%s",sep,$0; sep=","} END{print "\"]"}'` -N_SERVERS=$SLURM_JOB_NUM_NODES +N_SERVERS=$(( $SLURM_JOB_NUM_NODES > $MAX_NOMAD_SERVERS ? $MAX_NOMAD_SERVERS : $SLURM_JOB_NUM_NODES )) +SERVER_ADRESSES=`scontrol show hostnames $SLURM_JOB_NODELIST | head -$N_SERVERS | awk 'BEGIN{printf "["} {printf "%s\"%s\"",sep,$0; sep=","} END{print "]"}'` +ASAPO_LIGHTWEIGHT_SERVICE_NODES=`scontrol show hostnames $SLURM_JOB_NODELIST | head -$N_ASAPO_LIGHTWEIGHT_SERVICE_NODES | awk 'BEGIN{printf "["} {printf "%s\"%s\"",sep,$0; sep=","} END{print "]"}'` mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED @@ -33,9 +35,14 @@ if (( mmc < 262144 )); then # exit 1 fi -docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \ +docker rm -f asapo + +docker pull yakser/asapo-cluster + + +dockerrun --rm \ -u $ASAPO_USER \ - -v /var/lib/docker:/var/lib/docker \ + -v /scratch/docker/100000.100000:/scratch/docker/100000.100000 \ -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \ -v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \ -v $DOCKER_TLS_CA:/etc/nomad/ca.pem \ @@ -50,7 +57,8 @@ docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \ -e TF_VAR_asapo_user=$ASAPO_USER \ -e IB_ADDRESS=$IB_ADDRESS \ -e SERVER_ADRESSES=$SERVER_ADRESSES \ + -e ASAPO_LIGHTWEIGHT_SERVICE_NODES=$ASAPO_LIGHTWEIGHT_SERVICE_NODES \ -e DOCKER_ENDPOINT=$DOCKER_ENDPOINT \ -e N_SERVERS=$N_SERVERS \ - --name asapo --net=host yakser/asapo-cluster + --name asapo yakser/asapo-cluster diff --git a/deploy/docker/cluster/scripts/asapo-brokers.nmd.tpl b/deploy/docker/cluster/scripts/asapo-brokers.nmd.tpl index 525fcc9df9609cd422ff887cce4fa520f54abff7..f0fc6ad7d3767c43a82e23f1f772d9b8ab67329a 100644 --- a/deploy/docker/cluster/scripts/asapo-brokers.nmd.tpl +++ b/deploy/docker/cluster/scripts/asapo-brokers.nmd.tpl @@ -1,5 +1,10 @@ job "asapo-brokers" { datacenters = ["dc1"] + affinity { + attribute = "${meta.asapo_service}" + value = "false" + weight = 100 + } update { max_parallel = 1 diff --git a/deploy/docker/cluster/scripts/asapo-logging.nmd.tpl b/deploy/docker/cluster/scripts/asapo-logging.nmd.tpl index a34c531dd143c8bf7424410dd71f18f8b7245dc0..f56d0e97838feb7650e1968492008f483fe1b2bc 100644 --- a/deploy/docker/cluster/scripts/asapo-logging.nmd.tpl +++ b/deploy/docker/cluster/scripts/asapo-logging.nmd.tpl @@ -1,5 +1,11 @@ job "asapo-logging" { datacenters = ["dc1"] + affinity { + attribute = "${meta.asapo_service}" + value = "true" + weight = 100 + } + # update { # max_parallel = 1 diff --git a/deploy/docker/cluster/scripts/asapo-mongo.nmd.tpl b/deploy/docker/cluster/scripts/asapo-mongo.nmd.tpl index 78050cc19926105f7b3cf7d0247de358b44ee871..3ac6e4945473ea22f8f6d9a1b35ad30220c706dd 100644 --- a/deploy/docker/cluster/scripts/asapo-mongo.nmd.tpl +++ b/deploy/docker/cluster/scripts/asapo-mongo.nmd.tpl @@ -1,6 +1,10 @@ job "asapo-mongo" { datacenters = ["dc1"] - + affinity { + attribute = "${meta.asapo_service}" + value = "false" + weight = 100 + } update { max_parallel = 1 min_healthy_time = "10s" diff --git a/deploy/docker/cluster/scripts/asapo-perfmetrics.nmd.tpl b/deploy/docker/cluster/scripts/asapo-perfmetrics.nmd.tpl index e65405d9259e6d014b6c6c78fd0743e527ada568..59ea060338d6cd23c79398d6384d162d00effe71 100644 --- a/deploy/docker/cluster/scripts/asapo-perfmetrics.nmd.tpl +++ b/deploy/docker/cluster/scripts/asapo-perfmetrics.nmd.tpl @@ -1,5 +1,10 @@ job "asapo-perfmetrics" { datacenters = ["dc1"] + affinity { + attribute = "${meta.asapo_service}" + value = "true" + weight = 100 + } # update { # max_parallel = 1 diff --git a/deploy/docker/cluster/scripts/asapo-receivers.nmd.tpl b/deploy/docker/cluster/scripts/asapo-receivers.nmd.tpl index 264718d066e8875a65a3367fd94a0951e1d68f47..ecdeb2252b1930dee13d864113bfd8ebcab09c24 100644 --- a/deploy/docker/cluster/scripts/asapo-receivers.nmd.tpl +++ b/deploy/docker/cluster/scripts/asapo-receivers.nmd.tpl @@ -1,5 +1,10 @@ job "asapo-receivers" { datacenters = ["dc1"] + affinity { + attribute = "${meta.asapo_service}" + value = "false" + weight = 100 + } update { max_parallel = 1 diff --git a/deploy/docker/cluster/scripts/asapo-services.nmd.tpl b/deploy/docker/cluster/scripts/asapo-services.nmd.tpl index 628deaba4e8566bfb6505432a76f50fc0492f03a..ed3838639cb9e474e0f615a33ea92072bde1c2eb 100644 --- a/deploy/docker/cluster/scripts/asapo-services.nmd.tpl +++ b/deploy/docker/cluster/scripts/asapo-services.nmd.tpl @@ -1,5 +1,10 @@ job "asapo-services" { datacenters = ["dc1"] + affinity { + attribute = "${meta.asapo_service}" + value = "true" + weight = 100 + } type = "service"