diff --git a/deploy/docker/cluster/Dockerfile b/deploy/docker/cluster/Dockerfile index 2316d8811393091c18560cb251ae29c33a64846d..08e2e87ed3035699c04ce931edd0d0d5de41519b 100644 --- a/deploy/docker/cluster/Dockerfile +++ b/deploy/docker/cluster/Dockerfile @@ -40,9 +40,11 @@ RUN set -eux && \ nomad version && \ terraform version +RUN apt-get install -y iproute2 + ADD supervisord.conf /etc/ -RUN mkdir -p /var/log/supervisord/ +RUN mkdir -p /var/log/supervisord/ /etc/consul.d /etc/nomad.d COPY scripts/ /var/run/asapo/ @@ -50,4 +52,7 @@ RUN cd /var/run/asapo asapo && terraform init COPY asapo-* /usr/bin/ +COPY *.py /etc/asapo/ +COPY *.hcl.tpl /etc/asapo/ + ENTRYPOINT ["supervisord", "--configuration", "/etc/supervisord.conf"] diff --git a/deploy/docker/cluster/asapo-start b/deploy/docker/cluster/asapo-start index 21b345a2a562d5216c51a5463ceabece4abfd766..0a4483cc29393679c9944d8722dac9d92df3c965 100755 --- a/deploy/docker/cluster/asapo-start +++ b/deploy/docker/cluster/asapo-start @@ -1,3 +1,11 @@ #!/usr/bin/env bash + +if [ ! -f /var/nomad/token_all ]; then + nomad acl bootstrap > /var/nomad/bootstrap + cat /var/nomad/bootstrap | grep Secret | awk '{print $4}' > /var/nomad/token +fi + +#export NOMAD_TOKEN=`cat /var/nomad/token ` + cd /var/run/asapo && terraform apply -auto-approve "$@" \ No newline at end of file diff --git a/deploy/docker/cluster/asapo-stop b/deploy/docker/cluster/asapo-stop index e4fad7e65f8014ad71696ea08d4f142a434b1e61..c4401c97c3683225a3855ec2b6707a327c7172de 100755 --- a/deploy/docker/cluster/asapo-stop +++ b/deploy/docker/cluster/asapo-stop @@ -1,3 +1,5 @@ #!/usr/bin/env bash +#export NOMAD_TOKEN=`cat /var/nomad/token ` + cd /var/run/asapo && terraform destroy -auto-approve "$@" diff --git a/deploy/docker/cluster/consul.hcl.tpl b/deploy/docker/cluster/consul.hcl.tpl new file mode 100644 index 0000000000000000000000000000000000000000..d87a2c9dc60497cd6db628dbdb1b4d8fa9cd8d86 --- /dev/null +++ b/deploy/docker/cluster/consul.hcl.tpl @@ -0,0 +1,31 @@ +"bind_addr" = "$advertise_ip" + +enable_script_checks = true + +recursors = $recursors + +domain = "asapo" + +datacenter = "dc1" +data_dir = "/var/consul" +log_level = "INFO" + +enable_syslog = false +enable_debug = false +ui = true + +addresses = { +"http" = "0.0.0.0" +} + +node_meta = { + ib_address = "$ib_address" +} + +server = $is_server +bootstrap_expect = $n_servers + +rejoin_after_leave = true +retry_join = $server_adresses + + diff --git a/deploy/docker/cluster/nomad.hcl.tpl b/deploy/docker/cluster/nomad.hcl.tpl index 3a3974642ae5b2d7ea177a659bc09bd7c08efd3d..6b1a19b3e74acaf7e5df04fc1aeacb5187b0afdc 100644 --- a/deploy/docker/cluster/nomad.hcl.tpl +++ b/deploy/docker/cluster/nomad.hcl.tpl @@ -1,5 +1,3 @@ -"data_dir" = "$data_dir" - advertise { http = "$advertise_ip" rpc = "$advertise_ip" @@ -12,11 +10,15 @@ acl { server { enabled = $is_server - bootstrap_expect = 3 + bootstrap_expect = $n_servers } +data_dir = "/var/nomad" + client { -# network_interface = "$network_interface" enabled = true - node_class = "$node_class" + alloc_dir="$nomad_alloc_dir" } + + + diff --git a/deploy/docker/cluster/orchestr_config.py b/deploy/docker/cluster/orchestr_config.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..29086301c70f5d408b25b6d31411537802c36bb3 100644 --- a/deploy/docker/cluster/orchestr_config.py +++ b/deploy/docker/cluster/orchestr_config.py @@ -0,0 +1,71 @@ +import time +import argparse +from string import Template +import socket +import json +import os + +def is_server(ip,server_names): + servers = json.loads(server_names) + if len(servers) == 1: + return "true" + for server in json.loads(server_names): + try: + server_ip = socket.gethostbyname(server) + except: + server_ip = server + if ip == server_ip: + return "true" + return "false" + +def my_get_env(name,default): + res = os.getenv(name) + if res == None or res=='': + res = default + if default=='': + raise Exception(name + ' not set') + return res + +def set_parameters(): + d = {} + try: + my_ip = socket.gethostbyname(socket.gethostname()) + except: + print ("cannot define own ip") + my_ip = "127.0.0.1" + d['advertise_ip']=my_get_env('ADVERTISE_IP',my_ip) + d['n_servers']=my_get_env('N_SERVERS',1) + d['server_adresses']=my_get_env('SERVER_ADRESSES','["'+socket.gethostname()+'"]') + d['is_server']=is_server(d['advertise_ip'],d['server_adresses']) + d['ib_address']=my_get_env('IB_ADDRESS',"none") + d['nomad_alloc_dir']=my_get_env('NOMAD_ALLOC_DIR','') + d['recursors']=my_get_env('RECURSORS','["8.8.8.8"]') + return d + +def process_file(file_in,file_out): + print ("processing " + file_in+" to "+file_out) + filein = open(file_in) + src = Template(filein.read()) + d = set_parameters() + print d + + with open(file_out, "w") as out: + out.write(src.substitute(d)) + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + + parser.add_argument('--input-files', action='store', dest='input_files',nargs="*",required=True) + parser.add_argument('--output-files', action='store', dest='output_files',nargs="*",required=True) + + args = parser.parse_args() + + pairs = list(zip(args.input_files, args.output_files)) + + for pair in pairs: + process_file(pair[0],pair[1]) + + print ("finished configuring nomad and consul") + time.sleep(1) + diff --git a/deploy/docker/cluster/run.sh b/deploy/docker/cluster/run.sh index a2aa3699b62d7aea6c92af14fff21a047530f542..b26fbc88e286e15bae696adaf6a28d26ab9349a4 100755 --- a/deploy/docker/cluster/run.sh +++ b/deploy/docker/cluster/run.sh @@ -4,6 +4,12 @@ NOMAD_ALLOC_HOST_SHARED=/tmp/asapo/container_host_shared/nomad_alloc SERVICE_DATA_CLUSTER_SHARED=/tmp/asapo/asapo_cluster_shared/service_data DATA_GLOBAL_SHARED=/tmp/asapo/global_shared/data +#ADVERTISE_IP= +#RECURSORS= +#IB_ADDRESS= +#SERVER_ADRESSES= +#N_SERVERS= + mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED @@ -18,14 +24,18 @@ if (( mmc < 262144 )); then exit 1 fi - docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \ -v /var/lib/docker:/var/lib/docker \ -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \ -v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \ -v $DATA_GLOBAL_SHARED:$DATA_GLOBAL_SHARED \ - -e NOMAD_ALLOC=$NOMAD_ALLOC_HOST_SHARED \ + -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \ -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \ -e TF_VAR_data_dir=$DATA_GLOBAL_SHARED \ + -e ADVERTISE_IP=$ADVERTISE_IP \ + -e RECURSORS=$RECURSORS \ + -e IB_ADDRESS=$IB_ADDRESS \ + -e SERVER_ADRESSES=$SERVER_ADRESSES \ + -e N_SERVERS=$N_SERVERS \ --name asapo --net=host -d yakser/asapo-cluster diff --git a/deploy/docker/cluster/scripts/provider.tf b/deploy/docker/cluster/scripts/provider.tf index 19058d84779c97a33d9dafc55181127ee389902e..0df2412ca44f14d2a2f03f3c3b0d41d33a9e6cb3 100644 --- a/deploy/docker/cluster/scripts/provider.tf +++ b/deploy/docker/cluster/scripts/provider.tf @@ -1,4 +1,5 @@ provider "nomad" { address = "http://localhost:4646" + secret_id = "${chomp(file("/var/nomad/token"))}" } diff --git a/deploy/docker/cluster/supervisord.conf b/deploy/docker/cluster/supervisord.conf index d6cc9070ea2527df05e7a721c288480165c0f3c0..7ca29780b4221eb4294a9a142a4f3a3af8bc9dc8 100644 --- a/deploy/docker/cluster/supervisord.conf +++ b/deploy/docker/cluster/supervisord.conf @@ -7,10 +7,14 @@ loglevel=info ; info, debug, warn, trace childlogdir=/var/log/supervisord/ ; where child log files will live use=root +[program:prepare_scripts] +command=/usr/bin/python /etc/asapo/orchestr_config.py --input-files /etc/asapo/nomad.hcl.tpl /etc/asapo/consul.hcl.tpl --output-files /etc/nomad.d/nomad.hcl /etc/consul.d/consul.hcl +autorestart = false +startsecs = 1 +priority=1 + [program:consul] -command=/bin/consul agent -dev -client 0.0.0.0 -domain asapo -recursor=8.8.8.8 -#-config-dir=/etc/consul.d +command=/bin/consul agent -config-dir=/etc/consul.d [program:nomad] -command=/bin/nomad agent -dev -client -bind 0.0.0.0 -alloc-dir=%(ENV_NOMAD_ALLOC)s -# -config=/etc/nomad.d +command=/bin/nomad agent -config=/etc/nomad.d diff --git a/receiver/src/requests_dispatcher.cpp b/receiver/src/requests_dispatcher.cpp index a5814a9369a522079cfe51191bc06d15554a8fd4..35e46ce0032677bb734c7b36b999e26f2f3558d1 100644 --- a/receiver/src/requests_dispatcher.cpp +++ b/receiver/src/requests_dispatcher.cpp @@ -62,8 +62,8 @@ std::unique_ptr<Request> RequestsDispatcher::GetNextRequest(Error* err) const no log__->Debug("error getting next request from " + producer_uri_ + " - " + "peer has performed an orderly shutdown"); } else { log__->Error("error getting next request from " + producer_uri_ + " - " + (*err)-> - Explain() - ); + Explain() + ); } return nullptr; } diff --git a/receiver/unittests/test_requests_dispatcher.cpp b/receiver/unittests/test_requests_dispatcher.cpp index 6acd02eb89d280a9888e6a086dbaac3cc443f596..e5ecb59850063164245e509b668eed47872a3910 100644 --- a/receiver/unittests/test_requests_dispatcher.cpp +++ b/receiver/unittests/test_requests_dispatcher.cpp @@ -193,12 +193,12 @@ TEST_F(RequestsDispatcherTests, ErrorReceivetNextRequest) { TEST_F(RequestsDispatcherTests, ClosedConnectionOnReceivetNextRequest) { EXPECT_CALL(mock_statictics, StartTimer_t(StatisticEntity::kNetwork)); EXPECT_CALL(mock_io, Receive_t(_, _, _, _)) - .WillOnce( - DoAll(SetArgPointee<3>(asapo::ErrorTemplates::kEndOfFile.Generate().release()), - Return(0)) - ); + .WillOnce( + DoAll(SetArgPointee<3>(asapo::ErrorTemplates::kEndOfFile.Generate().release()), + Return(0)) + ); EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("peer has performed an orderly shutdown"), - HasSubstr("getting next request"), HasSubstr(connected_uri)))); + HasSubstr("getting next request"), HasSubstr(connected_uri)))); Error err; dispatcher->GetNextRequest(&err);