Merge pull request #67 in ASAPO/asapo from feature_clients-devops to develop

* commit 'f228dbee':
  update configuration
  add license info
  update test, move asapo client template to submodule
  refactor docker images, add examples
  improve output
  add nginx and co to the image
  refactor deploy scripts#

deploy/asapo_orchestration_docker/scripts/telegraf.conf.tpl

+[[inputs.statsd]]
+  protocol = "udp"
+  service_address = ":{{ env "NOMAD_PORT_telegraf_stream" }}"
+  namepass= ["nomad_client_allocs"]
+  templates = [
+    "nomad.client.allocs.* measurement.measurement.measurement.field.field.job.group.alloc-id.task.hostname",
+    "nomad.nomad.job_summary.* measurement.measurement.measurement.field.job.group.hostname"
+  ]
+
+[[inputs.consul]]
+
+[[outputs.file]]
+	files=["stdout"]
+
+
+[[outputs.influxdb]]
+    urls = ["http://localhost:{{ env "NOMAD_META_nginx_port" }}/influxdb"]
+

deploy/asapo_orchestration_docker/scripts/templates.tf

+data "template_file" "monitoring_template" {
+  template = "${file("${var.job_scripts_dir}/monitoring.nmd.tpl")}"
+  vars = {
+  	monitoring = "${var.monitoring}"
+    service_dir = "${var.service_dir}"
+    grafana_version = "${var.grafana_version}"
+    grafana_total_memory_size = "${var.grafana_total_memory_size}"
+    grafana_port = "${var.grafana_port}"
+    grafana_version = "${var.grafana_version}"
+    grafana_total_memory_size = "${var.grafana_total_memory_size}"
+    grafana_port = "${var.grafana_port}"
+    influxdb_version = "${var.influxdb_version}"
+    influxdb_total_memory_size = "${var.influxdb_total_memory_size}"
+    influxdb_port = "${var.influxdb_port}"
+    telegraf_version = "${var.telegraf_version}"
+    telegraf_total_memory_size = "${var.telegraf_total_memory_size}"
+    telegraf_port_stream = "${var.telegraf_port_stream}"
+    nginx_port = "${var.nginx_port}"
+    asapo_user = "${var.asapo_user}"
+    scripts_dir = "${var.job_scripts_dir}"
+    }
+}
+
+data "template_file" "nginx_template" {
+  template = "${file("${var.job_scripts_dir}/nginx.nmd.tpl")}"
+  vars = {
+    scripts_dir = "${var.job_scripts_dir}"
+    nginx_total_memory_size = "${var.nginx_total_memory_size}"
+    nginx_version = "${var.nginx_version}"
+    nginx_port = "${var.nginx_port}"
+    nginx_port_stream = "${var.nginx_port_stream}"
+    telegraf_port_stream = "${var.telegraf_port_stream}"
+    grafana_port = "${var.grafana_port}"
+    influxdb_port = "${var.influxdb_port}"
+    asapo_user = "${var.asapo_user}"
+    consul_dns_port = "${var.consul_dns_port}"
+  }
+}
+

deploy/asapo_orchestration_docker/scripts/vars.tf

+variable "asapo_user" {}
+
+variable "monitoring" {}
+
+variable "grafana_version" {}
+variable "telegraf_version" {}
+variable "influxdb_version" {}
+variable "nginx_version" {}
+
+variable "job_scripts_dir" {}
+variable "service_dir" {}
+variable "data_dir" {}
+
+variable "grafana_total_memory_size" {}
+variable "grafana_port" {}
+
+variable "telegraf_port_stream" {}
+variable "telegraf_total_memory_size" {}
+
+variable "influxdb_total_memory_size" {}
+variable "influxdb_port" {}
+
+variable "nginx_total_memory_size" {}
+variable "nginx_port" {}
+variable "nginx_port_stream" {}
+
+variable "consul_dns_port" {}
+

deploy/asapo_services/Dockerfile

+FROM yakser/asapo-nomad-cluster
+
+MAINTAINER DESY IT
+
+COPY scripts/ /var/run/asapo/
+
+COPY asapo-wait-service  /usr/bin/
+
+RUN cd /var/run/asapo asapo && terraform init

deploy/asapo/asapo-wait-service → deploy/asapo_services/asapo-wait-service

@@ -4,8 +4,6 @@ if [ "$2" == "false" ]; then
   exit 0
 fi
 
-
 until dig +short @127.0.0.1 -p 8600 $1.service.asapo | grep . ; do
 sleep 1
 done
-

deploy/asapo/run.sh → deploy/asapo_services/run.sh

 #!/usr/bin/env bash
 
-NOMAD_ALLOC_HOST_SHARED=/tmp/asapo/container_host_shared/nomad_alloc
-SERVICE_DATA_CLUSTER_SHARED=/tmp/asapo/asapo_cluster_shared/service_data
-DATA_GLOBAL_SHARED=/tmp/asapo/global_shared/data
+NOMAD_ALLOC_HOST_SHARED=/var/tmp/asapo/container_host_shared/nomad_alloc
+SERVICE_DATA_CLUSTER_SHARED=/var/tmp/asapo/asapo_cluster_shared/service_data
+DATA_GLOBAL_SHARED=/var/tmp/asapo/global_shared/data
 MONGO_DIR=$SERVICE_DATA_CLUSTER_SHARED/mongodb
 
 ASAPO_USER=`id -u`:`id -g`
@@ -14,7 +14,7 @@ ASAPO_USER=`id -u`:`id -g`
 #N_SERVERS=
 
 ASAPO_VAR_FILE=`pwd`/asapo_overwrite_vars.tfvars
-
+ACL_ENABLED=true
 
 mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED
 chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED
@@ -52,6 +52,7 @@ docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \
   -e RECURSORS=$RECURSORS \
   -e TF_VAR_asapo_user=$ASAPO_USER \
   -e IB_ADDRESS=$IB_ADDRESS \
+  -e ACL_ENABLED=$ACL_ENABLED \
   -e SERVER_ADRESSES=$SERVER_ADRESSES \
   -e N_SERVERS=$N_SERVERS \
   --name asapo --net=host -d yakser/asapo-cluster

deploy/asapo/run_maxwell.sh → deploy/asapo_services/run_maxwell.sh

 #!/usr/bin/env bash
 
 #folders
-NOMAD_ALLOC_HOST_SHARED=/tmp/asapo/container_host_shared/nomad_alloc
+NOMAD_ALLOC_HOST_SHARED=/var/tmp/asapo/container_host_shared/nomad_alloc
 SERVICE_DATA_CLUSTER_SHARED=/home/yakubov/asapo/asapo_cluster_shared/service_data
 DATA_GLOBAL_SHARED=/gpfs/petra3/scratch/yakubov/asapo_shared
 MONGO_DIR=/scratch/mongodb # due to performance reasons mongodb can benefit from writing to local filesystem (HA to be worked on)
@@ -15,6 +15,7 @@ RECURSORS=["\"131.169.40.200\"",\""131.169.194.200\""]
 ASAPO_USER=`id -u`:`id -g`
 
 ASAPO_VAR_FILE=`pwd`/asapo_overwrite_vars.tfvars
+ACL_ENABLED=true
 
 
 #docker stuff
@@ -77,6 +78,7 @@ dockerrun --rm  \
 	-e RECURSORS=$RECURSORS \
 	-e TF_VAR_asapo_user=$ASAPO_USER \
 	-e IB_ADDRESS=$IB_ADDRESS \
+    -e ACL_ENABLED=$ACL_ENABLED \
 	-e SERVER_ADRESSES=$SERVER_ADRESSES \
 	-e ASAPO_LIGHTWEIGHT_SERVICE_NODES=$ASAPO_LIGHTWEIGHT_SERVICE_NODES \
 	-e DOCKER_ENDPOINT=$DOCKER_ENDPOINT \