Merge pull request #103 in ASAPO/asapo from fixes_during_p021tests to develop

* commit '6abf309f':
  update changelog, fix fluentd pattern
  update orchestration, fix retrieve data in python lib

CHANGELOG.md

+## 20.06.3
+
+BUG FIXES
+* fix retrieve_data in Python modules for data ingested using metadata only mode
+* fix asapo orchestration image stabilize nginx and update fluentd configuration to follow Nomad jobs log rotation 
+
 ## 20.06.2
 
 BUG FIXES

consumer/api/python/asapo_consumer.pyx.in

@@ -144,6 +144,8 @@ cdef class PyDataBroker:
         if err:
             throw_exception(err)
         cdef np.npy_intp dims[1]
+        info_str = _str(info.Json())
+        meta = json.loads(info_str)
         dims[0] = meta['size']
         cdef char* ptr = <char*> data.release()
         arr =  np.PyArray_SimpleNewFromData(1, dims, np.NPY_BYTE, ptr)

deploy/asapo_orchestration_docker/scripts/fluent-bit-cluster.conf.tpl

@@ -15,7 +15,7 @@
 {{ range ("influxdb grafana" | split " ") }}
 [INPUT]
     Name tail
-    Path ${NOMAD_ALLOC_DIR}/${NOMAD_ALLOC_ID}/alloc/logs/{{.}}*.0
+    Path ${NOMAD_ALLOC_DIR}/${NOMAD_ALLOC_ID}/alloc/logs/{{.}}*
     Tag asapo.{{.}}
     # Tag_Regex .*/(?<service>.*).(?<channel>stderr|stdout).*
     DB ${NOMAD_TASK_DIR}/logs.db

deploy/asapo_orchestration_docker/scripts/fluent-bit-nginx.conf.tpl

@@ -4,7 +4,7 @@
 
 [INPUT]
     Name tail
-    Path ${NOMAD_ALLOC_DIR}/${NOMAD_ALLOC_ID}/alloc/logs/nginx*.0
+    Path ${NOMAD_ALLOC_DIR}/${NOMAD_ALLOC_ID}/alloc/logs/nginx*
     Tag asapo.nginx
     DB ${NOMAD_TASK_DIR}/logs.db
     Parser level

deploy/asapo_orchestration_docker/scripts/nginx.nmd.tpl

@@ -63,8 +63,7 @@ job "asapo-nginx" {
       template {
          source        = "${scripts_dir}/nginx.conf.tpl"
          destination   = "local/nginx.conf"
-         change_mode   = "signal"
-         change_signal   = "SIGHUP"
+         change_mode   = "restart"
       }
    }
 

deploy/asapo_orchestration_docker/scripts/resources.tf

@@ -4,4 +4,5 @@ resource "nomad_job" "monitoring" {
 
 resource "nomad_job" "nginx" {
   jobspec = "${data.template_file.nginx_template.rendered}"
+  depends_on = [nomad_job.monitoring,null_resource.influxdb,null_resource.grafana,null_resource.telegraf]
 }

deploy/asapo_orchestration_docker/scripts/resources_services.tf

+resource "null_resource" "nginx" {
+  provisioner "local-exec" {
+    command = "asapo-wait-service nginx"
+  }
+  depends_on = [nomad_job.nginx]
+}
+
+resource "null_resource" "influxdb" {
+  provisioner "local-exec" {
+    command = "asapo-wait-service influxdb"
+  }
+  depends_on = [nomad_job.monitoring]
+}
+
+resource "null_resource" "grafana" {
+  provisioner "local-exec" {
+    command = "asapo-wait-service grafana"
+  }
+  depends_on = [nomad_job.monitoring]
+}
+
+
+resource "null_resource" "telegraf" {
+  provisioner "local-exec" {
+    command = "asapo-wait-service telegraf"
+  }
+  depends_on = [nomad_job.monitoring]
+}

deploy/nomad_consul_docker/Dockerfile

@@ -24,9 +24,9 @@ RUN add-apt-repository \
 RUN apt-get update && apt-get install -y docker-ce-cli wget unzip iproute2 vim td-agent-bit
 
 
-ENV CONSUL_VERSION=1.6.0
-ENV NOMAD_VERSION=0.9.5
-ENV TERRAFORM_VERSION=0.12.9
+ENV CONSUL_VERSION=1.8.0
+ENV NOMAD_VERSION=0.11.3
+ENV TERRAFORM_VERSION=0.12.28
 ENV HASHICORP_RELEASES=https://releases.hashicorp.com
 
 RUN set -eux && \
@@ -53,7 +53,7 @@ COPY scripts/ /var/run/asapo/
 
 RUN cd /var/run/asapo && terraform init
 
-COPY jobs-*  /usr/bin/
+COPY jobs-* asapo-wait-service /usr/bin/
 
 COPY *.py  /etc/asapo/
 COPY *.hcl.tpl  /etc/asapo/

deploy/nomad_consul_docker/asapo-wait-service

+#!/usr/bin/env bash
+
+if [ "$2" == "false" ]; then
+  exit 0
+fi
+
+until dig +short @127.0.0.1 -p 8600 $1.service.asapo | grep . ; do
+sleep 1
+done