Merge pull request #67 in ASAPO/asapo from feature_clients-devops to develop

* commit 'f228dbee':
  update configuration
  add license info
  update test, move asapo client template to submodule
  refactor docker images, add examples
  improve output
  add nginx and co to the image
  refactor deploy scripts#

.gitmodules

+[submodule "deploy/asapo-client-template"]
+	path = deploy/asapo_client_template
+	url = ssh://yakubov@stash.desy.de:7999/asapo/asapo-client-template.git

3d_party/ctest_junit_convert/convert.py

 #!/usr/bin/env python
-# coding: utf-8
-# originally from:
-#  http://www.warp1337.com/content/how-use-ctest-jenkins-xunit-or-junit-plugin
-# improved by:
-#  Jorge Araya Navarro <elcorreo@deshackra.com>
-
-#                         Veni, Sancte Spiritus.
 
 from lxml import etree
 import argparse
-from os.path import expanduser
 from os.path import join
-import logging
-
-# configure logging
-logging.basicConfig(format="%(levelname)s: %(message)s",
-                    level=logging.ERROR)
 
-desc = ("Converts ctest XML file to xUnit/JUnit XML "
-        "compatible file to use with Jenkins-CI. "
-        "Did you found any bug? please report it on: "
-        "https://bitbucket.org/shackra/ctest-jenkins/issues")
-
-# configure argument parser.
-parser = argparse.ArgumentParser(description=desc)
-parser.add_argument("-x", "--xslt", help="the XSLT file to use", required=True)
-parser.add_argument("-t", "--tag", help=("the directory where 'Testing/TAG'"
-                                         "file is. Remember to call ctest with"
-                                         " '-T test' option to generate it"),
-                    required=True)
+parser = argparse.ArgumentParser()
+parser.add_argument("-x",dest='xsl', required=True)
+parser.add_argument("-t", dest='tag',required=True)
 
 parsed = parser.parse_args()
-# expanding user symbol "~"
-parsed.xsl = expanduser(parsed.xslt)
-parsed.tag = expanduser(parsed.tag)
-
-# opening the TAG file
-directory = None
-try:
-    with open(join(parsed.tag, "Testing", "TAG")) as tagfile:
-        directory = tagfile.readline().strip()
 
-except NotADirectoryError:
-    logging.error(
-        "'Testing/TAG' wasn't found on directory '{}'.".format(parsed.tag))
-    exit(1)
-except FileNotFoundError:
-    logging.error(
-        "File '{}' not found.".format(join(parsed.tag, "Testing", "TAG")))
-    exit(1)
 
-xmldoc = None
-transform = None
-try:
-    with open(join(parsed.tag, "Testing", directory, "Test.xml"))\
-            as testxmlfile:
-        xmldoc = etree.parse(testxmlfile)
+with open(join(parsed.tag, "Testing", "TAG")) as tagfile:
+	directory = tagfile.readline().strip()
 
-except FileNotFoundError:
-    logging.error("File {} not found. Was it deleted or moved?".format(
-        join(parsed.tag, "Testing", directory, "Test.xml")))
-    exit(1)
+with open(join(parsed.tag, "Testing", directory, "Test.xml")) as testxmlfile:
+	xmldoc = etree.parse(testxmlfile)
 
-try:
-    with open(parsed.xslt) as xsltfile:
-        xslt_root = etree.XML(xsltfile.read())
-        transform = etree.XSLT(xslt_root)
-except FileNotFoundError:
-    logging.error("File {} not found.".format(parsed.xslt))
-    exit(1)
+with open(parsed.xsl) as xslfile:
+	xsl_root = etree.XML(xslfile.read())
+	transform = etree.XSLT(xsl_root)
 
 result_tree = transform(xmldoc)
 print(result_tree)

LICENSE-CREDITS-3RD-PARTY.txt

+There are following 3rd party libraries and tools used in various parts of the ASAP::O:
+
+========
+MIT - https://opensource.org/licenses/MIT
+========
+ctest to junit converter - https://github.com/genome/build-common
+rapidjson - https://github.com/Tencent/rapidjson
+go testing packages - https://github.com/stretchr/testify
+go unique id generator package https://github.com/rs/xid
+go influxdb client - https://github.com/influxdata/influxdb1-client
+using Visual Studio to compile for Windows - https://visualstudio.microsoft.com
+
+========
+Apache2 - https://www.apache.org/licenses/LICENSE-2.0
+========
+mongo-c-driver - https://github.com/mongodb/mongo-c-driver
+go mongodb packages - https://github.com/mongodb/mongo-go-driver
+go SQL parser package - https://github.com/knocknote/vitess-sqlparser/sqlparser
+cython - https://github.com/cython/cython
+Docker open source components - https://www.docker.com/
+
+========
+BSD 3-Clause - https://opensource.org/licenses/BSD-3-Clause
+========
+go request router package - https://github.com/gorilla/mux
+using go to compile binaries - http://golang.org
+
+========
+Mozilla Public License, v. 2.0 - http://mozilla.org/MPL/2.0/
+========
+go consul client - github.com/hashicorp/consul/api
+
+========
+Custom licenses
+========
+libcurl - https://curl.haxx.se/ - MIT-like license - https://curl.haxx.se/docs/copyright.html
+
+========
+Remarks
+========
+gcc compiled binaries fall into the GCC Runtime Library Exception - https://www.gnu.org/licenses/gcc-exception.html
+

LICENSE.txt

+Copyright (c) 2019 Deutsches Elektronen-Synchrotron (DESY)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -36,3 +36,6 @@ Need googletest >= [1.8.0](https://github.com/google/googletest/releases/tag/rel
  - mkdir build
  - cd build
  - cmake -DBUILD_TESTS=ON ..
+
+The software is MIT licensed (see LICENSE.txt) and uses third party libraries that are distributed under their own terms
+(see LICENSE-3RD-PARTY.txt)
\ No newline at end of file

broker/src/asapo_broker/database/mongodb_query.go

@@ -5,7 +5,7 @@ package database
 import (
 	"errors"
 	"fmt"
-	"github.com/globalsign/mgo/bson"
+	"go.mongodb.org/mongo-driver/bson"
 	"github.com/knocknote/vitess-sqlparser/sqlparser"
 	"strconv"
 )

deploy/CMakeLists.txt

@@ -4,8 +4,8 @@ else()
     SET (NOMAD_INSTALL ${CMAKE_INSTALL_PREFIX}/nomad_jobs)
 endif()
 
-configure_files(${CMAKE_CURRENT_SOURCE_DIR}/asapo ${CMAKE_CURRENT_BINARY_DIR}/asapo)
-configure_files(${CMAKE_CURRENT_SOURCE_DIR}/asapo/scripts ${CMAKE_CURRENT_BINARY_DIR}/asapo/scripts)
+configure_files(${CMAKE_CURRENT_SOURCE_DIR}/asapo_services ${CMAKE_CURRENT_BINARY_DIR}/asapo_services)
+configure_files(${CMAKE_CURRENT_SOURCE_DIR}/asapo_services/scripts ${CMAKE_CURRENT_BINARY_DIR}/asapo_services/scripts)
 
 
 

asapo_client_template @ e7f7552c

+Subproject commit e7f7552c8a8672577787db52fb32e2a009e14c98

deploy/asapo_orchestration_docker/Dockerfile

+FROM yakser/asapo-nomad-cluster
+
+MAINTAINER DESY IT
+
+COPY scripts/ /var/run/asapo/
+
+RUN cd /var/run/asapo && terraform init

deploy/asapo_orchestration_docker/build_image.sh

+#!/usr/bin/env bash
+docker build -t yakser/asapo-orc .
+docker push yakser/asapo-orc
+

deploy/asapo_orchestration_docker/scripts/asapo.auto.tfvars

+job_scripts_dir = "/var/run/asapo"
+
+consul_dns_port = 8600
+
+grafana_total_memory_size = "256"
+grafana_port = 3000
+grafana_version="latest"
+
+nginx_total_memory_size = 256
+nginx_port = 8401
+nginx_port_stream = 8402
+nginx_version = "1.14"
+
+influxdb_total_memory_size = "256"
+influxdb_port = 8086
+influxdb_version = "latest"
+
+telegraf_total_memory_size = "256"
+telegraf_port_stream = 8125
+telegraf_version = "latest"
+
+monitoring = true

deploy/asapo_orchestration_docker/scripts/grafana/grafana.ini.tpl

+[server]
+http_port = {{ env "NOMAD_PORT_grafana" }}
+
+[auth.anonymous]
+enabled = true
+org_name = Main Org.
+org_role = Admin
+
+#[auth]
+#disable_login_form = true

deploy/asapo_orchestration_docker/scripts/grafana/provisioning/dashboards/dashboards.yaml

+apiVersion: 1
+
+providers:
+- name: 'client monitoring'
+  folder: ''
+  type: file
+  disableDeletion: false
+  editable: true
+  updateIntervalSeconds: 10000
+  allowUiUpdates: true
+  options:
+    path: /var/lib/grafana_config/dashboards
\ No newline at end of file

deploy/asapo_orchestration_docker/scripts/grafana/provisioning/datasources/datasources.yaml.tpl

+apiVersion: 1
+
+datasources:
+  - name: InfluxDB
+    type: influxdb
+    access: proxy
+    database: telegraf
+    url: http://localhost:{{ env "NOMAD_META_nginx_port" }}/influxdb
+    jsonData:
+      httpMode: GET

deploy/asapo_orchestration_docker/scripts/monitoring.nmd.tpl

+job "monitoring" {
+  datacenters = ["dc1"]
+  affinity {
+    attribute = "$${meta.asapo_service}"
+    value     = "true"
+    weight    = 100
+  }
+
+#  update {
+#    max_parallel = 1
+#    min_healthy_time = "10s"
+#    healthy_deadline = "3m"
+#    auto_revert = false
+#  }
+
+  group "monitoring" {
+    count = "%{ if monitoring }1%{ else }0%{ endif }"
+    restart {
+      attempts = 2
+      interval = "3m"
+      delay = "15s"
+      mode = "delay"
+    }
+
+     meta {
+          nginx_port = "${nginx_port}"
+      }
+
+    task "influxdb" {
+      driver = "docker"
+      user = "${asapo_user}"
+      config {
+        network_mode = "host"
+	    security_opt = ["no-new-privileges"]
+	    userns_mode = "host"
+        image = "influxdb:${influxdb_version}"
+        volumes = ["/${service_dir}/influxdb:/var/lib/influxdb"]
+      }
+
+      env {
+        PRE_CREATE_DB="asapo_receivers;asapo_brokers"
+      }
+
+      resources {
+        memory = "${influxdb_total_memory_size}"
+        network {
+          port "influxdb" {
+          static = "${influxdb_port}"
+          }
+        }
+      }
+
+     service {
+       port = "influxdb"
+       name = "influxdb"
+       check {
+           name = "alive"
+           type     = "http"
+           path     = "/ping"
+           interval = "10s"
+           timeout  = "1s"
+       }
+       check_restart {
+         limit = 2
+         grace = "90s"
+         ignore_warnings = false
+       }
+     }
+
+   } #influxdb
+
+
+    task "grafana" {
+      driver = "docker"
+      user = "${asapo_user}"
+      env {
+        GF_SERVER_DOMAIN = "$${attr.unique.hostname}"
+        GF_SERVER_ROOT_URL = "%(protocol)s://%(domain)s/monitoring/"
+        GF_PATHS_PROVISIONING = "/var/lib/grafana_config/provisioning"
+      }
+
+      config {
+        network_mode = "host"
+	    security_opt = ["no-new-privileges"]
+	    userns_mode = "host"
+        image = "grafana/grafana:${grafana_version}"
+        volumes = ["/${service_dir}/grafana:/var/lib/grafana",
+        "local/grafana:/var/lib/grafana_config",
+        "local/grafana/grafana.ini:/etc/grafana/grafana.ini"
+        ]
+      }
+
+      resources {
+        memory = "${grafana_total_memory_size}"
+        network {
+          port "grafana" {
+          static = "${grafana_port}"
+          }
+        }
+      }
+
+     service {
+       port = "grafana"
+       name = "grafana"
+       check {
+           name = "alive"
+           type     = "http"
+           path     = "/api/health"
+           interval = "10s"
+           timeout  = "1s"
+       }
+       check_restart {
+         limit = 2
+         grace = "90s"
+         ignore_warnings = false
+       }
+     }
+
+      template {
+         source        = "${scripts_dir}/grafana/grafana.ini.tpl"
+         destination   = "local/grafana/grafana.ini"
+         change_mode   = "restart"
+      }
+      template {
+         source        = "${scripts_dir}/grafana/provisioning/dashboards/dashboards.yaml"
+         destination   = "local/grafana/provisioning/dashboards/dashboards.yaml"
+         change_mode   = "restart"
+      }
+      template {
+         source        = "${scripts_dir}/grafana/provisioning/datasources/datasources.yaml.tpl"
+         destination   = "local/grafana/provisioning/datasources/datasources.yaml"
+         change_mode   = "restart"
+      }
+      template {
+         source        = "${scripts_dir}/grafana/dashboards/grafana.json"
+         destination   = "local/grafana/dashboards/grafana.json"
+         change_mode   = "restart"
+      }
+
+   } #grafana
+
+    task "telegraf" {
+      driver = "docker"
+      user = "${asapo_user}"
+       config {
+        network_mode = "host"
+	    security_opt = ["no-new-privileges"]
+	    userns_mode = "host"
+        image = "telegraf:${telegraf_version}"
+        volumes = [
+          "local/telegraf.conf:/etc/telegraf/telegraf.conf"
+        ]
+      }
+
+      resources {
+        memory = "${telegraf_total_memory_size}"
+        network {
+          port "telegraf_stream" {
+          static = "${telegraf_port_stream}"
+          }
+        }
+      }
+
+	  service {
+        name = "telegraf"
+        port = "telegraf_stream"
+        check {
+          name     = "telegraf-alive"
+          type     = "script"
+          command  = "/bin/ps"
+          args     = ["-fC","telegraf"]
+          interval = "10s"
+          timeout  = "2s"
+        }
+        check_restart {
+          limit = 2
+          grace = "15s"
+          ignore_warnings = false
+        }
+      }
+
+      template {
+         source        = "${scripts_dir}/telegraf.conf.tpl"
+         destination   = "local/telegraf.conf"
+         change_mode   = "restart"
+      }
+
+   } #telegraf
+
+  }
+}

deploy/asapo_orchestration_docker/scripts/nginx.conf.tpl

+worker_processes  1;
+
+events {
+    worker_connections  100000;
+}
+
+error_log         "/dev/stdout";
+pid               "/tmp/nginx.pid";
+
+http {
+#    include       mime.types;
+#    default_type  application/octet-stream;
+
+#    sendfile        on;
+#    tcp_nopush     on;
+
+#    keepalive_timeout  0;
+#    keepalive_timeout  65;
+
+	 access_log off;
+
+
+    client_body_temp_path  "/tmp/client_body" 1 2;
+    proxy_temp_path        "/tmp/proxy" 1 2;
+    fastcgi_temp_path      "/tmp/fastcgi" 1 2;
+    scgi_temp_path         "/tmp/scgi" 1 2;
+    uwsgi_temp_path        "/tmp/uwsgi" 1 2;
+
+
+    resolver 127.0.0.1:{{ env "NOMAD_META_consul_dns_port" }} valid=1s;
+    server {
+          listen {{ env "NOMAD_PORT_nginx" }} reuseport;
+          set $grafana_endpoint grafana.service.asapo;
+          set $influxdb_endpoint influxdb.service.asapo;
+
+   		  location /influxdb/ {
+            rewrite ^/influxdb(/.*) $1 break;
+            proxy_pass http://$influxdb_endpoint:{{ env "NOMAD_META_influxdb_port" }}$uri$is_args$args;
+          }
+
+           location /monitoring/ {
+            rewrite ^/monitoring(/.*) $1 break;
+            proxy_pass http://$grafana_endpoint:{{ env "NOMAD_META_grafana_port" }}$uri$is_args$args;
+          }
+
+      	  location /nginx-health {
+  	        return 200 "healthy\n";
+	      }
+    }
+}
+
+stream {
+    resolver 127.0.0.1:{{ env "NOMAD_META_consul_dns_port" }} valid=1s;
+
+    map $remote_addr $upstream {
+        default telegraf.service.asapo;
+    }
+
+    server {
+        listen {{ env "NOMAD_PORT_nginx_stream" }} udp;
+        proxy_pass $upstream:{{ env "NOMAD_META_telegraf_port_stream" }};
+    }
+}

deploy/asapo_orchestration_docker/scripts/nginx.nmd.tpl

+job "asapo-nginx" {
+  datacenters = ["dc1"]
+
+  type = "system"
+
+  group "nginx" {
+    count = 1
+
+    restart {
+      attempts = 2
+      interval = "3m"
+      delay = "16s"
+      mode = "delay"
+    }
+
+    task "nginx" {
+      driver = "docker"
+
+      user = "${asapo_user}"
+
+      meta {
+        telegraf_port_stream = "${telegraf_port_stream}"
+        grafana_port = "${grafana_port}"
+        influxdb_port = "${influxdb_port}"
+        consul_dns_port = "${consul_dns_port}"
+      }
+
+      config {
+        network_mode = "host"
+	    security_opt = ["no-new-privileges"]
+	    userns_mode = "host"
+        image = "nginx:${nginx_version}"
+        volumes = [
+          "local/nginx.conf:/etc/nginx/nginx.conf"
+        ]
+      }
+
+      resources {
+        cpu    = 500
+        memory = ${nginx_total_memory_size}
+        network {
+          mbits = 10
+          port "nginx" {
+          	static = ${nginx_port}
+          }
+          port "nginx_stream" {
+          	static = ${nginx_port_stream}
+          }
+        }
+      }
+
+      service {
+        port = "nginx"
+        name = "nginx"
+        check {
+          name     = "alive"
+          type     = "http"
+	      path     = "/nginx-health"
+          timeout  = "2s"
+	      interval = "10s"
+        }
+
+        check_restart {
+          limit = 2
+          grace = "15s"
+          ignore_warnings = false
+        }
+      }
+
+      template {
+         source        = "${scripts_dir}/nginx.conf.tpl"
+         destination   = "local/nginx.conf"
+         change_mode   = "restart"
+      }
+   }
+  }
+}

deploy/asapo_orchestration_docker/scripts/provider.tf

+provider "nomad" {
+  address = "http://localhost:4646"
+#  secret_id = "${chomp(file("/var/nomad/token"))}"
+}
+

deploy/asapo_orchestration_docker/scripts/resources.tf

+resource "nomad_job" "monitoring" {
+  jobspec = "${data.template_file.monitoring_template.rendered}"
+}
+
+resource "nomad_job" "nginx" {
+  jobspec = "${data.template_file.nginx_template.rendered}"
+}