From edd514b0c1c2f70f6cf681c600b5aa29ae9ebc21 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon, 16 Dec 2019 10:46:49 +0100
Subject: [PATCH] update test, move asapo client template to submodule
---
.gitmodules | 3 +
deploy/asapo_client_template | 1 +
deploy/asapo_client_template/Dockerfile | 6 -
.../asapo_client_overwrite_vars.tfvars | 5 -
deploy/asapo_client_template/build_image.sh | 3 -
deploy/asapo_client_template/job.sh | 7 -
deploy/asapo_client_template/run.sh | 44 -
deploy/asapo_client_template/run_maxwell.sh | 85 --
.../scripts/grafana/dashboards/grafana.json | 1197 -----------------
.../scripts/job.auto.tfvars | 3 -
.../scripts/job_resources.tf | 7 -
.../scripts/job_templates.tf | 18 -
.../asapo_client_template/scripts/job_vars.tf | 3 -
.../scripts/stress_cpu.nmd.tpl | 42 -
.../scripts/stress_memory.nmd.tpl | 39 -
.../consumer_api_python/consumer_api.py | 2 +-
tests/automatic/settings/nginx.conf.tpl | 2 +-
17 files changed, 6 insertions(+), 1461 deletions(-)
create mode 100644 .gitmodules
create mode 160000 deploy/asapo_client_template
delete mode 100644 deploy/asapo_client_template/Dockerfile
delete mode 100644 deploy/asapo_client_template/asapo_client_overwrite_vars.tfvars
delete mode 100755 deploy/asapo_client_template/build_image.sh
delete mode 100755 deploy/asapo_client_template/job.sh
delete mode 100755 deploy/asapo_client_template/run.sh
delete mode 100755 deploy/asapo_client_template/run_maxwell.sh
delete mode 100644 deploy/asapo_client_template/scripts/grafana/dashboards/grafana.json
delete mode 100644 deploy/asapo_client_template/scripts/job.auto.tfvars
delete mode 100644 deploy/asapo_client_template/scripts/job_resources.tf
delete mode 100644 deploy/asapo_client_template/scripts/job_templates.tf
delete mode 100644 deploy/asapo_client_template/scripts/job_vars.tf
delete mode 100644 deploy/asapo_client_template/scripts/stress_cpu.nmd.tpl
delete mode 100644 deploy/asapo_client_template/scripts/stress_memory.nmd.tpl
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..480842551
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "deploy/asapo-client-template"]
+ path = deploy/asapo_client_template
+ url = ssh://yakubov@stash.desy.de:7999/asapo/asapo-client-template.git
diff --git a/deploy/asapo_client_template b/deploy/asapo_client_template
new file mode 160000
index 000000000..0378d53c9
--- /dev/null
+++ b/deploy/asapo_client_template
@@ -0,0 +1 @@
+Subproject commit 0378d53c95ce268d0f715a0377f4fd0e70b60a7d
diff --git a/deploy/asapo_client_template/Dockerfile b/deploy/asapo_client_template/Dockerfile
deleted file mode 100644
index e0b164b97..000000000
--- a/deploy/asapo_client_template/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-FROM yakser/asapo-orc
-
-MAINTAINER DESY IT
-
-COPY scripts/ /var/run/asapo/
-
diff --git a/deploy/asapo_client_template/asapo_client_overwrite_vars.tfvars b/deploy/asapo_client_template/asapo_client_overwrite_vars.tfvars
deleted file mode 100644
index bd5fde5a5..000000000
--- a/deploy/asapo_client_template/asapo_client_overwrite_vars.tfvars
+++ /dev/null
@@ -1,5 +0,0 @@
-# uncomment to change
-
-#stress_cpu_n_cpus = 1
-#stress_memory_total_memory_size = 250
-#stress_memory_alloc_size_mb = 200
diff --git a/deploy/asapo_client_template/build_image.sh b/deploy/asapo_client_template/build_image.sh
deleted file mode 100755
index 70afada2f..000000000
--- a/deploy/asapo_client_template/build_image.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-docker build -t yakser/asapo-client-template .
-
diff --git a/deploy/asapo_client_template/job.sh b/deploy/asapo_client_template/job.sh
deleted file mode 100755
index b809a675f..000000000
--- a/deploy/asapo_client_template/job.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-#SBATCH --nodes=1
-#SBATCH -t 00:40:00
-
-srun --ntasks=$SLURM_JOB_NUM_NODES --ntasks-per-node=1 ./run_maxwell.sh
-
diff --git a/deploy/asapo_client_template/run.sh b/deploy/asapo_client_template/run.sh
deleted file mode 100755
index 1858a5e3a..000000000
--- a/deploy/asapo_client_template/run.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-
-NOMAD_ALLOC_HOST_SHARED=$HOME/asapo_client/container_host_shared/nomad_alloc
-SERVICE_DATA_CLUSTER_SHARED=$HOME/asapo_client/cluster_shared/service_data
-DATA_GLOBAL_SHARED=$HOME/asapo_client/global_shared/data
-
-ASAPO_USER=`id -u`:`id -g`
-
-NOMAD_TELEMETRY=true
-NGINX_PORT_STREAM=8402
-TELEGRAF_ADDRESS="127.0.0.1:${NGINX_PORT_STREAM}"
-
-ASAPO_VAR_FILE=`pwd`/asapo_client_overwrite_vars.tfvars
-
-mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED
-chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED
-
-cd $SERVICE_DATA_CLUSTER_SHARED
-mkdir grafana influxdb
-chmod 777 *
-
-if [ -f $ASAPO_VAR_FILE ]; then
- chmod 666 $ASAPO_VAR_FILE
- MOUNT_VAR_FILE="-v $ASAPO_VAR_FILE:/var/run/asapo/user_vars.tfvars"
-fi
-
-docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \
- -u $ASAPO_USER \
- --group-add `getent group docker | cut -d: -f3` \
- -v /var/lib/docker:/var/lib/docker \
- -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \
- -v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \
- -v $DATA_GLOBAL_SHARED:$DATA_GLOBAL_SHARED \
- -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \
- -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \
- -e TF_VAR_data_dir=$DATA_GLOBAL_SHARED \
- -e TF_VAR_nginx_port_stream=$NGINX_PORT_STREAM \
- $MOUNT_VAR_FILE \
- -e TF_VAR_asapo_user=$ASAPO_USER \
- -e TELEGRAF_ADDRESS=$TELEGRAF_ADDRESS \
- -e NOMAD_TELEMETRY=$NOMAD_TELEMETRY \
- --name asapo-client --net=host -d yakser/asapo-client-template
-
-sleep 5
\ No newline at end of file
diff --git a/deploy/asapo_client_template/run_maxwell.sh b/deploy/asapo_client_template/run_maxwell.sh
deleted file mode 100755
index ba02c1078..000000000
--- a/deploy/asapo_client_template/run_maxwell.sh
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/usr/bin/env bash
-
-#folders
-NOMAD_ALLOC_HOST_SHARED=/var/tmp/asapo_client/container_host_shared/nomad_alloc
-SERVICE_DATA_CLUSTER_SHARED=/home/yakubov/asapo/asapo_client_cluster_shared/service_data
-DATA_GLOBAL_SHARED=/gpfs/petra3/scratch/yakubov/asapo_client_shared
-MONGO_DIR=/scratch/mongodb # due to performance reasons mongodb can benefit from writing to local filesystem (HA to be worked on)
-#service distribution
-MAX_NOMAD_SERVERS=3 # rest are clients
-N_ASAPO_LIGHTWEIGHT_SERVICE_NODES=1 # where to put influx, elk, ... . Rest are receivers, brokers, mongodb
-
-#DESY stuff
-RECURSORS=["\"131.169.40.200\"",\""131.169.194.200\""]
-
-ASAPO_USER=`id -u`:`id -g`
-
-ASAPO_VAR_FILE=`pwd`/asapo_overwrite_vars.tfvars
-
-
-#docker stuff
-DOCKER_ENDPOINT="127.0.0.1:2376" #comment to use unix sockets
-DOCKER_TLS_CA=/data/netapp/docker/certs/ca.pem
-DOCKER_TLS_KEY=/data/netapp/docker/certs/$USER/key.pem
-DOCKER_TLS_CERT=/data/netapp/docker/certs/$USER/cert.pem
-
-#adresses to use
-USE_IB_FOR_RECEIVER=true
-if [ "$USE_IB_FOR_RECEIVER" == "true" ]; then
- IB_HOSTNAME=`hostname --short`-ib
- IB_ADDRESS=`getent hosts $IB_HOSTNAME | awk '{ print $1 }'`
-fi
-#ADVERTISE_IP= #set if differs from default
-
-#prepare env variables based on the above input
-N_SERVERS=$(( $SLURM_JOB_NUM_NODES > $MAX_NOMAD_SERVERS ? $MAX_NOMAD_SERVERS : $SLURM_JOB_NUM_NODES ))
-
-SERVER_ADRESSES=`scontrol show hostnames $SLURM_JOB_NODELIST | head -$N_SERVERS | awk 'BEGIN{printf "["} {printf "%s\"%s\"",sep,$0; sep=","} END{print "]"}'`
-ASAPO_LIGHTWEIGHT_SERVICE_NODES=`scontrol show hostnames $SLURM_JOB_NODELIST | head -$N_ASAPO_LIGHTWEIGHT_SERVICE_NODES | awk 'BEGIN{printf "["} {printf "%s\"%s\"",sep,$0; sep=","} END{print "]"}'`
-
-# make folders if not exist
-mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED $MONGO_DIR
-chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED $MONGO_DIR
-cd $SERVICE_DATA_CLUSTER_SHARED
-mkdir esdatadir fluentd grafana influxdb mongodb
-chmod 777 *
-
-#todo: elastic search check
-mmc=`cat /proc/sys/vm/max_map_count`
-if (( mmc < 262144 )); then
- echo consider increasing max_map_count - needed for elasticsearch
-# exit 1
-fi
-
-docker rm -f asapo
-
-docker pull yakser/asapo-client
-
-if [ -f $ASAPO_VAR_FILE ]; then
- MOUNT_VAR_FILE="-v $ASAPO_VAR_FILE:/var/run/asapo/user_vars.tfvars"
-fi
-
-dockerrun --rm \
- -u $ASAPO_USER \
- -v /scratch/docker/100000.100000:/scratch/docker/100000.100000 \
- -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \
- -v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \
- -v $DOCKER_TLS_CA:/etc/nomad/ca.pem \
- -v $DOCKER_TLS_KEY:/etc/nomad/key.pem \
- -v $DOCKER_TLS_CERT:/etc/nomad/cert.pem \
- -v $DATA_GLOBAL_SHARED:$DATA_GLOBAL_SHARED \
- $MOUNT_VAR_FILE \
- -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \
- -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \
- -e TF_VAR_data_dir=$DATA_GLOBAL_SHARED \
- -e TF_VAR_mongo_dir=$MONGO_DIR \
- -e ADVERTISE_IP=$ADVERTISE_IP \
- -e RECURSORS=$RECURSORS \
- -e TF_VAR_asapo_user=$ASAPO_USER \
- -e IB_ADDRESS=$IB_ADDRESS \
- -e SERVER_ADRESSES=$SERVER_ADRESSES \
- -e ASAPO_LIGHTWEIGHT_SERVICE_NODES=$ASAPO_LIGHTWEIGHT_SERVICE_NODES \
- -e DOCKER_ENDPOINT=$DOCKER_ENDPOINT \
- -e N_SERVERS=$N_SERVERS \
- --name asapo yakser/asapo-client-template
-
diff --git a/deploy/asapo_client_template/scripts/grafana/dashboards/grafana.json b/deploy/asapo_client_template/scripts/grafana/dashboards/grafana.json
deleted file mode 100644
index 39d39dc34..000000000
--- a/deploy/asapo_client_template/scripts/grafana/dashboards/grafana.json
+++ /dev/null
@@ -1,1197 +0,0 @@
-{
- "annotations": {
- "list": [
- {
- "builtIn": 1,
- "datasource": "-- Grafana --",
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard"
- }
- ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 0,
- "id": 1,
- "iteration": 1576248483218,
- "links": [],
- "panels": [
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "InfluxDB",
- "fill": 1,
- "fillGradient": 0,
- "gridPos": {
- "h": 8,
- "w": 22,
- "x": 0,
- "y": 0
- },
- "hiddenSeries": false,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "groupBy": [
- {
- "params": [
- "$__interval"
- ],
- "type": "time"
- },
- {
- "params": [
- "linear"
- ],
- "type": "fill"
- }
- ],
- "measurement": "nomad_client_allocs",
- "orderByTime": "ASC",
- "policy": "default",
- "refId": "A",
- "resultFormat": "time_series",
- "select": [
- [
- {
- "params": [
- "memory_rss"
- ],
- "type": "field"
- },
- {
- "params": [],
- "type": "sum"
- }
- ]
- ],
- "tags": [
- {
- "key": "task",
- "operator": "=~",
- "value": "/^$task$/"
- }
- ]
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
- "title": "Total Memory Usage",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "decbytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
- },
- {
- "cacheTimeout": null,
- "colorBackground": true,
- "colorPostfix": false,
- "colorPrefix": false,
- "colorValue": false,
- "colors": [
- "#d44a3a",
- "rgba(237, 129, 40, 0.89)",
- "#299c46"
- ],
- "datasource": "InfluxDB",
- "description": "",
- "format": "none",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "gridPos": {
- "h": 5,
- "w": 4,
- "x": 1,
- "y": 8
- },
- "id": 13,
- "interval": null,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "options": {},
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false,
- "ymax": null,
- "ymin": null
- },
- "tableColumn": "",
- "targets": [
- {
- "groupBy": [
- {
- "params": [
- "$__interval"
- ],
- "type": "time"
- },
- {
- "params": [
- "null"
- ],
- "type": "fill"
- }
- ],
- "measurement": "consul_health_checks",
- "orderByTime": "ASC",
- "policy": "default",
- "refId": "A",
- "resultFormat": "time_series",
- "select": [
- [
- {
- "params": [
- "passing"
- ],
- "type": "field"
- },
- {
- "params": [],
- "type": "mean"
- }
- ]
- ],
- "tags": [
- {
- "key": "service_name",
- "operator": "=",
- "value": "telegraf"
- }
- ]
- }
- ],
- "thresholds": "1,1",
- "timeFrom": "20s",
- "timeShift": null,
- "title": "Telegraf",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "Not running",
- "value": "0"
- },
- {
- "op": "=",
- "text": "Running",
- "value": "1"
- },
- {
- "op": "=",
- "text": "",
- "value": "null"
- }
- ],
- "valueName": "current"
- },
- {
- "cacheTimeout": null,
- "colorBackground": true,
- "colorPostfix": false,
- "colorPrefix": false,
- "colorValue": false,
- "colors": [
- "#d44a3a",
- "rgba(237, 129, 40, 0.89)",
- "#299c46"
- ],
- "datasource": "InfluxDB",
- "description": "",
- "format": "none",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "gridPos": {
- "h": 5,
- "w": 4,
- "x": 5,
- "y": 8
- },
- "id": 15,
- "interval": null,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "options": {},
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false,
- "ymax": null,
- "ymin": null
- },
- "tableColumn": "",
- "targets": [
- {
- "groupBy": [
- {
- "params": [
- "$__interval"
- ],
- "type": "time"
- },
- {
- "params": [
- "null"
- ],
- "type": "fill"
- }
- ],
- "measurement": "consul_health_checks",
- "orderByTime": "ASC",
- "policy": "default",
- "refId": "A",
- "resultFormat": "time_series",
- "select": [
- [
- {
- "params": [
- "passing"
- ],
- "type": "field"
- },
- {
- "params": [],
- "type": "mean"
- }
- ]
- ],
- "tags": [
- {
- "key": "service_name",
- "operator": "=",
- "value": "influxdb"
- }
- ]
- }
- ],
- "thresholds": "1,1",
- "timeFrom": "20s",
- "timeShift": null,
- "title": "InfluxDB",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "Not running",
- "value": "0"
- },
- {
- "op": "=",
- "text": "Running",
- "value": "1"
- },
- {
- "op": "=",
- "text": "",
- "value": "null"
- }
- ],
- "valueName": "current"
- },
- {
- "cacheTimeout": null,
- "colorBackground": true,
- "colorPostfix": false,
- "colorPrefix": false,
- "colorValue": false,
- "colors": [
- "#d44a3a",
- "rgba(237, 129, 40, 0.89)",
- "#299c46"
- ],
- "datasource": "InfluxDB",
- "description": "",
- "format": "none",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "gridPos": {
- "h": 5,
- "w": 4,
- "x": 9,
- "y": 8
- },
- "id": 17,
- "interval": null,
- "links": [],
- "mappingType": 2,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "options": {},
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- },
- {
- "from": "0",
- "text": "",
- "to": "0"
- }
- ],
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false,
- "ymax": null,
- "ymin": null
- },
- "tableColumn": "",
- "targets": [
- {
- "groupBy": [
- {
- "params": [
- "$__interval"
- ],
- "type": "time"
- },
- {
- "params": [
- "null"
- ],
- "type": "fill"
- }
- ],
- "measurement": "consul_health_checks",
- "orderByTime": "ASC",
- "policy": "default",
- "refId": "A",
- "resultFormat": "time_series",
- "select": [
- [
- {
- "params": [
- "passing"
- ],
- "type": "field"
- },
- {
- "params": [],
- "type": "sum"
- }
- ]
- ],
- "tags": [
- {
- "key": "service_name",
- "operator": "=",
- "value": "nginx"
- }
- ]
- }
- ],
- "thresholds": "1,1",
- "timeFrom": "20s",
- "timeShift": null,
- "title": "Nginx",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "Not running",
- "value": "0"
- },
- {
- "op": "=",
- "text": "Running",
- "value": "1"
- },
- {
- "op": "=",
- "text": "",
- "value": "null"
- }
- ],
- "valueName": "total"
- },
- {
- "cacheTimeout": null,
- "colorBackground": true,
- "colorPostfix": false,
- "colorPrefix": false,
- "colorValue": false,
- "colors": [
- "#d44a3a",
- "rgba(237, 129, 40, 0.89)",
- "#299c46"
- ],
- "datasource": "InfluxDB",
- "description": "",
- "format": "none",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "gridPos": {
- "h": 5,
- "w": 4,
- "x": 14,
- "y": 8
- },
- "id": 14,
- "interval": null,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "options": {},
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false,
- "ymax": null,
- "ymin": null
- },
- "tableColumn": "",
- "targets": [
- {
- "groupBy": [
- {
- "params": [
- "$__interval"
- ],
- "type": "time"
- },
- {
- "params": [
- "null"
- ],
- "type": "fill"
- }
- ],
- "measurement": "consul_health_checks",
- "orderByTime": "ASC",
- "policy": "default",
- "refId": "A",
- "resultFormat": "time_series",
- "select": [
- [
- {
- "params": [
- "passing"
- ],
- "type": "field"
- },
- {
- "params": [],
- "type": "mean"
- }
- ]
- ],
- "tags": [
- {
- "key": "service_name",
- "operator": "=",
- "value": "grafana"
- }
- ]
- }
- ],
- "thresholds": "1,1",
- "timeFrom": "20s",
- "timeShift": null,
- "title": "Grafana",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "Not running",
- "value": "0"
- },
- {
- "op": "=",
- "text": "Running",
- "value": "1"
- },
- {
- "op": "=",
- "text": "",
- "value": "null"
- }
- ],
- "valueName": "current"
- },
- {
- "cacheTimeout": null,
- "colorBackground": true,
- "colorPostfix": false,
- "colorPrefix": false,
- "colorValue": false,
- "colors": [
- "#d44a3a",
- "rgba(237, 129, 40, 0.89)",
- "#299c46"
- ],
- "datasource": "InfluxDB",
- "description": "",
- "format": "none",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "gridPos": {
- "h": 5,
- "w": 4,
- "x": 1,
- "y": 13
- },
- "id": 18,
- "interval": null,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "options": {},
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false,
- "ymax": null,
- "ymin": null
- },
- "tableColumn": "",
- "targets": [
- {
- "groupBy": [
- {
- "params": [
- "$__interval"
- ],
- "type": "time"
- },
- {
- "params": [
- "null"
- ],
- "type": "fill"
- }
- ],
- "measurement": "consul_health_checks",
- "orderByTime": "ASC",
- "policy": "default",
- "refId": "A",
- "resultFormat": "time_series",
- "select": [
- [
- {
- "params": [
- "passing"
- ],
- "type": "field"
- },
- {
- "params": [],
- "type": "mean"
- }
- ]
- ],
- "tags": [
- {
- "key": "service_name",
- "operator": "=",
- "value": "stress-cpu"
- }
- ]
- }
- ],
- "thresholds": "1,1",
- "timeFrom": "20s",
- "timeShift": null,
- "title": "Stress CPU",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "Not running",
- "value": "0"
- },
- {
- "op": "=",
- "text": "Running",
- "value": "1"
- },
- {
- "op": "=",
- "text": "",
- "value": "null"
- }
- ],
- "valueName": "current"
- },
- {
- "datasource": "InfluxDB",
- "gridPos": {
- "h": 5,
- "w": 4,
- "x": 5,
- "y": 13
- },
- "hideTimeOverride": true,
- "id": 9,
- "options": {
- "fieldOptions": {
- "calcs": [
- "mean"
- ],
- "defaults": {
- "mappings": [],
- "max": 1600,
- "min": 0,
- "thresholds": [
- {
- "color": "green",
- "value": null
- }
- ],
- "title": "",
- "unit": "percent"
- },
- "override": {},
- "values": false
- },
- "orientation": "horizontal",
- "showThresholdLabels": false,
- "showThresholdMarkers": true
- },
- "pluginVersion": "6.5.2",
- "targets": [
- {
- "groupBy": [
- {
- "params": [
- "$__interval"
- ],
- "type": "time"
- },
- {
- "params": [
- "null"
- ],
- "type": "fill"
- }
- ],
- "measurement": "nomad_client_allocs",
- "orderByTime": "ASC",
- "policy": "default",
- "refId": "A",
- "resultFormat": "time_series",
- "select": [
- [
- {
- "params": [
- "cpu_total_percent"
- ],
- "type": "field"
- },
- {
- "params": [],
- "type": "max"
- }
- ]
- ],
- "tags": [
- {
- "key": "task",
- "operator": "=",
- "value": "stress-cpu"
- }
- ]
- }
- ],
- "timeFrom": "20s",
- "timeShift": null,
- "title": "CPU Usage",
- "transparent": true,
- "type": "gauge"
- },
- {
- "cacheTimeout": null,
- "colorBackground": true,
- "colorPostfix": false,
- "colorPrefix": false,
- "colorValue": false,
- "colors": [
- "#d44a3a",
- "rgba(237, 129, 40, 0.89)",
- "#299c46"
- ],
- "datasource": "InfluxDB",
- "description": "",
- "format": "none",
- "gauge": {
- "maxValue": 100,
- "minValue": 0,
- "show": false,
- "thresholdLabels": false,
- "thresholdMarkers": true
- },
- "gridPos": {
- "h": 5,
- "w": 4,
- "x": 1,
- "y": 18
- },
- "id": 19,
- "interval": null,
- "links": [],
- "mappingType": 1,
- "mappingTypes": [
- {
- "name": "value to text",
- "value": 1
- },
- {
- "name": "range to text",
- "value": 2
- }
- ],
- "maxDataPoints": 100,
- "nullPointMode": "connected",
- "nullText": null,
- "options": {},
- "postfix": "",
- "postfixFontSize": "50%",
- "prefix": "",
- "prefixFontSize": "50%",
- "rangeMaps": [
- {
- "from": "null",
- "text": "N/A",
- "to": "null"
- }
- ],
- "sparkline": {
- "fillColor": "rgba(31, 118, 189, 0.18)",
- "full": false,
- "lineColor": "rgb(31, 120, 193)",
- "show": false,
- "ymax": null,
- "ymin": null
- },
- "tableColumn": "",
- "targets": [
- {
- "groupBy": [
- {
- "params": [
- "$__interval"
- ],
- "type": "time"
- },
- {
- "params": [
- "null"
- ],
- "type": "fill"
- }
- ],
- "measurement": "consul_health_checks",
- "orderByTime": "ASC",
- "policy": "default",
- "refId": "A",
- "resultFormat": "time_series",
- "select": [
- [
- {
- "params": [
- "passing"
- ],
- "type": "field"
- },
- {
- "params": [],
- "type": "mean"
- }
- ]
- ],
- "tags": [
- {
- "key": "service_name",
- "operator": "=",
- "value": "stress-memory"
- }
- ]
- }
- ],
- "thresholds": "1,1",
- "timeFrom": "20s",
- "timeShift": null,
- "title": "Stress Memory",
- "type": "singlestat",
- "valueFontSize": "80%",
- "valueMaps": [
- {
- "op": "=",
- "text": "Not running",
- "value": "0"
- },
- {
- "op": "=",
- "text": "Running",
- "value": "1"
- },
- {
- "op": "=",
- "text": "",
- "value": "null"
- }
- ],
- "valueName": "current"
- },
- {
- "datasource": "InfluxDB",
- "gridPos": {
- "h": 5,
- "w": 4,
- "x": 5,
- "y": 18
- },
- "hideTimeOverride": true,
- "id": 4,
- "options": {
- "fieldOptions": {
- "calcs": [
- "mean"
- ],
- "defaults": {
- "mappings": [],
- "max": 100,
- "min": 0,
- "thresholds": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ],
- "title": "",
- "unit": "percent"
- },
- "override": {},
- "values": false
- },
- "orientation": "horizontal",
- "showThresholdLabels": false,
- "showThresholdMarkers": true
- },
- "pluginVersion": "6.5.2",
- "targets": [
- {
- "groupBy": [
- {
- "params": [
- "$__interval"
- ],
- "type": "time"
- },
- {
- "params": [
- "null"
- ],
- "type": "fill"
- }
- ],
- "measurement": "nomad_client_allocs",
- "orderByTime": "ASC",
- "policy": "default",
- "query": "SELECT mean(\"memory_rss\") /mean([[memory_allocated]]) * 100 FROM \"nomad_client_allocs\" WHERE (\"task\" = 'grafana') AND $timeFilter GROUP BY time($__interval) fill(null)",
- "rawQuery": false,
- "refId": "A",
- "resultFormat": "time_series",
- "select": [
- [
- {
- "params": [
- "memory_rss"
- ],
- "type": "field"
- },
- {
- "params": [],
- "type": "mean"
- },
- {
- "params": [
- "/mean(\"memory_allocated\") * 100"
- ],
- "type": "math"
- }
- ]
- ],
- "tags": [
- {
- "key": "task",
- "operator": "=",
- "value": "stress-memory"
- }
- ]
- }
- ],
- "timeFrom": "20s",
- "timeShift": null,
- "title": "Memory Usage",
- "transparent": true,
- "type": "gauge"
- }
- ],
- "refresh": "10s",
- "schemaVersion": 21,
- "style": "dark",
- "tags": [],
- "templating": {
- "list": [
- {
- "allValue": null,
- "current": {
- "text": "All",
- "value": [
- "$__all"
- ]
- },
- "datasource": "InfluxDB",
- "definition": "SHOW TAG VALUES WITH KEY = \"task\"",
- "hide": 0,
- "includeAll": true,
- "label": null,
- "multi": true,
- "name": "task",
- "options": [],
- "query": "SHOW TAG VALUES WITH KEY = \"task\"",
- "refresh": 2,
- "regex": "",
- "skipUrlSync": false,
- "sort": 5,
- "tagValuesQuery": "",
- "tags": [],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-30m",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ]
- },
- "timezone": "",
- "title": "Asapo Client Monitor",
- "uid": "acm",
- "version": 7
-}
diff --git a/deploy/asapo_client_template/scripts/job.auto.tfvars b/deploy/asapo_client_template/scripts/job.auto.tfvars
deleted file mode 100644
index 660ed776e..000000000
--- a/deploy/asapo_client_template/scripts/job.auto.tfvars
+++ /dev/null
@@ -1,3 +0,0 @@
-stress_cpu_n_cpus = 1
-stress_memory_total_memory_size = 250
-stress_memory_alloc_size_mb = 200
diff --git a/deploy/asapo_client_template/scripts/job_resources.tf b/deploy/asapo_client_template/scripts/job_resources.tf
deleted file mode 100644
index 9fb597aa1..000000000
--- a/deploy/asapo_client_template/scripts/job_resources.tf
+++ /dev/null
@@ -1,7 +0,0 @@
-resource "nomad_job" "stress_cpu" {
- jobspec = "${data.template_file.stress_cpu_template.rendered}"
-}
-
-resource "nomad_job" "stress_memory" {
- jobspec = "${data.template_file.stress_memory_template.rendered}"
-}
diff --git a/deploy/asapo_client_template/scripts/job_templates.tf b/deploy/asapo_client_template/scripts/job_templates.tf
deleted file mode 100644
index 742f45f07..000000000
--- a/deploy/asapo_client_template/scripts/job_templates.tf
+++ /dev/null
@@ -1,18 +0,0 @@
-data "template_file" "stress_cpu_template" {
- template = "${file("${var.job_scripts_dir}/stress_cpu.nmd.tpl")}"
- vars = {
- asapo_user = "${var.asapo_user}"
- data_dir = "${var.data_dir}"
- n_cpus = "${var.stress_cpu_n_cpus}"
- }
-}
-
-data "template_file" "stress_memory_template" {
- template = "${file("${var.job_scripts_dir}/stress_memory.nmd.tpl")}"
- vars = {
- asapo_user = "${var.asapo_user}"
- total_memory_size = "${var.stress_memory_total_memory_size}"
- alloc_size_mb = "${var.stress_memory_alloc_size_mb}"
- }
-}
-
diff --git a/deploy/asapo_client_template/scripts/job_vars.tf b/deploy/asapo_client_template/scripts/job_vars.tf
deleted file mode 100644
index ac23f4b91..000000000
--- a/deploy/asapo_client_template/scripts/job_vars.tf
+++ /dev/null
@@ -1,3 +0,0 @@
-variable "stress_cpu_n_cpus" {}
-variable "stress_memory_total_memory_size" {}
-variable "stress_memory_alloc_size_mb" {}
diff --git a/deploy/asapo_client_template/scripts/stress_cpu.nmd.tpl b/deploy/asapo_client_template/scripts/stress_cpu.nmd.tpl
deleted file mode 100644
index 8be555b90..000000000
--- a/deploy/asapo_client_template/scripts/stress_cpu.nmd.tpl
+++ /dev/null
@@ -1,42 +0,0 @@
-job "stress-cpu" {
- datacenters = ["dc1"]
-
- type = "system"
-
- group "stress-cpu" {
- count = 1
-
- task "stress-cpu" {
- driver = "docker"
- user = "${asapo_user}"
-
- config {
- network_mode = "host"
- security_opt = ["no-new-privileges"]
- userns_mode = "host"
- image = "progrium/stress"
- args = ["--cpu","${n_cpus}"]
- volumes = [
- "${data_dir}:/data"
- ]
- }
-
- resources {
- cpu = 500
- }
-
- service {
- name = "stress-cpu"
- check {
- name = "stress-cpu-alive"
- type = "script"
- command = "echo"
- args = ["alive"]
- interval = "10s"
- timeout = "2s"
- }
- }
-
- } #task
- } #group
-} #job
diff --git a/deploy/asapo_client_template/scripts/stress_memory.nmd.tpl b/deploy/asapo_client_template/scripts/stress_memory.nmd.tpl
deleted file mode 100644
index cb7d15f23..000000000
--- a/deploy/asapo_client_template/scripts/stress_memory.nmd.tpl
+++ /dev/null
@@ -1,39 +0,0 @@
-job "stress-memory" {
- datacenters = ["dc1"]
-
- type = "system"
-
- group "stress-memory" {
- count = 1
-
- task "stress-memory" {
- driver = "docker"
- user = "${asapo_user}"
-
- config {
- network_mode = "host"
- security_opt = ["no-new-privileges"]
- userns_mode = "host"
- image = "progrium/stress"
- args = ["--vm","1","--vm-bytes","${alloc_size_mb}M","--vm-hang","30"]
- }
-
- resources {
- memory = ${total_memory_size}
- }
-
- service {
- name = "stress-memory"
- check {
- name = "stress-memory-alive"
- type = "script"
- command = "echo"
- args = ["alive"]
- interval = "10s"
- timeout = "2s"
- }
- }
-
- } #task
- } #group
-} #job
diff --git a/tests/automatic/consumer/consumer_api_python/consumer_api.py b/tests/automatic/consumer/consumer_api_python/consumer_api.py
index a41756a87..f706f11ec 100644
--- a/tests/automatic/consumer/consumer_api_python/consumer_api.py
+++ b/tests/automatic/consumer/consumer_api_python/consumer_api.py
@@ -187,7 +187,7 @@ def check_dataset(broker,group_id_new):
source, path, beamtime, token, mode = sys.argv[1:]
-broker = asapo_consumer.create_server_broker(source,path, beamtime,"",token,1000)
+broker = asapo_consumer.create_server_broker(source,path, beamtime,"",token,5000)
group_id_new = broker.generate_group_id()
diff --git a/tests/automatic/settings/nginx.conf.tpl b/tests/automatic/settings/nginx.conf.tpl
index 7d6c84e2c..9fa1ba8ee 100644
--- a/tests/automatic/settings/nginx.conf.tpl
+++ b/tests/automatic/settings/nginx.conf.tpl
@@ -16,7 +16,7 @@ http {
resolver 127.0.0.1:8600 valid=1s;
server {
- listen {{ env "NOMAD_PORT_nginx" }};
+ listen {{ env "NOMAD_PORT_nginx" }} reuseport;
set $discovery_endpoint discovery.service.asapo;
set $authorizer_endpoint authorizer.service.asapo;
set $fluentd_endpoint fluentd.service.asapo;
--
GitLab