From c1e793f2e78763e994201fbfb899cff2e516feaf Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Tue, 24 Apr 2018 15:19:00 +0200
Subject: [PATCH] added manual script for fullchain, grafana dashboard file
---
config/grafana/ASAP__O.json | 485 ++++++++++++++++++
.../performance_full_chain_simple/broker.json | 6 +
.../receiver.json | 9 +
.../performance_full_chain_simple/test.sh | 98 ++++
.../receiver.json | 7 +-
.../settings_tmp.json | 6 -
.../performance_producer_receiver/test.sh | 4 +-
7 files changed, 606 insertions(+), 9 deletions(-)
create mode 100644 config/grafana/ASAP__O.json
create mode 100644 tests/manual/performance_full_chain_simple/broker.json
create mode 100644 tests/manual/performance_full_chain_simple/receiver.json
create mode 100755 tests/manual/performance_full_chain_simple/test.sh
delete mode 100644 tests/manual/performance_producer_receiver/settings_tmp.json
diff --git a/config/grafana/ASAP__O.json b/config/grafana/ASAP__O.json
new file mode 100644
index 000000000..c8fcc2121
--- /dev/null
+++ b/config/grafana/ASAP__O.json
@@ -0,0 +1,485 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_TEST",
+ "label": "test",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "influxdb",
+ "pluginName": "InfluxDB"
+ }
+ ],
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "5.0.0-beta5"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph",
+ "version": "5.0.0"
+ },
+ {
+ "type": "datasource",
+ "id": "influxdb",
+ "name": "InfluxDB",
+ "version": "5.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST}",
+ "fill": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 0
+ },
+ "id": 6,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "alias": "Database",
+ "groupBy": [],
+ "measurement": "statistics",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "query": "SELECT \"db_share\" FROM \"statistics\" WHERE $timeFilter",
+ "rawQuery": false,
+ "refId": "A",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "db_share"
+ ],
+ "type": "field"
+ }
+ ]
+ ],
+ "tags": []
+ },
+ {
+ "alias": "Disk",
+ "groupBy": [],
+ "measurement": "statistics",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "B",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "disk_share"
+ ],
+ "type": "field"
+ }
+ ]
+ ],
+ "tags": []
+ },
+ {
+ "alias": "Network",
+ "groupBy": [],
+ "measurement": "statistics",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "C",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "network_share"
+ ],
+ "type": "field"
+ }
+ ]
+ ],
+ "tags": []
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Shares",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST}",
+ "fill": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 11,
+ "x": 12,
+ "y": 0
+ },
+ "id": 2,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": true,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "alias": "Total",
+ "groupBy": [],
+ "measurement": "statistics",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "data_volume"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [
+ " / elapsed_ms/1024/1024/1024*1000*8"
+ ],
+ "type": "math"
+ }
+ ]
+ ],
+ "tags": []
+ },
+ {
+ "alias": "Database",
+ "groupBy": [],
+ "measurement": "statistics",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "B",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "db_share"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [
+ " *data_volume/ elapsed_ms/1024/1024/1024*1000*8"
+ ],
+ "type": "math"
+ }
+ ]
+ ],
+ "tags": []
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Bandwidth",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_TEST}",
+ "fill": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 11,
+ "x": 12,
+ "y": 8
+ },
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "alias": "Receiver",
+ "groupBy": [
+ {
+ "params": [
+ "$__interval"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "null"
+ ],
+ "type": "fill"
+ }
+ ],
+ "measurement": "RequestsRate",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "query": "SELECT \"n_requests\" / elapsed_ms*1000 FROM \"statistics\" WHERE $timeFilter",
+ "rawQuery": true,
+ "refId": "A",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "n_requests"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
+ }
+ ]
+ ],
+ "tags": []
+ },
+ {
+ "alias": "Broker",
+ "groupBy": [],
+ "measurement": "RequestsRate",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "B",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "rate"
+ ],
+ "type": "field"
+ }
+ ]
+ ],
+ "tags": []
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Number of Requests",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "transparent": false,
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ }
+ ],
+ "refresh": false,
+ "schemaVersion": 16,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now/d",
+ "to": "now/d"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "ASAP::O",
+ "uid": "3JvTwliiz",
+ "version": 3
+}
\ No newline at end of file
diff --git a/tests/manual/performance_full_chain_simple/broker.json b/tests/manual/performance_full_chain_simple/broker.json
new file mode 100644
index 000000000..c45d16f2f
--- /dev/null
+++ b/tests/manual/performance_full_chain_simple/broker.json
@@ -0,0 +1,6 @@
+{
+ "BrokerDbAddress":"localhost:27017",
+ "MonitorDbAddress": "localhost:8086",
+ "MonitorDbName": "db_test",
+ "port":5005
+}
\ No newline at end of file
diff --git a/tests/manual/performance_full_chain_simple/receiver.json b/tests/manual/performance_full_chain_simple/receiver.json
new file mode 100644
index 000000000..5939e7d82
--- /dev/null
+++ b/tests/manual/performance_full_chain_simple/receiver.json
@@ -0,0 +1,9 @@
+{
+ "MonitorDbAddress":"localhost:8086",
+ "MonitorDbName": "db_test",
+ "BrokerDbAddress":"localhost:27017",
+ "BrokerDbName": "test_run",
+ "ListenPort":4200,
+ "WriteToDisk":true,
+ "WriteToDb":true
+}
\ No newline at end of file
diff --git a/tests/manual/performance_full_chain_simple/test.sh b/tests/manual/performance_full_chain_simple/test.sh
new file mode 100755
index 000000000..77c63a85c
--- /dev/null
+++ b/tests/manual/performance_full_chain_simple/test.sh
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+set -e
+
+trap Cleanup EXIT
+
+#clean-up
+Cleanup() {
+set +e
+ssh ${receiver_node} rm -f ${receiver_dir}/files/*
+ssh ${receiver_node} killall receiver
+ssh ${broker_node} killall hidra2-broker
+ssh ${broker_node} docker rm -f -v mongo
+}
+
+#monitoring_setup
+monitor_node=zitpcx27016
+monitor_port=8086
+
+
+# starts receiver on $receiver_node
+# runs producer with various file sizes from $producer_node and measures performance
+
+file_size=10000
+file_num=$((10000000 / $file_size))
+echo filesize: ${file_size}K, filenum: $file_num
+
+# receiver_setup
+receiver_node=max-wgs
+receiver_ip=`resolveip -s ${receiver_node}`
+receiver_port=4201
+receiver_dir=/gpfs/petra3/scratch/yakubov/receiver_tests
+ssh ${receiver_node} mkdir -p ${receiver_dir}
+ssh ${receiver_node} mkdir -p ${receiver_dir}/files
+scp ../../../cmake-build-release/receiver/receiver ${receiver_node}:${receiver_dir}
+cat receiver.json |
+ jq "to_entries |
+ map(if .key == \"MonitorDbAddress\"
+ then . + {value:\"${monitor_node}:${monitor_port}\"}
+ elif .key == \"ListenPort\"
+ then . + {value:${receiver_port}}
+ else .
+ end
+ ) |
+ from_entries" > settings_tmp.json
+scp settings_tmp.json ${receiver_node}:${receiver_dir}/settings.json
+
+#producer_setup
+producer_node=max-display001
+#producer_node=max-wgs
+producer_dir=~/fullchain_tests
+ssh ${producer_node} mkdir -p ${producer_dir}
+scp ../../../cmake-build-release/examples/producer/dummy-data-producer/dummy-data-producer ${producer_node}:${producer_dir}
+
+#broker_setup
+broker_node=max-wgs
+broker_dir=~/fullchain_tests
+cat broker.json |
+ jq "to_entries |
+ map(if .key == \"MonitorDbAddress\"
+ then . + {value:\"${monitor_node}:${monitor_port}\"}
+ else .
+ end
+ ) |
+ from_entries" > settings_tmp.json
+scp settings_tmp.json ${broker_node}:${broker_dir}/broker.json
+rm settings_tmp.json
+scp ../../../cmake-build-release/broker/hidra2-broker ${broker_node}:${broker_dir}
+
+
+#worker_setup
+worker_node=max-display002
+worker_dir=~/fullchain_tests
+nthreads=16
+scp ../../../cmake-build-release/examples/worker/getnext_broker/getnext_broker ${worker_node}:${worker_dir}
+
+#monitoring_start
+ssh ${monitor_node} influx -execute \"create database db_test\"
+#ssh ${monitor_node} docker run -d -p 8086 -p 8086 --name influxdb influxdb
+
+#mongo_start
+ssh ${broker_node} docker run -d -p 27017:27017 --name mongo mongo
+
+#receiver_start
+ssh ${receiver_node} "bash -c 'cd ${receiver_dir}; nohup ./receiver settings.json &> ${receiver_dir}/receiver.log &'"
+sleep 0.3
+
+#broker_start
+ssh ${broker_node} "bash -c 'cd ${broker_dir}; nohup ./hidra2-broker broker.json &> ${broker_dir}/broker.log &'"
+sleep 0.3
+
+#producer_start
+ssh ${producer_node} "bash -c 'cd ${producer_dir}; nohup ./dummy-data-producer ${receiver_ip}:${receiver_port} ${file_size} ${file_num} &> ${producer_dir}/producer.log &'"
+sleep 0.3
+
+#worker_start
+ssh ${worker_node} ${worker_dir}/getnext_broker ${broker_node}:5005 test_run ${nthreads}
+
diff --git a/tests/manual/performance_producer_receiver/receiver.json b/tests/manual/performance_producer_receiver/receiver.json
index e23a556d1..2479a88d5 100644
--- a/tests/manual/performance_producer_receiver/receiver.json
+++ b/tests/manual/performance_producer_receiver/receiver.json
@@ -1,6 +1,9 @@
{
"MonitorDbAddress":"localhost:8086",
"MonitorDbName": "db_test",
+ "BrokerDbAddress":"localhost:27017",
+ "BrokerDbName": "test_run",
"ListenPort":4200,
- "WriteToDisk":true
-}
\ No newline at end of file
+ "WriteToDisk":true,
+ "WriteToDb":true
+}
diff --git a/tests/manual/performance_producer_receiver/settings_tmp.json b/tests/manual/performance_producer_receiver/settings_tmp.json
deleted file mode 100644
index 35b3649bc..000000000
--- a/tests/manual/performance_producer_receiver/settings_tmp.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "MonitorDbAddress": "zitpcx27016:8086",
- "MonitorDbName": "db_test",
- "ListenPort": 4201,
- "WriteToDisk": false
-}
diff --git a/tests/manual/performance_producer_receiver/test.sh b/tests/manual/performance_producer_receiver/test.sh
index 317f32d77..862ae2ec8 100755
--- a/tests/manual/performance_producer_receiver/test.sh
+++ b/tests/manual/performance_producer_receiver/test.sh
@@ -30,7 +30,6 @@ ssh ${worker_node} mkdir -p ${worker_dir}
scp ../../../cmake-build-release/receiver/receiver ${service_node}:${service_dir}
scp ../../../cmake-build-release/examples/producer/dummy-data-producer/dummy-data-producer ${worker_node}:${worker_dir}
-
function do_work {
cat receiver.json |
jq "to_entries |
@@ -49,9 +48,11 @@ ssh ${service_node} "bash -c 'cd ${service_dir}; nohup ./receiver settings.json
sleep 0.3
for size in 100 1000 10000
do
+ssh ${service_node} docker run -d -p 27017:27017 --name mongo mongo
echo ===================================================================
ssh ${worker_node} ${worker_dir}/dummy-data-producer ${service_ip}:${service_port} ${size} 1000
ssh ${service_node} rm -f ${service_dir}/files/*
+ssh ${service_node} docker rm -f -v mongo
done
ssh ${service_node} killall receiver
}
@@ -67,3 +68,4 @@ do_work false
#rm settings_tmp.json
#ssh ${service_node} docker rm -f influxdb
+
--
GitLab