From 2cd785fe1819215d4ca14660b978d89564014911 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Tue, 29 May 2018 16:47:51 +0200
Subject: [PATCH] update grafana dashboard
---
config/grafana/ASAP__O.json | 527 ++++++++++++++++--
.../dummy_data_producer.cpp | 2 +-
.../worker/getnext_broker/getnext_broker.cpp | 2 +-
.../full_chain/simple_chain/check_linux.sh | 4 +-
.../full_chain/simple_chain/check_windows.bat | 28 +-
.../transfer_single_file/check_windows.bat | 17 +-
.../performance_full_chain_simple/test.sh | 6 +-
7 files changed, 507 insertions(+), 79 deletions(-)
diff --git a/config/grafana/ASAP__O.json b/config/grafana/ASAP__O.json
index 7ed15c7ee..8311cfdbc 100644
--- a/config/grafana/ASAP__O.json
+++ b/config/grafana/ASAP__O.json
@@ -1,8 +1,8 @@
{
"__inputs": [
{
- "name": "DS_TEST",
- "label": "test",
+ "name": "DS_ASAPO",
+ "label": "asapo",
"description": "",
"type": "datasource",
"pluginId": "influxdb",
@@ -27,6 +27,12 @@
"id": "influxdb",
"name": "InfluxDB",
"version": "5.0.0"
+ },
+ {
+ "type": "panel",
+ "id": "singlestat",
+ "name": "Singlestat",
+ "version": "5.0.0"
}
],
"annotations": {
@@ -48,20 +54,144 @@
"id": null,
"links": [],
"panels": [
+ {
+ "cacheTimeout": null,
+ "colorBackground": false,
+ "colorValue": false,
+ "colors": [
+ "#299c46",
+ "rgba(237, 129, 40, 0.89)",
+ "#d44a3a"
+ ],
+ "datasource": "${DS_ASAPO}",
+ "format": "none",
+ "gauge": {
+ "maxValue": 100,
+ "minValue": 0,
+ "show": false,
+ "thresholdLabels": false,
+ "thresholdMarkers": true
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 5,
+ "x": 0,
+ "y": 0
+ },
+ "id": 9,
+ "interval": null,
+ "links": [],
+ "mappingType": 1,
+ "mappingTypes": [
+ {
+ "name": "value to text",
+ "value": 1
+ },
+ {
+ "name": "range to text",
+ "value": 2
+ }
+ ],
+ "maxDataPoints": 100,
+ "nullPointMode": "connected",
+ "nullText": null,
+ "postfix": "",
+ "postfixFontSize": "50%",
+ "prefix": "",
+ "prefixFontSize": "50%",
+ "rangeMaps": [
+ {
+ "from": "null",
+ "text": "N/A",
+ "to": "null"
+ }
+ ],
+ "sparkline": {
+ "fillColor": "rgba(31, 118, 189, 0.18)",
+ "full": false,
+ "lineColor": "rgb(31, 120, 193)",
+ "show": false
+ },
+ "tableColumn": "sum",
+ "targets": [
+ {
+ "groupBy": [
+ {
+ "params": [
+ "10s"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "null"
+ ],
+ "type": "fill"
+ }
+ ],
+ "measurement": "statistics",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "query": "SHOW TAG VALUES CARDINALITY FROM \"statistics\" WITH KEY = \"connection_from\"\n",
+ "rawQuery": false,
+ "refId": "A",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "elapsed_ms"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [],
+ "type": "sum"
+ },
+ {
+ "params": [
+ " / 1000/10"
+ ],
+ "type": "math"
+ }
+ ]
+ ],
+ "tags": [
+ {
+ "key": "receiver_tag",
+ "operator": "=",
+ "value": "test_receiver"
+ }
+ ]
+ }
+ ],
+ "thresholds": "",
+ "title": "Number of Connections",
+ "type": "singlestat",
+ "valueFontSize": "80%",
+ "valueMaps": [
+ {
+ "op": "=",
+ "text": "N/A",
+ "value": "null"
+ }
+ ],
+ "valueName": "avg"
+ },
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "${DS_TEST}",
- "fill": 0,
+ "datasource": "${DS_ASAPO}",
+ "fill": 1,
"gridPos": {
"h": 9,
"w": 12,
- "x": 0,
+ "x": 12,
"y": 0
},
- "id": 6,
+ "id": 7,
"legend": {
"avg": false,
"current": false,
@@ -85,13 +215,24 @@
"steppedLine": false,
"targets": [
{
- "alias": "Database",
- "groupBy": [],
+ "alias": "Database share",
+ "groupBy": [
+ {
+ "params": [
+ "10s"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "connection_from"
+ ],
+ "type": "tag"
+ }
+ ],
"measurement": "statistics",
"orderByTime": "ASC",
"policy": "default",
- "query": "SELECT \"db_share\" FROM \"statistics\" WHERE $timeFilter",
- "rawQuery": false,
"refId": "A",
"resultFormat": "time_series",
"select": [
@@ -101,14 +242,37 @@
"db_share"
],
"type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
}
]
],
"tags": []
},
{
- "alias": "Disk",
- "groupBy": [],
+ "alias": "Disk share",
+ "groupBy": [
+ {
+ "params": [
+ "10s"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "connection_from"
+ ],
+ "type": "tag"
+ },
+ {
+ "params": [
+ "null"
+ ],
+ "type": "fill"
+ }
+ ],
"measurement": "statistics",
"orderByTime": "ASC",
"policy": "default",
@@ -121,14 +285,37 @@
"disk_share"
],
"type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
}
]
],
"tags": []
},
{
- "alias": "Network",
- "groupBy": [],
+ "alias": "Network share",
+ "groupBy": [
+ {
+ "params": [
+ "10s"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "connection_from"
+ ],
+ "type": "tag"
+ },
+ {
+ "params": [
+ "null"
+ ],
+ "type": "fill"
+ }
+ ],
"measurement": "statistics",
"orderByTime": "ASC",
"policy": "default",
@@ -141,6 +328,10 @@
"network_share"
],
"type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
}
]
],
@@ -150,7 +341,7 @@
"thresholds": [],
"timeFrom": null,
"timeShift": null,
- "title": "Shares",
+ "title": "Work shares",
"tooltip": {
"shared": true,
"sort": 0,
@@ -188,15 +379,15 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "${DS_TEST}",
- "fill": 0,
+ "datasource": "${DS_ASAPO}",
+ "fill": 1,
"gridPos": {
- "h": 8,
- "w": 11,
- "x": 12,
- "y": 0
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 3
},
- "id": 2,
+ "id": 6,
"legend": {
"avg": false,
"current": false,
@@ -210,7 +401,7 @@
"linewidth": 1,
"links": [],
"nullPointMode": "null",
- "percentage": true,
+ "percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
@@ -220,8 +411,15 @@
"steppedLine": false,
"targets": [
{
- "alias": "Total",
- "groupBy": [],
+ "alias": "Database share",
+ "groupBy": [
+ {
+ "params": [
+ "10s"
+ ],
+ "type": "time"
+ }
+ ],
"measurement": "statistics",
"orderByTime": "ASC",
"policy": "default",
@@ -231,15 +429,87 @@
[
{
"params": [
- "data_volume"
+ "db_share"
],
"type": "field"
},
+ {
+ "params": [],
+ "type": "mean"
+ }
+ ]
+ ],
+ "tags": []
+ },
+ {
+ "alias": "Disk share",
+ "groupBy": [
+ {
+ "params": [
+ "10s"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "null"
+ ],
+ "type": "fill"
+ }
+ ],
+ "measurement": "statistics",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "B",
+ "resultFormat": "time_series",
+ "select": [
+ [
{
"params": [
- " / elapsed_ms/1024/1024/1024*1000*8"
+ "disk_share"
],
- "type": "math"
+ "type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
+ }
+ ]
+ ],
+ "tags": []
+ },
+ {
+ "alias": "Network share",
+ "groupBy": [
+ {
+ "params": [
+ "10s"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "null"
+ ],
+ "type": "fill"
+ }
+ ],
+ "measurement": "statistics",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "C",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "network_share"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
}
]
],
@@ -249,7 +519,7 @@
"thresholds": [],
"timeFrom": null,
"timeShift": null,
- "title": "Bandwidth",
+ "title": "Work shares",
"tooltip": {
"shared": true,
"sort": 0,
@@ -287,15 +557,15 @@
"bars": false,
"dashLength": 10,
"dashes": false,
- "datasource": "${DS_TEST}",
- "fill": 0,
+ "datasource": "${DS_ASAPO}",
+ "fill": 1,
"gridPos": {
- "h": 8,
- "w": 11,
+ "h": 9,
+ "w": 12,
"x": 12,
- "y": 8
+ "y": 9
},
- "id": 4,
+ "id": 2,
"legend": {
"avg": false,
"current": false,
@@ -323,7 +593,7 @@
"groupBy": [
{
"params": [
- "$__interval"
+ "10s"
],
"type": "time"
},
@@ -334,11 +604,11 @@
"type": "fill"
}
],
- "measurement": "RequestsRate",
+ "measurement": "statistics",
"orderByTime": "ASC",
"policy": "default",
- "query": "SELECT \"n_requests\" / elapsed_ms*1000 FROM \"statistics\" WHERE $timeFilter",
- "rawQuery": true,
+ "query": "select SUM(n_requests)/10 from statistics where time>1527594865972143840 and time<=now() group by time(10s)\n",
+ "rawQuery": false,
"refId": "A",
"resultFormat": "time_series",
"select": [
@@ -351,15 +621,28 @@
},
{
"params": [],
- "type": "mean"
+ "type": "sum"
+ },
+ {
+ "params": [
+ " / 10"
+ ],
+ "type": "math"
}
]
],
"tags": []
},
{
- "alias": "Broker",
- "groupBy": [],
+ "alias": "Worker",
+ "groupBy": [
+ {
+ "params": [
+ "10s"
+ ],
+ "type": "time"
+ }
+ ],
"measurement": "RequestsRate",
"orderByTime": "ASC",
"policy": "default",
@@ -372,6 +655,151 @@
"rate"
],
"type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
+ }
+ ]
+ ],
+ "tags": []
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Receiver/Worker Rates",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ]
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_ASAPO}",
+ "fill": 1,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 12
+ },
+ "id": 4,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "groupBy": [
+ {
+ "params": [
+ "10s"
+ ],
+ "type": "time"
+ }
+ ],
+ "measurement": "statistics",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "data_volume"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [],
+ "type": "sum"
+ },
+ {
+ "params": [
+ " /10/1000/1000/1000*8"
+ ],
+ "type": "math"
+ }
+ ]
+ ],
+ "tags": []
+ },
+ {
+ "groupBy": [
+ {
+ "params": [
+ "connection_from"
+ ],
+ "type": "tag"
+ }
+ ],
+ "measurement": "statistics",
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "B",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "data_volume"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [
+ " / elapsed_ms/1000/1000*8"
+ ],
+ "type": "math"
}
]
],
@@ -381,13 +809,12 @@
"thresholds": [],
"timeFrom": null,
"timeShift": null,
- "title": "Number of Requests",
+ "title": "Bandwidth Gbits/s",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
- "transparent": false,
"type": "graph",
"xaxis": {
"buckets": null,
@@ -416,7 +843,7 @@
]
}
],
- "refresh": false,
+ "refresh": "10s",
"schemaVersion": 16,
"style": "dark",
"tags": [],
@@ -424,8 +851,8 @@
"list": []
},
"time": {
- "from": "now/d",
- "to": "now/d"
+ "from": "now-5m",
+ "to": "now"
},
"timepicker": {
"refresh_intervals": [
@@ -453,7 +880,7 @@
]
},
"timezone": "",
- "title": "ASAP::O",
- "uid": "3JvTwliiz",
- "version": 4
+ "title": "Asapo",
+ "uid": "jZtUsU4mz",
+ "version": 6
}
\ No newline at end of file
diff --git a/examples/producer/dummy-data-producer/dummy_data_producer.cpp b/examples/producer/dummy-data-producer/dummy_data_producer.cpp
index bb0f49bc1..976a049ea 100644
--- a/examples/producer/dummy-data-producer/dummy_data_producer.cpp
+++ b/examples/producer/dummy-data-producer/dummy_data_producer.cpp
@@ -83,7 +83,7 @@ int main (int argc, char* argv[]) {
producer = asapo::Producer::Create(receiver_address, nthreads, asapo::RequestHandlerType::kFilesystem, &err);
}
producer->EnableLocalLog(true);
- producer->SetLogLevel(asapo::LogLevel::Debug);
+ producer->SetLogLevel(asapo::LogLevel::Info);
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
diff --git a/examples/worker/getnext_broker/getnext_broker.cpp b/examples/worker/getnext_broker/getnext_broker.cpp
index d3628406e..b8bcd17ed 100644
--- a/examples/worker/getnext_broker/getnext_broker.cpp
+++ b/examples/worker/getnext_broker/getnext_broker.cpp
@@ -34,7 +34,7 @@ std::vector<std::thread> StartThreads(const std::string& server, const std::stri
asapo::FileInfo fi;
Error err;
auto broker = asapo::DataBrokerFactory::CreateServerBroker(server, run_name, &err);
- broker->SetTimeout(1000);
+ broker->SetTimeout(10000);
while ((err = broker->GetNext(&fi, nullptr)) == nullptr) {
(*nfiles)[i] ++;
}
diff --git a/tests/automatic/full_chain/simple_chain/check_linux.sh b/tests/automatic/full_chain/simple_chain/check_linux.sh
index b3ab88bb1..e770d0395 100644
--- a/tests/automatic/full_chain/simple_chain/check_linux.sh
+++ b/tests/automatic/full_chain/simple_chain/check_linux.sh
@@ -26,7 +26,7 @@ nomad run receiver.nmd
nomad run discovery.nmd
nomad run broker.nmd
-sleep 1
+sleep 5
#producer
@@ -35,4 +35,4 @@ $1 localhost:5006 100 1000 4 0 &
#producerid=`echo $!`
-$2 ${broker_address} ${broker_database_name} 2 #| grep "Processed 1000 file(s)"
+$2 ${broker_address} ${broker_database_name} 2 | grep "Processed 1000 file(s)"
diff --git a/tests/automatic/full_chain/simple_chain/check_windows.bat b/tests/automatic/full_chain/simple_chain/check_windows.bat
index 7c3fc7a03..39ba4c7c4 100644
--- a/tests/automatic/full_chain/simple_chain/check_windows.bat
+++ b/tests/automatic/full_chain/simple_chain/check_windows.bat
@@ -1,24 +1,22 @@
-REM receiver
-set full_recv_name="%2"
-set short_recv_name="%~nx2"
-start /B "" "%full_recv_name%" receiver.json
-ping 1.0.0.0 -n 1 -w 100 > nul
+SET mongo_exe="c:\Program Files\MongoDB\Server\3.6\bin\mongo.exe"
+set broker_database_name="test_run"
+
+echo db.%broker_database_name%.insert({dummy:1})" | %mongo_exe% %broker_database_name%
+
+c:\opt\consul\nomad run receiver.nmd
+c:\opt\consul\nomad run discovery.nmd
+c:\opt\consul\nomad run broker.nmd
-REM broker
-set full_broker_name="%3"
-set short_broker_name="%~nx3"
-start /B "" "%full_broker_name%" -config broker.json
ping 1.0.0.0 -n 1 -w 100 > nul
REM producer
mkdir files
-start /B "" "%1" localhost:4200 100 100
+start /B "" "%1" localhost:5006 100 1000 4 0
ping 1.0.0.0 -n 1 -w 100 > nul
REM worker
set broker_address="127.0.0.1:5005"
-set broker_database_name="test_run"
-"%4" %broker_address% %broker_database_name% 2 | findstr "Processed 100 file(s)" || goto :error
+"%2" %broker_address% %broker_database_name% 2 | findstr "Processed 1000 file(s)" || goto :error
goto :clean
@@ -28,10 +26,10 @@ call :clean
exit /b 1
:clean
-Taskkill /IM "%short_recv_name%" /F
-Taskkill /IM "%short_broker_name%" /F
+c:\opt\consul\nomad stop receiver
+c:\opt\consul\nomad stop discovery
+c:\opt\consul\nomad stop broker
rmdir /S /Q files
-SET mongo_exe="c:\Program Files\MongoDB\Server\3.6\bin\mongo.exe"
echo db.dropDatabase() | %mongo_exe% %broker_database_name%
diff --git a/tests/automatic/producer_receiver/transfer_single_file/check_windows.bat b/tests/automatic/producer_receiver/transfer_single_file/check_windows.bat
index 8a470b403..6462502f8 100644
--- a/tests/automatic/producer_receiver/transfer_single_file/check_windows.bat
+++ b/tests/automatic/producer_receiver/transfer_single_file/check_windows.bat
@@ -1,13 +1,17 @@
-set full_recv_name="%2"
-set short_recv_name="%~nx2"
+SET mongo_exe="c:\Program Files\MongoDB\Server\3.6\bin\mongo.exe"
+SET database_name=test_run
+
+echo db.%database_name%.insert({dummy:1})" | %mongo_exe% %database_name%
-start /B "" "%full_recv_name%" receiver.json
+
+c:\opt\consul\nomad run receiver.nmd
+c:\opt\consul\nomad run discovery.nmd
ping 1.0.0.0 -n 1 -w 100 > nul
mkdir files
-%1 localhost:4200 100 1
+%1 localhost:5006 100 1 1 0
ping 1.0.0.0 -n 1 -w 100 > nul
@@ -22,10 +26,9 @@ call :clean
exit /b 1
:clean
-Taskkill /IM "%short_recv_name%" /F
+c:\opt\consul\nomad stop receiver
+c:\opt\consul\nomad stop discovery
rmdir /S /Q files
-SET database_name=test_run
-SET mongo_exe="c:\Program Files\MongoDB\Server\3.6\bin\mongo.exe"
echo db.dropDatabase() | %mongo_exe% %database_name%
diff --git a/tests/manual/performance_full_chain_simple/test.sh b/tests/manual/performance_full_chain_simple/test.sh
index 08039120d..880e03885 100755
--- a/tests/manual/performance_full_chain_simple/test.sh
+++ b/tests/manual/performance_full_chain_simple/test.sh
@@ -25,8 +25,8 @@ log_dir=~/fullchain_tests/logs
# starts receiver on $receiver_node
# runs producer with various file sizes from $producer_node and measures performance
-file_size=100
-file_num=$((10000000 / $file_size))
+file_size=10000
+file_num=$((100000000 / $file_size))
echo filesize: ${file_size}K, filenum: $file_num
# receiver_setup
@@ -72,7 +72,7 @@ producer_node=max-display001
#producer_node=max-wgs
producer_dir=~/fullchain_tests
scp ../../../cmake-build-release/examples/producer/dummy-data-producer/dummy-data-producer ${producer_node}:${producer_dir}
-producer_nthreads=8
+producer_nthreads=16
#broker_setup
--
GitLab