From eaf7e5611aa07f6d78f060208369048a1ae099d4 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Wed, 6 Jun 2018 19:20:56 +0200
Subject: [PATCH] update manual tests
---
.../request_handler_consul_test.go | 2 +-
.../request_handler/request_handler_static.go | 4 +--
.../request_handler_static_test.go | 4 +--
.../src/asapo_discovery/server/routes_test.go | 4 +--
.../asapo_discovery/server/settings_test.go | 8 +++---
discovery/src/asapo_discovery/utils/stucts.go | 8 +++---
.../broker/check_monitoring/check_linux.sh | 1 +
.../manual/performance_broker/discovery.json | 8 ++++--
tests/manual/performance_broker/test.sh | 25 +++++++++++--------
.../discovery.json | 11 ++++----
.../performance_full_chain_simple/test.sh | 17 ++++++++-----
.../discovery.json | 11 ++++++--
.../performance_producer_receiver/test.sh | 14 +++++++----
13 files changed, 71 insertions(+), 46 deletions(-)
diff --git a/discovery/src/asapo_discovery/request_handler/request_handler_consul_test.go b/discovery/src/asapo_discovery/request_handler/request_handler_consul_test.go
index 79f349ee8..e3e5b78c9 100644
--- a/discovery/src/asapo_discovery/request_handler/request_handler_consul_test.go
+++ b/discovery/src/asapo_discovery/request_handler/request_handler_consul_test.go
@@ -42,7 +42,7 @@ func (suite *ConsulHandlerTestSuite) registerAgents(name string) {
func (suite *ConsulHandlerTestSuite) SetupTest() {
var err error
- consul_settings = utils.Settings{Receiver: utils.ReceiverInfo{MaxConnections: 10, ForceEndpoints: []string{}}}
+ consul_settings = utils.Settings{Receiver: utils.ReceiverInfo{MaxConnections: 10, StaticEndpoints: []string{}}}
suite.client, err = api.NewClient(api.DefaultConfig())
if err != nil {
diff --git a/discovery/src/asapo_discovery/request_handler/request_handler_static.go b/discovery/src/asapo_discovery/request_handler/request_handler_static.go
index 90522d830..1fbe2b4a3 100644
--- a/discovery/src/asapo_discovery/request_handler/request_handler_static.go
+++ b/discovery/src/asapo_discovery/request_handler/request_handler_static.go
@@ -21,7 +21,7 @@ func (rh *StaticRequestHandler) GetBroker() ([]byte, error) {
func (rh *StaticRequestHandler) Init(settings utils.Settings) error {
rh.receiverResponce.MaxConnections = settings.Receiver.MaxConnections
- rh.receiverResponce.Uris = settings.Receiver.ForceEndpoints
- rh.broker = settings.Broker.ForceEndpoint
+ rh.receiverResponce.Uris = settings.Receiver.StaticEndpoints
+ rh.broker = settings.Broker.StaticEndpoint
return nil
}
diff --git a/discovery/src/asapo_discovery/request_handler/request_handler_static_test.go b/discovery/src/asapo_discovery/request_handler/request_handler_static_test.go
index 35b08f2f5..af9bf7d80 100644
--- a/discovery/src/asapo_discovery/request_handler/request_handler_static_test.go
+++ b/discovery/src/asapo_discovery/request_handler/request_handler_static_test.go
@@ -10,8 +10,8 @@ import (
var uris = []string{"ip1","ip2"}
const max_conn = 1
-var static_settings utils.Settings= utils.Settings{Receiver:utils.ReceiverInfo{MaxConnections:max_conn,ForceEndpoints:uris},Broker:utils.BrokerInfo{
- ForceEndpoint:"ip_broker"}}
+var static_settings utils.Settings= utils.Settings{Receiver:utils.ReceiverInfo{MaxConnections:max_conn,StaticEndpoints:uris},Broker:utils.BrokerInfo{
+ StaticEndpoint:"ip_broker"}}
diff --git a/discovery/src/asapo_discovery/server/routes_test.go b/discovery/src/asapo_discovery/server/routes_test.go
index a5bfe3e2f..4d35c2c09 100644
--- a/discovery/src/asapo_discovery/server/routes_test.go
+++ b/discovery/src/asapo_discovery/server/routes_test.go
@@ -30,8 +30,8 @@ type GetReceiversTestSuite struct {
func (suite *GetReceiversTestSuite) SetupTest() {
requestHandler = new(request_handler.StaticRequestHandler)
- var s utils.Settings= utils.Settings{Receiver:utils.ReceiverInfo{MaxConnections:10,ForceEndpoints:[]string{"ip1","ip2"}},
- Broker:utils.BrokerInfo{ForceEndpoint:"ip_broker"}}
+ var s utils.Settings= utils.Settings{Receiver:utils.ReceiverInfo{MaxConnections:10,StaticEndpoints:[]string{"ip1","ip2"}},
+ Broker:utils.BrokerInfo{StaticEndpoint:"ip_broker"}}
requestHandler.Init(s)
logger.SetMockLog()
diff --git a/discovery/src/asapo_discovery/server/settings_test.go b/discovery/src/asapo_discovery/server/settings_test.go
index d03b1b0af..2d85beb1d 100644
--- a/discovery/src/asapo_discovery/server/settings_test.go
+++ b/discovery/src/asapo_discovery/server/settings_test.go
@@ -12,8 +12,8 @@ func fillSettings(mode string) utils.Settings {
settings.Mode = mode
settings.Receiver.MaxConnections = 10
settings.LogLevel = "info"
- settings.Receiver.ForceEndpoints=[]string{"ip1","ip2"}
- settings.Broker.ForceEndpoint="ip_b"
+ settings.Receiver.StaticEndpoints=[]string{"ip1","ip2"}
+ settings.Broker.StaticEndpoint="ip_b"
settings.ConsulEndpoints=[]string{"ipc1","ipc2"}
return settings
}
@@ -32,14 +32,14 @@ func TestSettingsWrongMode(t *testing.T) {
func TestSettingsStaticModeNoReceiverEndpoints(t *testing.T) {
settings := fillSettings("static")
- settings.Receiver.ForceEndpoints=[]string{}
+ settings.Receiver.StaticEndpoints=[]string{}
err := settings.Validate()
assert.NotNil(t, err)
}
func TestSettingsStaticModeNoBrokerEndpoints(t *testing.T) {
settings := fillSettings("static")
- settings.Broker.ForceEndpoint=""
+ settings.Broker.StaticEndpoint=""
err := settings.Validate()
assert.NotNil(t, err)
}
diff --git a/discovery/src/asapo_discovery/utils/stucts.go b/discovery/src/asapo_discovery/utils/stucts.go
index 23c4a655d..f9bf6c316 100644
--- a/discovery/src/asapo_discovery/utils/stucts.go
+++ b/discovery/src/asapo_discovery/utils/stucts.go
@@ -3,12 +3,12 @@ package utils
import "errors"
type ReceiverInfo struct {
- ForceEndpoints []string
+ StaticEndpoints []string
MaxConnections int
}
type BrokerInfo struct {
- ForceEndpoint string
+ StaticEndpoint string
}
@@ -23,8 +23,8 @@ type Settings struct {
func (settings *Settings) Validate() error {
if settings.Mode != "consul"{
- if len(settings.Receiver.ForceEndpoints) == 0 || len(settings.Broker.ForceEndpoint) == 0 {
- return errors.New("Endpoints not set")
+ if len(settings.Receiver.StaticEndpoints) == 0 || len(settings.Broker.StaticEndpoint) == 0 {
+ return errors.New("receiver or broker endpoints not set")
}
}
diff --git a/tests/automatic/broker/check_monitoring/check_linux.sh b/tests/automatic/broker/check_monitoring/check_linux.sh
index 254e8ea57..b0a2de9ab 100644
--- a/tests/automatic/broker/check_monitoring/check_linux.sh
+++ b/tests/automatic/broker/check_monitoring/check_linux.sh
@@ -7,6 +7,7 @@ set -e
trap Cleanup EXIT
Cleanup() {
+ set +e
echo cleanup
influx -execute "drop database ${database_name}"
kill -9 $brokerid
diff --git a/tests/manual/performance_broker/discovery.json b/tests/manual/performance_broker/discovery.json
index bb3394784..a1438446b 100644
--- a/tests/manual/performance_broker/discovery.json
+++ b/tests/manual/performance_broker/discovery.json
@@ -1,8 +1,12 @@
{
"Mode": "static",
"Broker": {
- "Endpoint": [
- "localhost:5005"
+ "StaticEndpoint": "localhost:5005"
+ },
+ "Receiver": {
+ "MaxConnections":1,
+ "StaticEndpoints": [
+ "test"
]
},
"Port": 5006,
diff --git a/tests/manual/performance_broker/test.sh b/tests/manual/performance_broker/test.sh
index dbb1d4d20..9c36fce45 100755
--- a/tests/manual/performance_broker/test.sh
+++ b/tests/manual/performance_broker/test.sh
@@ -4,7 +4,7 @@
# reads fileset into database
# calls getnext_broker example from $worker_node
-nthreads=1
+nthreads=16
# a directory with many files in it
dir=/gpfs/petra3/scratch/yakubov/test
run_name=test
@@ -20,14 +20,10 @@ worker_dir=~/broker_test
service_dir=~/broker_test
-cat settings.json |
- jq "to_entries |
- map(if .key == \"MonitorDbAddress\"
- then . + {value:\"${monitor_node}:${monitor_port}\"}
- else .
- end
- ) |
- from_entries" > settings_tmp.json
+cat settings.json | jq ".MonitorDbAddress = \"${monitor_node}:${monitor_port}\"" > settings_tmp.json
+
+cat discovery.json | jq ".Broker.StaticEndpoint = \"${service_node}:5005\"" > discovery_tmp.json
+
ssh ${monitor_node} influx -execute \"create database db_test\"
@@ -38,6 +34,13 @@ ssh ${service_node} mkdir -p ${service_dir}
ssh ${worker_node} mkdir -p ${worker_dir}
+scp ../../../cmake-build-release/discovery/asapo-discovery ${service_node}:${service_dir}
+scp discovery_tmp.json ${service_node}:${service_dir}/discovery.json
+rm discovery_tmp.json
+
+ssh ${service_node} "bash -c 'cd ${service_dir}; nohup ./asapo-discovery -config discovery.json &> ${service_dir}/discovery.log &'"
+
+
scp settings_tmp.json ${service_node}:${service_dir}/settings.json
rm settings_tmp.json
scp ../../../cmake-build-release/broker/asapo-broker ${service_node}:${service_dir}
@@ -49,10 +52,12 @@ ssh ${worker_node} ${worker_dir}/folder2db -n ${nthreads} ${dir} ${run_name} ${s
sleep 3
scp ../../../cmake-build-release/examples/worker/getnext_broker/getnext_broker ${worker_node}:${worker_dir}
-ssh ${worker_node} ${worker_dir}/getnext_broker ${service_node}:5005 ${run_name} ${nthreads}
+ssh ${worker_node} ${worker_dir}/getnext_broker ${service_node}:8400 ${run_name} ${nthreads}
ssh ${service_node} killall asapo-broker
+ssh ${service_node} killall asapo-discovery
+
ssh ${service_node} docker rm -f mongo
#ssh ${service_node} docker rm -f influxdb
diff --git a/tests/manual/performance_full_chain_simple/discovery.json b/tests/manual/performance_full_chain_simple/discovery.json
index a0b12327c..8cdd433a7 100644
--- a/tests/manual/performance_full_chain_simple/discovery.json
+++ b/tests/manual/performance_full_chain_simple/discovery.json
@@ -1,16 +1,15 @@
{
"Mode": "static",
- "Recevier": {
- "Endpoints": [
+ "Receiver": {
+ "StaticEndpoints": [
"localhost:4200"
],
"MaxConnections": 32
},
"Broker": {
- "Endpoint": [
- "localhost:5005"
- ]
+ "StaticEndpoint": "localhost:5005"
},
"Port": 5006,
"LogLevel": "info"
-}
\ No newline at end of file
+}
+
diff --git a/tests/manual/performance_full_chain_simple/test.sh b/tests/manual/performance_full_chain_simple/test.sh
index e4f8fed91..3ddd186fb 100755
--- a/tests/manual/performance_full_chain_simple/test.sh
+++ b/tests/manual/performance_full_chain_simple/test.sh
@@ -25,8 +25,8 @@ log_dir=~/fullchain_tests/logs
# starts receiver on $receiver_node
# runs producer with various file sizes from $producer_node and measures performance
-file_size=100
-file_num=$((100000000 / $file_size))
+file_size=10000
+file_num=$((10000000 / $file_size))
echo filesize: ${file_size}K, filenum: $file_num
# receiver_setup
@@ -62,10 +62,15 @@ cat discovery.json |
end
) |
from_entries" > discovery_tmp.json
+
+cat discovery_tmp.json | jq ".Broker.StaticEndpoint = \"${receiver_node}:5005\"" > discovery_tmp1.json
+
+cat discovery_tmp1.json | jq ".Receiver.StaticEndpoints = [\"${receiver_node}:${receiver_port}\"]" > discovery_tmp2.json
+
scp ../../../cmake-build-release/discovery/asapo-discovery ${receiver_node}:${receiver_dir}
-scp discovery_tmp.json ${receiver_node}:${receiver_dir}/discovery.json
+scp discovery_tmp2.json ${receiver_node}:${receiver_dir}/discovery.json
discovery_ip=`resolveip -s ${receiver_node}`
-rm discovery_tmp.json
+rm discovery_tmp*.json
#producer_setup
producer_node=max-display001
@@ -120,10 +125,10 @@ sleep 0.3
sleep 5
#producer_start
-ssh ${producer_node} "bash -c 'cd ${producer_dir}; nohup ./dummy-data-producer ${discovery_ip}:${discovery_port} ${file_size} ${file_num} ${producer_nthreads} 0 &> ${log_dir}/producer.log &'"
+ssh ${producer_node} "bash -c 'cd ${producer_dir}; nohup ./dummy-data-producer ${receiver_node}:8400 ${file_size} ${file_num} ${producer_nthreads} 0 &> ${log_dir}/producer.log &'"
sleep 1
#worker_start
-ssh ${worker_node} ${worker_dir}/getnext_broker ${broker_node}:5005 test_run ${nthreads}
+ssh ${worker_node} ${worker_dir}/getnext_broker ${receiver_node}:8400 test_run ${nthreads}
diff --git a/tests/manual/performance_producer_receiver/discovery.json b/tests/manual/performance_producer_receiver/discovery.json
index 476f732bb..e741be430 100644
--- a/tests/manual/performance_producer_receiver/discovery.json
+++ b/tests/manual/performance_producer_receiver/discovery.json
@@ -1,7 +1,14 @@
{
- "MaxConnections": 32,
"Mode": "static",
- "Endpoints":["localhost:4200"],
+ "Receiver": {
+ "StaticEndpoints": [
+ "localhost:4200"
+ ],
+ "MaxConnections": 32
+ },
+ "Broker": {
+ "StaticEndpoint": "localhost:5005"
+ },
"Port":5006,
"LogLevel":"info"
}
\ No newline at end of file
diff --git a/tests/manual/performance_producer_receiver/test.sh b/tests/manual/performance_producer_receiver/test.sh
index 4498cee1c..53f9dfe6f 100755
--- a/tests/manual/performance_producer_receiver/test.sh
+++ b/tests/manual/performance_producer_receiver/test.sh
@@ -56,20 +56,24 @@ cat receiver.json |
end
) |
from_entries" > receiver_tmp.json
+
cat discovery.json |
jq "to_entries |
map(if .key == \"Port\"
then . + {value:${discovery_port}}
- elif .key == \"Endpoints\"
- then . + {value:[\"${service_node}:${receiver_port}\"]}
else .
end
) |
from_entries" > discovery_tmp.json
-scp discovery_tmp.json ${service_node}:${service_dir}/discovery.json
+cat discovery.json | jq ".Port = ${discovery_port}" > discovery_tmp.json
+
+cat discovery_tmp.json | jq ".Receiver.StaticEndpoints = [\"${service_node}:${receiver_port}\"]" > discovery_tmp1.json
+
+
+scp discovery_tmp1.json ${service_node}:${service_dir}/discovery.json
scp receiver_tmp.json ${service_node}:${service_dir}/receiver.json
-rm discovery_tmp.json receiver_tmp.json
+rm discovery_tmp*.json receiver_tmp.json
ssh ${service_node} "bash -c 'cd ${service_dir}; nohup ./receiver receiver.json &> ${service_dir}/receiver.log &'"
ssh ${service_node} "bash -c 'cd ${service_dir}; nohup ./asapo-discovery -config discovery.json &> ${service_dir}/discovery.log &'"
@@ -78,7 +82,7 @@ for size in 100 1000 10000
do
ssh ${service_node} docker run -d -p 27017:27017 --name mongo mongo
echo ===================================================================
-ssh ${worker_node} ${worker_dir}/dummy-data-producer ${service_ip}:${discovery_port} ${size} 1000 8 0
+ssh ${worker_node} ${worker_dir}/dummy-data-producer ${service_ip}:8400 ${size} 1000 8 0
if [ "$1" == "true" ]
then
ssh ${service_node} rm -f ${service_dir}/files/*
--
GitLab