update manual tests

discovery/src/asapo_discovery/request_handler/request_handler_consul_test.go

@@ -42,7 +42,7 @@ func (suite *ConsulHandlerTestSuite) registerAgents(name string) {
 
 func (suite *ConsulHandlerTestSuite) SetupTest() {
 	var err error
-	consul_settings = utils.Settings{Receiver: utils.ReceiverInfo{MaxConnections: 10, ForceEndpoints: []string{}}}
+	consul_settings = utils.Settings{Receiver: utils.ReceiverInfo{MaxConnections: 10, StaticEndpoints: []string{}}}
 
 	suite.client, err = api.NewClient(api.DefaultConfig())
 	if err != nil {

discovery/src/asapo_discovery/request_handler/request_handler_static.go

@@ -21,7 +21,7 @@ func (rh *StaticRequestHandler) GetBroker() ([]byte, error) {
 
 func (rh *StaticRequestHandler) Init(settings utils.Settings) error {
 	rh.receiverResponce.MaxConnections = settings.Receiver.MaxConnections
-	rh.receiverResponce.Uris = settings.Receiver.ForceEndpoints
-	rh.broker = settings.Broker.ForceEndpoint
+	rh.receiverResponce.Uris = settings.Receiver.StaticEndpoints
+	rh.broker = settings.Broker.StaticEndpoint
 	return nil
 }

discovery/src/asapo_discovery/request_handler/request_handler_static_test.go

@@ -10,8 +10,8 @@ import (
 var uris = []string{"ip1","ip2"}
 const max_conn = 1
 
-var static_settings utils.Settings= utils.Settings{Receiver:utils.ReceiverInfo{MaxConnections:max_conn,ForceEndpoints:uris},Broker:utils.BrokerInfo{
-	ForceEndpoint:"ip_broker"}}
+var static_settings utils.Settings= utils.Settings{Receiver:utils.ReceiverInfo{MaxConnections:max_conn,StaticEndpoints:uris},Broker:utils.BrokerInfo{
+	StaticEndpoint:"ip_broker"}}
 
 
 

discovery/src/asapo_discovery/server/routes_test.go

@@ -30,8 +30,8 @@ type GetReceiversTestSuite struct {
 
 func (suite *GetReceiversTestSuite) SetupTest() {
 	requestHandler = new(request_handler.StaticRequestHandler)
-	var s utils.Settings= utils.Settings{Receiver:utils.ReceiverInfo{MaxConnections:10,ForceEndpoints:[]string{"ip1","ip2"}},
-	Broker:utils.BrokerInfo{ForceEndpoint:"ip_broker"}}
+	var s utils.Settings= utils.Settings{Receiver:utils.ReceiverInfo{MaxConnections:10,StaticEndpoints:[]string{"ip1","ip2"}},
+	Broker:utils.BrokerInfo{StaticEndpoint:"ip_broker"}}
 
 	requestHandler.Init(s)
 	logger.SetMockLog()

discovery/src/asapo_discovery/server/settings_test.go

@@ -12,8 +12,8 @@ func fillSettings(mode string) utils.Settings {
 	settings.Mode = mode
 	settings.Receiver.MaxConnections = 10
 	settings.LogLevel = "info"
-	settings.Receiver.ForceEndpoints=[]string{"ip1","ip2"}
-	settings.Broker.ForceEndpoint="ip_b"
+	settings.Receiver.StaticEndpoints=[]string{"ip1","ip2"}
+	settings.Broker.StaticEndpoint="ip_b"
 	settings.ConsulEndpoints=[]string{"ipc1","ipc2"}
 	return settings
 }
@@ -32,14 +32,14 @@ func TestSettingsWrongMode(t *testing.T) {
 
 func TestSettingsStaticModeNoReceiverEndpoints(t *testing.T) {
 	settings := fillSettings("static")
-	settings.Receiver.ForceEndpoints=[]string{}
+	settings.Receiver.StaticEndpoints=[]string{}
 	err := settings.Validate()
 	assert.NotNil(t, err)
 }
 
 func TestSettingsStaticModeNoBrokerEndpoints(t *testing.T) {
 	settings := fillSettings("static")
-	settings.Broker.ForceEndpoint=""
+	settings.Broker.StaticEndpoint=""
 	err := settings.Validate()
 	assert.NotNil(t, err)
 }

discovery/src/asapo_discovery/utils/stucts.go

@@ -3,12 +3,12 @@ package utils
 import "errors"
 
 type ReceiverInfo struct {
-	ForceEndpoints	 []string
+	StaticEndpoints	 []string
 	MaxConnections   int
 }
 
 type BrokerInfo struct {
-	ForceEndpoint		 string
+	StaticEndpoint		 string
 }
 
 
@@ -23,8 +23,8 @@ type Settings struct {
 
 func (settings *Settings) Validate() error {
 	if settings.Mode != "consul"{
-		if len(settings.Receiver.ForceEndpoints) == 0 || len(settings.Broker.ForceEndpoint) == 0 {
-		return errors.New("Endpoints not set")
+		if len(settings.Receiver.StaticEndpoints) == 0 || len(settings.Broker.StaticEndpoint) == 0 {
+		return errors.New("receiver or broker endpoints not set")
 		}
 	}
 

tests/automatic/broker/check_monitoring/check_linux.sh

@@ -7,6 +7,7 @@ set -e
 trap Cleanup EXIT
 
 Cleanup() {
+    set +e
 	echo cleanup
 	influx -execute "drop database ${database_name}"
 	kill -9 $brokerid

tests/manual/performance_broker/discovery.json

 {
   "Mode": "static",
   "Broker": {
-    "Endpoint": [
-      "localhost:5005"
+    "StaticEndpoint": "localhost:5005"
+  },
+  "Receiver": {
+    "MaxConnections":1,
+    "StaticEndpoints": [
+      "test"
     ]
   },
   "Port": 5006,

tests/manual/performance_broker/test.sh

@@ -4,7 +4,7 @@
 # reads fileset into database
 # calls getnext_broker example from $worker_node
 
-nthreads=1
+nthreads=16
 # a directory with many files in it
 dir=/gpfs/petra3/scratch/yakubov/test
 run_name=test
@@ -20,14 +20,10 @@ worker_dir=~/broker_test
 service_dir=~/broker_test
 
 
-cat settings.json |
-  jq "to_entries |
-       map(if .key == \"MonitorDbAddress\"
-          then . + {value:\"${monitor_node}:${monitor_port}\"}
-          else .
-          end
-         ) |
-      from_entries" > settings_tmp.json
+cat settings.json | jq ".MonitorDbAddress = \"${monitor_node}:${monitor_port}\"" > settings_tmp.json
+
+cat discovery.json | jq ".Broker.StaticEndpoint = \"${service_node}:5005\"" > discovery_tmp.json
+
 
 ssh ${monitor_node} influx -execute \"create database db_test\"
 
@@ -38,6 +34,13 @@ ssh ${service_node} mkdir -p ${service_dir}
 ssh ${worker_node} mkdir -p ${worker_dir}
 
 
+scp ../../../cmake-build-release/discovery/asapo-discovery ${service_node}:${service_dir}
+scp discovery_tmp.json ${service_node}:${service_dir}/discovery.json
+rm discovery_tmp.json
+
+ssh ${service_node} "bash -c 'cd ${service_dir}; nohup ./asapo-discovery -config discovery.json &> ${service_dir}/discovery.log &'"
+
+
 scp settings_tmp.json ${service_node}:${service_dir}/settings.json
 rm settings_tmp.json
 scp ../../../cmake-build-release/broker/asapo-broker ${service_node}:${service_dir}
@@ -49,10 +52,12 @@ ssh ${worker_node} ${worker_dir}/folder2db -n ${nthreads} ${dir} ${run_name} ${s
 sleep 3
 
 scp ../../../cmake-build-release/examples/worker/getnext_broker/getnext_broker ${worker_node}:${worker_dir}
-ssh ${worker_node} ${worker_dir}/getnext_broker ${service_node}:5005 ${run_name} ${nthreads}
+ssh ${worker_node} ${worker_dir}/getnext_broker ${service_node}:8400 ${run_name} ${nthreads}
 
 
 
 ssh ${service_node} killall asapo-broker
+ssh ${service_node} killall asapo-discovery
+
 ssh ${service_node} docker rm -f mongo
 #ssh ${service_node} docker rm -f influxdb

tests/manual/performance_full_chain_simple/discovery.json

 {
   "Mode": "static",
-  "Recevier": {
-    "Endpoints": [
+  "Receiver": {
+    "StaticEndpoints": [
       "localhost:4200"
     ],
     "MaxConnections": 32
   },
   "Broker": {
-    "Endpoint": [
-      "localhost:5005"
-    ]
+    "StaticEndpoint": "localhost:5005"
   },
   "Port": 5006,
   "LogLevel": "info"
-}
\ No newline at end of file
+}
+

tests/manual/performance_full_chain_simple/test.sh

@@ -25,8 +25,8 @@ log_dir=~/fullchain_tests/logs
 # starts receiver on $receiver_node
 # runs producer with various file sizes from $producer_node and measures performance
 
-file_size=100
-file_num=$((100000000 / $file_size))
+file_size=10000
+file_num=$((10000000 / $file_size))
 echo filesize: ${file_size}K, filenum: $file_num
 
 # receiver_setup
@@ -62,10 +62,15 @@ cat discovery.json |
           end
          ) |
       from_entries" > discovery_tmp.json
+
+cat discovery_tmp.json | jq ".Broker.StaticEndpoint = \"${receiver_node}:5005\"" > discovery_tmp1.json
+
+cat discovery_tmp1.json | jq ".Receiver.StaticEndpoints = [\"${receiver_node}:${receiver_port}\"]" > discovery_tmp2.json
+
 scp ../../../cmake-build-release/discovery/asapo-discovery ${receiver_node}:${receiver_dir}
-scp discovery_tmp.json ${receiver_node}:${receiver_dir}/discovery.json
+scp discovery_tmp2.json ${receiver_node}:${receiver_dir}/discovery.json
 discovery_ip=`resolveip -s ${receiver_node}`
-rm discovery_tmp.json
+rm discovery_tmp*.json
 
 #producer_setup
 producer_node=max-display001
@@ -120,10 +125,10 @@ sleep 0.3
 sleep 5
 
 #producer_start
-ssh ${producer_node} "bash -c 'cd ${producer_dir}; nohup ./dummy-data-producer ${discovery_ip}:${discovery_port} ${file_size} ${file_num} ${producer_nthreads} 0 &> ${log_dir}/producer.log &'"
+ssh ${producer_node} "bash -c 'cd ${producer_dir}; nohup ./dummy-data-producer ${receiver_node}:8400 ${file_size} ${file_num} ${producer_nthreads} 0 &> ${log_dir}/producer.log &'"
 
 sleep 1
 
 #worker_start
-ssh ${worker_node} ${worker_dir}/getnext_broker ${broker_node}:5005 test_run ${nthreads}
+ssh ${worker_node} ${worker_dir}/getnext_broker ${receiver_node}:8400 test_run ${nthreads}
 

tests/manual/performance_producer_receiver/discovery.json

 {
-  "MaxConnections": 32,
   "Mode": "static",
-  "Endpoints":["localhost:4200"],
+  "Receiver": {
+    "StaticEndpoints": [
+      "localhost:4200"
+    ],
+    "MaxConnections": 32
+  },
+  "Broker": {
+    "StaticEndpoint": "localhost:5005"
+  },
   "Port":5006,
   "LogLevel":"info"
 }
\ No newline at end of file

tests/manual/performance_producer_receiver/test.sh

@@ -56,20 +56,24 @@ cat receiver.json |
           end
          ) |
       from_entries" > receiver_tmp.json
+
 cat discovery.json |
   jq "to_entries |
        map(if .key == \"Port\"
           then . + {value:${discovery_port}}
-          elif .key == \"Endpoints\"
-          then . + {value:[\"${service_node}:${receiver_port}\"]}
           else .
           end
          ) |
       from_entries" > discovery_tmp.json
 
-scp discovery_tmp.json ${service_node}:${service_dir}/discovery.json
+cat discovery.json | jq ".Port = ${discovery_port}" > discovery_tmp.json
+
+cat discovery_tmp.json | jq ".Receiver.StaticEndpoints = [\"${service_node}:${receiver_port}\"]" > discovery_tmp1.json
+
+
+scp discovery_tmp1.json ${service_node}:${service_dir}/discovery.json
 scp receiver_tmp.json ${service_node}:${service_dir}/receiver.json
-rm discovery_tmp.json receiver_tmp.json
+rm discovery_tmp*.json receiver_tmp.json
 ssh ${service_node} "bash -c 'cd ${service_dir}; nohup ./receiver receiver.json &> ${service_dir}/receiver.log &'"
 ssh ${service_node} "bash -c 'cd ${service_dir}; nohup ./asapo-discovery -config discovery.json &> ${service_dir}/discovery.log &'"
 
@@ -78,7 +82,7 @@ for size  in 100 1000 10000
 do
 ssh ${service_node} docker run -d -p 27017:27017 --name mongo mongo
 echo ===================================================================
-ssh ${worker_node} ${worker_dir}/dummy-data-producer ${service_ip}:${discovery_port} ${size} 1000 8 0
+ssh ${worker_node} ${worker_dir}/dummy-data-producer ${service_ip}:8400 ${size} 1000 8 0
 if [ "$1" == "true" ]
 then
     ssh ${service_node} rm -f ${service_dir}/files/*