correct behaviour in case of broker restart

config/nomad/broker.nmd.in

@@ -19,7 +19,6 @@ job "broker" {
         memory = 256 # 256MB
         network {
           port "broker" {
-            static = "5005"
           }
         }
       }

tests/automatic/CMakeLists.txt

@@ -26,3 +26,8 @@ add_subdirectory(producer_receiver)
 add_subdirectory(full_chain)
 
 add_subdirectory(spd_logger)
+
+
+if (UNIX)
+    add_subdirectory(high_avail)
+endif()
\ No newline at end of file

tests/automatic/broker/CMakeLists.txt

@@ -3,5 +3,4 @@ add_subdirectory(read_config)
 
 if (UNIX)
 add_subdirectory(check_monitoring)
-add_subdirectory(check_mongo_restart)
 endif()
\ No newline at end of file

tests/automatic/high_avail/CMakeLists.txt

+add_subdirectory(broker_mongo_restart)
+add_subdirectory(receiver_mongo_restart)
+add_subdirectory(services_restart)

tests/automatic/broker/check_mongo_restart/CMakeLists.txt → tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt

-set(TARGET_NAME check_broker_mongo_restart)
+set(TARGET_NAME broker_mongo_restart)
 
 ################################
 # Testing

tests/automatic/producer_receiver/check_mongo_restart/CMakeLists.txt → tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt

-set(TARGET_NAME check-mongo-restart)
+set(TARGET_NAME receiver-mongo-restart)
 
 ################################
 # Testing

tests/automatic/high_avail/services_restart/CMakeLists.txt

+set(TARGET_NAME service_restart)
+
+################################
+# Testing
+################################
+prepare_asapo()
+add_script_test("${TARGET_NAME}-all" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext_broker> $<TARGET_PROPERTY:asapo,EXENAME> broker 1000 998" nomem)
+add_script_test("${TARGET_NAME}-all-but-broker" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext_broker> $<TARGET_PROPERTY:asapo,EXENAME> receiver 1000 1000" nomem)

tests/automatic/high_avail/services_restart/check_linux.sh

+#!/usr/bin/env bash
+
+set -e
+
+trap Cleanup EXIT
+
+beamtime_id=asapo_test
+token=`$3 token -secret broker_secret.key $beamtime_id`
+
+monitor_database_name=db_test
+proxy_address=127.0.0.1:8400
+
+beamline=test
+
+Cleanup() {
+    echo cleanup
+    nomad stop nginx
+    nomad stop receiver
+    nomad stop discovery
+    nomad stop broker
+    nomad stop authorizer
+    echo "db.dropDatabase()" | mongo ${beamtime_id}
+    influx -execute "drop database ${monitor_database_name}"
+}
+
+influx -execute "create database ${monitor_database_name}"
+
+sed -i 's/"WriteToDisk":true/"WriteToDisk":false/g' receiver.json.tpl
+sed -i 's/info/debug/g' broker.json.tpl
+
+nomad run nginx.nmd
+nomad run authorizer.nmd
+nomad run receiver.nmd
+nomad run discovery.nmd
+nomad run broker.nmd
+
+sleep 1
+
+echo "db.${beamtime_id}.insert({dummy:1})" | mongo  ${beamtime_id}
+
+
+
+#producer
+$1 localhost:8400 ${beamtime_id} 100 $5 4 0 100 &
+#producerid=`echo $!`
+
+
+#worker
+$2 ${proxy_address} ${beamtime_id} 2 $token 20000 &> output.txt &
+
+sleep 1
+
+nomad stop $4
+nomad stop authorizer
+nomad stop discovery
+nomad stop nginx
+nomad stop receiver
+
+nomad run nginx.nmd
+nomad run authorizer.nmd
+nomad run discovery.nmd
+nomad run receiver.nmd
+
+nomad run $4.nmd
+
+wait
+
+cat output.txt
+nfiles=`cat output.txt | grep "Processed" | awk   '{print $2;}'`
+test  $nfiles -ge $6
+rm output.txt
\ No newline at end of file

tests/automatic/producer_receiver/CMakeLists.txt

 add_subdirectory(transfer_single_file)
 if (UNIX)
     add_subdirectory(check_monitoring)
-    add_subdirectory(check_mongo_restart)
 endif()
\ No newline at end of file

worker/api/cpp/src/server_data_broker.cpp

@@ -88,6 +88,7 @@ Error ServerDataBroker::ProcessRequest(std::string* response, std::string reques
     HttpCode code;
     *response = httpclient__->Get(RequestWithToken(request_uri), &code, &err);
     if (err != nullptr) {
+        current_broker_uri_ = "";
         return err;
     }
     return HttpCodeToWorkerError(code);

worker/api/cpp/unittests/test_server_broker.cpp

@@ -75,6 +75,14 @@ class ServerDataBrokerTests : public Test {
                 ));
     }
 
+    void MockGetError() {
+        EXPECT_CALL(mock_http_client, Get_t(HasSubstr(expected_broker_uri), _, _)).WillOnce(DoAll(
+                    SetArgPointee<1>(HttpCode::NotFound),
+                    SetArgPointee<2>(asapo::IOErrorTemplates::kUnknownIOError.Generate().release()),
+                    Return("")
+                ));
+    }
+
     void MockGetBrokerUri() {
         EXPECT_CALL(mock_http_client, Get_t(HasSubstr(expected_server_uri + "/discovery/broker"), _, _)).WillOnce(DoAll(
                     SetArgPointee<1>(HttpCode::OK),
@@ -192,6 +200,19 @@ TEST_F(ServerDataBrokerTests, GetDoNotCallBrokerUriIfAlreadyFound) {
 }
 
 
+TEST_F(ServerDataBrokerTests, GetBrokerUriAgainAfterConnectionError) {
+    MockGetBrokerUri();
+    MockGetError();
+
+    data_broker->SetTimeout(0);
+    data_broker->GetNext(&info, nullptr);
+    Mock::VerifyAndClearExpectations(&mock_http_client);
+
+    MockGetBrokerUri();
+    MockGet("error_response");
+    data_broker->GetNext(&info, nullptr);
+}
+
 
 TEST_F(ServerDataBrokerTests, GetNextReturnsEOFFromHttpClientUntilTimeout) {
     MockGetBrokerUri();