From 83cb6cc87325c82cee6130774e4b440e250afc77 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Mon, 20 Aug 2018 13:41:13 +0200
Subject: [PATCH] correct behaviour in case of broker restart

---
 config/nomad/broker.nmd.in                    |  1 -
 tests/automatic/CMakeLists.txt                |  5 ++
 tests/automatic/broker/CMakeLists.txt         |  1 -
 tests/automatic/high_avail/CMakeLists.txt     |  3 +
 .../broker_mongo_restart}/CMakeLists.txt      |  2 +-
 .../broker_mongo_restart}/check_linux.sh      |  0
 .../receiver_mongo_restart}/CMakeLists.txt    |  2 +-
 .../receiver_mongo_restart}/check_linux.sh    |  0
 .../services_restart/CMakeLists.txt           |  8 +++
 .../services_restart/check_linux.sh           | 71 +++++++++++++++++++
 .../producer_receiver/CMakeLists.txt          |  1 -
 worker/api/cpp/src/server_data_broker.cpp     |  1 +
 .../api/cpp/unittests/test_server_broker.cpp  | 21 ++++++
 13 files changed, 111 insertions(+), 5 deletions(-)
 create mode 100644 tests/automatic/high_avail/CMakeLists.txt
 rename tests/automatic/{broker/check_mongo_restart => high_avail/broker_mongo_restart}/CMakeLists.txt (84%)
 rename tests/automatic/{broker/check_mongo_restart => high_avail/broker_mongo_restart}/check_linux.sh (100%)
 rename tests/automatic/{producer_receiver/check_mongo_restart => high_avail/receiver_mongo_restart}/CMakeLists.txt (80%)
 rename tests/automatic/{producer_receiver/check_mongo_restart => high_avail/receiver_mongo_restart}/check_linux.sh (100%)
 create mode 100644 tests/automatic/high_avail/services_restart/CMakeLists.txt
 create mode 100644 tests/automatic/high_avail/services_restart/check_linux.sh

diff --git a/config/nomad/broker.nmd.in b/config/nomad/broker.nmd.in
index c0032fca9..2d1ad5c1d 100644
--- a/config/nomad/broker.nmd.in
+++ b/config/nomad/broker.nmd.in
@@ -19,7 +19,6 @@ job "broker" {
         memory = 256 # 256MB
         network {
           port "broker" {
-            static = "5005"
           }
         }
       }
diff --git a/tests/automatic/CMakeLists.txt b/tests/automatic/CMakeLists.txt
index 8377afd4a..c10641f7d 100644
--- a/tests/automatic/CMakeLists.txt
+++ b/tests/automatic/CMakeLists.txt
@@ -26,3 +26,8 @@ add_subdirectory(producer_receiver)
 add_subdirectory(full_chain)
 
 add_subdirectory(spd_logger)
+
+
+if (UNIX)
+    add_subdirectory(high_avail)
+endif()
\ No newline at end of file
diff --git a/tests/automatic/broker/CMakeLists.txt b/tests/automatic/broker/CMakeLists.txt
index 24ec3cef4..6f1685c7d 100644
--- a/tests/automatic/broker/CMakeLists.txt
+++ b/tests/automatic/broker/CMakeLists.txt
@@ -3,5 +3,4 @@ add_subdirectory(read_config)
 
 if (UNIX)
 add_subdirectory(check_monitoring)
-add_subdirectory(check_mongo_restart)
 endif()
\ No newline at end of file
diff --git a/tests/automatic/high_avail/CMakeLists.txt b/tests/automatic/high_avail/CMakeLists.txt
new file mode 100644
index 000000000..1f966c6a9
--- /dev/null
+++ b/tests/automatic/high_avail/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_subdirectory(broker_mongo_restart)
+add_subdirectory(receiver_mongo_restart)
+add_subdirectory(services_restart)
diff --git a/tests/automatic/broker/check_mongo_restart/CMakeLists.txt b/tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt
similarity index 84%
rename from tests/automatic/broker/check_mongo_restart/CMakeLists.txt
rename to tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt
index a3d2838ce..7ec07a04e 100644
--- a/tests/automatic/broker/check_mongo_restart/CMakeLists.txt
+++ b/tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(TARGET_NAME check_broker_mongo_restart)
+set(TARGET_NAME broker_mongo_restart)
 
 ################################
 # Testing
diff --git a/tests/automatic/broker/check_mongo_restart/check_linux.sh b/tests/automatic/high_avail/broker_mongo_restart/check_linux.sh
similarity index 100%
rename from tests/automatic/broker/check_mongo_restart/check_linux.sh
rename to tests/automatic/high_avail/broker_mongo_restart/check_linux.sh
diff --git a/tests/automatic/producer_receiver/check_mongo_restart/CMakeLists.txt b/tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt
similarity index 80%
rename from tests/automatic/producer_receiver/check_mongo_restart/CMakeLists.txt
rename to tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt
index e76f25ff4..bebdc30dc 100644
--- a/tests/automatic/producer_receiver/check_mongo_restart/CMakeLists.txt
+++ b/tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt
@@ -1,4 +1,4 @@
-set(TARGET_NAME check-mongo-restart)
+set(TARGET_NAME receiver-mongo-restart)
 
 ################################
 # Testing
diff --git a/tests/automatic/producer_receiver/check_mongo_restart/check_linux.sh b/tests/automatic/high_avail/receiver_mongo_restart/check_linux.sh
similarity index 100%
rename from tests/automatic/producer_receiver/check_mongo_restart/check_linux.sh
rename to tests/automatic/high_avail/receiver_mongo_restart/check_linux.sh
diff --git a/tests/automatic/high_avail/services_restart/CMakeLists.txt b/tests/automatic/high_avail/services_restart/CMakeLists.txt
new file mode 100644
index 000000000..b76e24f60
--- /dev/null
+++ b/tests/automatic/high_avail/services_restart/CMakeLists.txt
@@ -0,0 +1,8 @@
+set(TARGET_NAME service_restart)
+
+################################
+# Testing
+################################
+prepare_asapo()
+add_script_test("${TARGET_NAME}-all" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext_broker> $<TARGET_PROPERTY:asapo,EXENAME> broker 1000 998" nomem)
+add_script_test("${TARGET_NAME}-all-but-broker" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext_broker> $<TARGET_PROPERTY:asapo,EXENAME> receiver 1000 1000" nomem)
diff --git a/tests/automatic/high_avail/services_restart/check_linux.sh b/tests/automatic/high_avail/services_restart/check_linux.sh
new file mode 100644
index 000000000..ce087be10
--- /dev/null
+++ b/tests/automatic/high_avail/services_restart/check_linux.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+set -e
+
+trap Cleanup EXIT
+
+beamtime_id=asapo_test
+token=`$3 token -secret broker_secret.key $beamtime_id`
+
+monitor_database_name=db_test
+proxy_address=127.0.0.1:8400
+
+beamline=test
+
+Cleanup() {
+    echo cleanup
+    nomad stop nginx
+    nomad stop receiver
+    nomad stop discovery
+    nomad stop broker
+    nomad stop authorizer
+    echo "db.dropDatabase()" | mongo ${beamtime_id}
+    influx -execute "drop database ${monitor_database_name}"
+}
+
+influx -execute "create database ${monitor_database_name}"
+
+sed -i 's/"WriteToDisk":true/"WriteToDisk":false/g' receiver.json.tpl
+sed -i 's/info/debug/g' broker.json.tpl
+
+nomad run nginx.nmd
+nomad run authorizer.nmd
+nomad run receiver.nmd
+nomad run discovery.nmd
+nomad run broker.nmd
+
+sleep 1
+
+echo "db.${beamtime_id}.insert({dummy:1})" | mongo  ${beamtime_id}
+
+
+
+#producer
+$1 localhost:8400 ${beamtime_id} 100 $5 4 0 100 &
+#producerid=`echo $!`
+
+
+#worker
+$2 ${proxy_address} ${beamtime_id} 2 $token 20000 &> output.txt &
+
+sleep 1
+
+nomad stop $4
+nomad stop authorizer
+nomad stop discovery
+nomad stop nginx
+nomad stop receiver
+
+nomad run nginx.nmd
+nomad run authorizer.nmd
+nomad run discovery.nmd
+nomad run receiver.nmd
+
+nomad run $4.nmd
+
+wait
+
+cat output.txt
+nfiles=`cat output.txt | grep "Processed" | awk   '{print $2;}'`
+test  $nfiles -ge $6
+rm output.txt
\ No newline at end of file
diff --git a/tests/automatic/producer_receiver/CMakeLists.txt b/tests/automatic/producer_receiver/CMakeLists.txt
index 70f089ea8..ad3639491 100644
--- a/tests/automatic/producer_receiver/CMakeLists.txt
+++ b/tests/automatic/producer_receiver/CMakeLists.txt
@@ -1,5 +1,4 @@
 add_subdirectory(transfer_single_file)
 if (UNIX)
     add_subdirectory(check_monitoring)
-    add_subdirectory(check_mongo_restart)
 endif()
\ No newline at end of file
diff --git a/worker/api/cpp/src/server_data_broker.cpp b/worker/api/cpp/src/server_data_broker.cpp
index f517a770f..5dfe70249 100644
--- a/worker/api/cpp/src/server_data_broker.cpp
+++ b/worker/api/cpp/src/server_data_broker.cpp
@@ -88,6 +88,7 @@ Error ServerDataBroker::ProcessRequest(std::string* response, std::string reques
     HttpCode code;
     *response = httpclient__->Get(RequestWithToken(request_uri), &code, &err);
     if (err != nullptr) {
+        current_broker_uri_ = "";
         return err;
     }
     return HttpCodeToWorkerError(code);
diff --git a/worker/api/cpp/unittests/test_server_broker.cpp b/worker/api/cpp/unittests/test_server_broker.cpp
index 58e24e8eb..005e1df4b 100644
--- a/worker/api/cpp/unittests/test_server_broker.cpp
+++ b/worker/api/cpp/unittests/test_server_broker.cpp
@@ -75,6 +75,14 @@ class ServerDataBrokerTests : public Test {
                 ));
     }
 
+    void MockGetError() {
+        EXPECT_CALL(mock_http_client, Get_t(HasSubstr(expected_broker_uri), _, _)).WillOnce(DoAll(
+                    SetArgPointee<1>(HttpCode::NotFound),
+                    SetArgPointee<2>(asapo::IOErrorTemplates::kUnknownIOError.Generate().release()),
+                    Return("")
+                ));
+    }
+
     void MockGetBrokerUri() {
         EXPECT_CALL(mock_http_client, Get_t(HasSubstr(expected_server_uri + "/discovery/broker"), _, _)).WillOnce(DoAll(
                     SetArgPointee<1>(HttpCode::OK),
@@ -192,6 +200,19 @@ TEST_F(ServerDataBrokerTests, GetDoNotCallBrokerUriIfAlreadyFound) {
 }
 
 
+TEST_F(ServerDataBrokerTests, GetBrokerUriAgainAfterConnectionError) {
+    MockGetBrokerUri();
+    MockGetError();
+
+    data_broker->SetTimeout(0);
+    data_broker->GetNext(&info, nullptr);
+    Mock::VerifyAndClearExpectations(&mock_http_client);
+
+    MockGetBrokerUri();
+    MockGet("error_response");
+    data_broker->GetNext(&info, nullptr);
+}
+
 
 TEST_F(ServerDataBrokerTests, GetNextReturnsEOFFromHttpClientUntilTimeout) {
     MockGetBrokerUri();
-- 
GitLab