diff --git a/config/nomad/broker.nmd.in b/config/nomad/broker.nmd.in index c0032fca94faf877999ea092142d98eed5d08e99..2d1ad5c1d42202ea3074690328f1c51346ba563c 100644 --- a/config/nomad/broker.nmd.in +++ b/config/nomad/broker.nmd.in @@ -19,7 +19,6 @@ job "broker" { memory = 256 # 256MB network { port "broker" { - static = "5005" } } } diff --git a/tests/automatic/CMakeLists.txt b/tests/automatic/CMakeLists.txt index 8377afd4aafa7681b21b0900119db67032fb1073..c10641f7dcde8fde613477b389539ca0f60481cd 100644 --- a/tests/automatic/CMakeLists.txt +++ b/tests/automatic/CMakeLists.txt @@ -26,3 +26,8 @@ add_subdirectory(producer_receiver) add_subdirectory(full_chain) add_subdirectory(spd_logger) + + +if (UNIX) + add_subdirectory(high_avail) +endif() \ No newline at end of file diff --git a/tests/automatic/broker/CMakeLists.txt b/tests/automatic/broker/CMakeLists.txt index 24ec3cef4137fb098dffcf31f475da4355ae1e01..6f1685c7d12bf036a9fa6f126b357fd962d2b954 100644 --- a/tests/automatic/broker/CMakeLists.txt +++ b/tests/automatic/broker/CMakeLists.txt @@ -3,5 +3,4 @@ add_subdirectory(read_config) if (UNIX) add_subdirectory(check_monitoring) -add_subdirectory(check_mongo_restart) endif() \ No newline at end of file diff --git a/tests/automatic/high_avail/CMakeLists.txt b/tests/automatic/high_avail/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..1f966c6a9b36ff669ab768bbeb196fb9adb61869 --- /dev/null +++ b/tests/automatic/high_avail/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(broker_mongo_restart) +add_subdirectory(receiver_mongo_restart) +add_subdirectory(services_restart) diff --git a/tests/automatic/broker/check_mongo_restart/CMakeLists.txt b/tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt similarity index 84% rename from tests/automatic/broker/check_mongo_restart/CMakeLists.txt rename to tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt index a3d2838ce60d8687f25707e867889cdf7a232b03..7ec07a04e360a84d57bcc234422db38336b42a86 100644 --- a/tests/automatic/broker/check_mongo_restart/CMakeLists.txt +++ b/tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt @@ -1,4 +1,4 @@ -set(TARGET_NAME check_broker_mongo_restart) +set(TARGET_NAME broker_mongo_restart) ################################ # Testing diff --git a/tests/automatic/broker/check_mongo_restart/check_linux.sh b/tests/automatic/high_avail/broker_mongo_restart/check_linux.sh similarity index 100% rename from tests/automatic/broker/check_mongo_restart/check_linux.sh rename to tests/automatic/high_avail/broker_mongo_restart/check_linux.sh diff --git a/tests/automatic/producer_receiver/check_mongo_restart/CMakeLists.txt b/tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt similarity index 80% rename from tests/automatic/producer_receiver/check_mongo_restart/CMakeLists.txt rename to tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt index e76f25ff44af36e70e25e5440bf17658446fb218..bebdc30dc0556f1c0d1d781230996503835ac92e 100644 --- a/tests/automatic/producer_receiver/check_mongo_restart/CMakeLists.txt +++ b/tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt @@ -1,4 +1,4 @@ -set(TARGET_NAME check-mongo-restart) +set(TARGET_NAME receiver-mongo-restart) ################################ # Testing diff --git a/tests/automatic/producer_receiver/check_mongo_restart/check_linux.sh b/tests/automatic/high_avail/receiver_mongo_restart/check_linux.sh similarity index 100% rename from tests/automatic/producer_receiver/check_mongo_restart/check_linux.sh rename to tests/automatic/high_avail/receiver_mongo_restart/check_linux.sh diff --git a/tests/automatic/high_avail/services_restart/CMakeLists.txt b/tests/automatic/high_avail/services_restart/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b76e24f603dd672e65a59a6cc16ec261d61d84e2 --- /dev/null +++ b/tests/automatic/high_avail/services_restart/CMakeLists.txt @@ -0,0 +1,8 @@ +set(TARGET_NAME service_restart) + +################################ +# Testing +################################ +prepare_asapo() +add_script_test("${TARGET_NAME}-all" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext_broker> $<TARGET_PROPERTY:asapo,EXENAME> broker 1000 998" nomem) +add_script_test("${TARGET_NAME}-all-but-broker" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext_broker> $<TARGET_PROPERTY:asapo,EXENAME> receiver 1000 1000" nomem) diff --git a/tests/automatic/high_avail/services_restart/check_linux.sh b/tests/automatic/high_avail/services_restart/check_linux.sh new file mode 100644 index 0000000000000000000000000000000000000000..ce087be1017a1fad029b86962de54cd189a68fa2 --- /dev/null +++ b/tests/automatic/high_avail/services_restart/check_linux.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +set -e + +trap Cleanup EXIT + +beamtime_id=asapo_test +token=`$3 token -secret broker_secret.key $beamtime_id` + +monitor_database_name=db_test +proxy_address=127.0.0.1:8400 + +beamline=test + +Cleanup() { + echo cleanup + nomad stop nginx + nomad stop receiver + nomad stop discovery + nomad stop broker + nomad stop authorizer + echo "db.dropDatabase()" | mongo ${beamtime_id} + influx -execute "drop database ${monitor_database_name}" +} + +influx -execute "create database ${monitor_database_name}" + +sed -i 's/"WriteToDisk":true/"WriteToDisk":false/g' receiver.json.tpl +sed -i 's/info/debug/g' broker.json.tpl + +nomad run nginx.nmd +nomad run authorizer.nmd +nomad run receiver.nmd +nomad run discovery.nmd +nomad run broker.nmd + +sleep 1 + +echo "db.${beamtime_id}.insert({dummy:1})" | mongo ${beamtime_id} + + + +#producer +$1 localhost:8400 ${beamtime_id} 100 $5 4 0 100 & +#producerid=`echo $!` + + +#worker +$2 ${proxy_address} ${beamtime_id} 2 $token 20000 &> output.txt & + +sleep 1 + +nomad stop $4 +nomad stop authorizer +nomad stop discovery +nomad stop nginx +nomad stop receiver + +nomad run nginx.nmd +nomad run authorizer.nmd +nomad run discovery.nmd +nomad run receiver.nmd + +nomad run $4.nmd + +wait + +cat output.txt +nfiles=`cat output.txt | grep "Processed" | awk '{print $2;}'` +test $nfiles -ge $6 +rm output.txt \ No newline at end of file diff --git a/tests/automatic/producer_receiver/CMakeLists.txt b/tests/automatic/producer_receiver/CMakeLists.txt index 70f089ea89556bb9f0c722f58a3fa0276018e836..ad36394912da0e110b2707f73e0270452030d41f 100644 --- a/tests/automatic/producer_receiver/CMakeLists.txt +++ b/tests/automatic/producer_receiver/CMakeLists.txt @@ -1,5 +1,4 @@ add_subdirectory(transfer_single_file) if (UNIX) add_subdirectory(check_monitoring) - add_subdirectory(check_mongo_restart) endif() \ No newline at end of file diff --git a/worker/api/cpp/src/server_data_broker.cpp b/worker/api/cpp/src/server_data_broker.cpp index f517a770ffcbc1a0c39de35d035166c76921bd33..5dfe70249273bcd277a346d022a9fdb588129bc9 100644 --- a/worker/api/cpp/src/server_data_broker.cpp +++ b/worker/api/cpp/src/server_data_broker.cpp @@ -88,6 +88,7 @@ Error ServerDataBroker::ProcessRequest(std::string* response, std::string reques HttpCode code; *response = httpclient__->Get(RequestWithToken(request_uri), &code, &err); if (err != nullptr) { + current_broker_uri_ = ""; return err; } return HttpCodeToWorkerError(code); diff --git a/worker/api/cpp/unittests/test_server_broker.cpp b/worker/api/cpp/unittests/test_server_broker.cpp index 58e24e8eb57d448fa65ecec72ce283d9fa740575..005e1df4be6c2e7ec57b9861b06e7410d6f259c0 100644 --- a/worker/api/cpp/unittests/test_server_broker.cpp +++ b/worker/api/cpp/unittests/test_server_broker.cpp @@ -75,6 +75,14 @@ class ServerDataBrokerTests : public Test { )); } + void MockGetError() { + EXPECT_CALL(mock_http_client, Get_t(HasSubstr(expected_broker_uri), _, _)).WillOnce(DoAll( + SetArgPointee<1>(HttpCode::NotFound), + SetArgPointee<2>(asapo::IOErrorTemplates::kUnknownIOError.Generate().release()), + Return("") + )); + } + void MockGetBrokerUri() { EXPECT_CALL(mock_http_client, Get_t(HasSubstr(expected_server_uri + "/discovery/broker"), _, _)).WillOnce(DoAll( SetArgPointee<1>(HttpCode::OK), @@ -192,6 +200,19 @@ TEST_F(ServerDataBrokerTests, GetDoNotCallBrokerUriIfAlreadyFound) { } +TEST_F(ServerDataBrokerTests, GetBrokerUriAgainAfterConnectionError) { + MockGetBrokerUri(); + MockGetError(); + + data_broker->SetTimeout(0); + data_broker->GetNext(&info, nullptr); + Mock::VerifyAndClearExpectations(&mock_http_client); + + MockGetBrokerUri(); + MockGet("error_response"); + data_broker->GetNext(&info, nullptr); +} + TEST_F(ServerDataBrokerTests, GetNextReturnsEOFFromHttpClientUntilTimeout) { MockGetBrokerUri();