From f0cb1ac2698cb8b3a36e179bed3778ba4e06a2f6 Mon Sep 17 00:00:00 2001 From: Sergey Yakubov <sergey.yakubov@desy.de> Date: Wed, 26 May 2021 11:32:08 +0200 Subject: [PATCH] refactored more tests --- .../consumer/consumer_api/CMakeLists.txt | 1 - .../consumer/consumer_api/check_linux.sh | 15 --------- .../consumer/consumer_api/check_windows.bat | 3 -- .../consumer_api_python/CMakeLists.txt | 6 ---- .../authorizer_settings.json.tpl.in | 15 --------- .../consumer_api_python/check_linux.sh | 14 +------- .../consumer_api_python/check_windows.bat | 5 ++- .../next_multithread_broker/CMakeLists.txt | 1 - .../next_multithread_broker/check_linux.sh | 15 +-------- .../curl_http_client_command/CMakeLists.txt | 4 --- .../authorizer_settings.json.tpl.in | 15 --------- .../curl_http_client_command/check_linux.sh | 8 +---- .../check_windows.bat | 11 +------ .../rest_api/CMakeLists.txt | 2 -- .../rest_api/authorizer_settings.json.tpl.in | 15 --------- .../rest_api/check_linux.sh | 9 +----- .../rest_api/check_windows.bat | 10 +----- .../send_recv_streams/CMakeLists.txt | 2 -- .../send_recv_streams/check_linux.sh | 13 -------- .../send_recv_streams/check_windows.bat | 3 -- .../send_recv_streams_python/CMakeLists.txt | 1 - .../send_recv_streams_python/check_linux.sh | 14 +------- .../check_windows.bat | 3 -- .../full_chain/simple_chain/CMakeLists.txt | 2 +- .../full_chain/simple_chain/check_linux.sh | 20 +----------- .../full_chain/simple_chain/check_windows.bat | 2 -- .../simple_chain_dataset/CMakeLists.txt | 2 +- .../simple_chain_dataset/check_linux.sh | 14 -------- .../simple_chain_dataset/check_windows.bat | 3 -- .../simple_chain_filegen/CMakeLists.txt | 4 +-- .../simple_chain_filegen/check_linux.sh | 15 --------- .../simple_chain_filegen/check_windows.bat | 5 --- .../CMakeLists.txt | 4 +-- .../check_linux.sh | 17 ---------- .../check_windows.bat | 5 --- .../CMakeLists.txt | 2 +- .../check_linux.sh | 14 -------- .../check_windows.bat | 5 --- .../CMakeLists.txt | 5 ++- .../check_linux.sh | 25 ++++++--------- .../check_windows.bat | 6 ---- .../CMakeLists.txt | 4 +-- .../check_linux.sh | 14 -------- .../check_windows.bat | 3 -- .../simple_chain_metadata/CMakeLists.txt | 4 +-- .../simple_chain_metadata/check_linux.sh | 14 -------- .../simple_chain_metadata/check_windows.bat | 3 -- .../simple_chain_raw/CMakeLists.txt | 15 +-------- .../simple_chain_raw/check_linux.sh | 32 +++++-------------- .../simple_chain_raw/check_windows.bat | 19 ++++------- .../simple_chain_raw/settings.json.tpl.in | 14 -------- .../CMakeLists.txt | 3 +- .../check_linux.sh | 14 -------- .../check_windows.bat | 3 -- .../full_chain/two_beamlines/CMakeLists.txt | 4 ++- .../full_chain/two_beamlines/check_linux.sh | 25 ++++++--------- .../two_beamlines/check_windows.bat | 3 -- .../full_chain/two_streams/CMakeLists.txt | 2 +- .../full_chain/two_streams/check_linux.sh | 14 -------- .../full_chain/two_streams/check_windows.bat | 3 -- .../broker_mongo_restart/CMakeLists.txt | 2 +- .../broker_mongo_restart/check_linux.sh | 15 --------- .../receiver_mongo_restart/CMakeLists.txt | 2 +- .../receiver_mongo_restart/check_linux.sh | 19 ++--------- .../services_restart/CMakeLists.txt | 4 +-- .../services_restart/check_linux.sh | 23 +++---------- tests/automatic/producer/aai/CMakeLists.txt | 12 +------ tests/automatic/producer/aai/check_linux.sh | 26 ++++++--------- .../automatic/producer/aai/check_windows.bat | 11 ++++--- tests/automatic/producer/aai/producer_aai.py | 2 +- .../producer/aai/settings.json.tpl.in | 9 ------ .../producer/python_api/CMakeLists.txt | 4 --- .../producer/python_api/check_linux.sh | 12 ------- .../producer/python_api/check_windows.bat | 3 -- 74 files changed, 94 insertions(+), 578 deletions(-) delete mode 100644 tests/automatic/consumer/consumer_api_python/authorizer_settings.json.tpl.in delete mode 100644 tests/automatic/curl_http_client/curl_http_client_command/authorizer_settings.json.tpl.in delete mode 100644 tests/automatic/file_transfer_service/rest_api/authorizer_settings.json.tpl.in delete mode 100644 tests/automatic/full_chain/simple_chain_raw/settings.json.tpl.in delete mode 100644 tests/automatic/producer/aai/settings.json.tpl.in diff --git a/tests/automatic/consumer/consumer_api/CMakeLists.txt b/tests/automatic/consumer/consumer_api/CMakeLists.txt index f31804954..d7f451aa2 100644 --- a/tests/automatic/consumer/consumer_api/CMakeLists.txt +++ b/tests/automatic/consumer/consumer_api/CMakeLists.txt @@ -11,7 +11,6 @@ target_link_libraries(${TARGET_NAME} test_common asapo-consumer) ################################ # Testing ################################ -prepare_asapo() add_script_test("${TARGET_NAME}" "$<TARGET_FILE:${TARGET_NAME}>" ) diff --git a/tests/automatic/consumer/consumer_api/check_linux.sh b/tests/automatic/consumer/consumer_api/check_linux.sh index ceb0979c9..c5c715180 100644 --- a/tests/automatic/consumer/consumer_api/check_linux.sh +++ b/tests/automatic/consumer/consumer_api/check_linux.sh @@ -6,30 +6,16 @@ database_name=${beamtime_id}_${data_source} token_test_run=$BT_TEST_RUN_RW_TOKEN - set -e trap Cleanup EXIT Cleanup() { set +e - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop discovery - nomad stop authorizer - nomad stop broker echo "db.dropDatabase()" | mongo ${database_name} rm -f 1_1 1 } - -nomad run nginx.nmd -nomad run discovery.nmd -nomad run authorizer.nmd -nomad run broker.nmd - -sleep 1 - for i in `seq 1 10`; do echo 'db.data_default.insert({"_id":'$i',"size":6,"name":"'$i'","timestamp":0,"source":"none","buf_id":0,"dataset_substream":0,"meta":{"test":10}})' | mongo ${database_name} @@ -53,7 +39,6 @@ echo hello1 > 1 $@ 127.0.0.1:8400 $beamtime_id $token_test_run single - #check datasets echo "db.dropDatabase()" | mongo ${database_name} diff --git a/tests/automatic/consumer/consumer_api/check_windows.bat b/tests/automatic/consumer/consumer_api/check_windows.bat index 57eac5617..040bd17ee 100644 --- a/tests/automatic/consumer/consumer_api/check_windows.bat +++ b/tests/automatic/consumer/consumer_api/check_windows.bat @@ -6,8 +6,6 @@ SET database_name=%beamtime_id%_%data_source% SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe" set token_test_run=%BT_TEST_RUN_RW_TOKEN% -call start_services.bat - for /l %%x in (1, 1, 10) do echo db.data_default.insert({"_id":%%x,"size":6,"name":"%%x","timestamp":0,"source":"none","buf_id":0,"dataset_substream":0,"meta":{"test":10}}) | %mongo_exe% %database_name% || goto :error for /l %%x in (1, 1, 5) do echo db.data_stream1.insert({"_id":%%x,"size":6,"name":"1%%x","timestamp":1000,"source":"none","buf_id":0,"dataset_substream":0,"meta":{"test":10}}) | %mongo_exe% %database_name% || goto :error @@ -39,7 +37,6 @@ call :clean exit /b 1 :clean -call stop_services.bat echo db.dropDatabase() | %mongo_exe% %database_name% del "1 1_1" diff --git a/tests/automatic/consumer/consumer_api_python/CMakeLists.txt b/tests/automatic/consumer/consumer_api_python/CMakeLists.txt index a4afb1b75..c5c60edc0 100644 --- a/tests/automatic/consumer/consumer_api_python/CMakeLists.txt +++ b/tests/automatic/consumer/consumer_api_python/CMakeLists.txt @@ -1,16 +1,10 @@ set(TARGET_NAME consumer_api_python) - -prepare_asapo() - if (UNIX) get_target_property(PYTHON_LIBS python-lib-consumer BINARY_DIR) else() get_target_property(PYTHON_LIBS asapo_consumer BINARY_DIR) endif() -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/authorizer_settings.json.tpl.in authorizer.json.tpl @ONLY) - - add_script_test("${TARGET_NAME}" "${PYTHON_LIBS} ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}" nomem) diff --git a/tests/automatic/consumer/consumer_api_python/authorizer_settings.json.tpl.in b/tests/automatic/consumer/consumer_api_python/authorizer_settings.json.tpl.in deleted file mode 100644 index 611bdefec..000000000 --- a/tests/automatic/consumer/consumer_api_python/authorizer_settings.json.tpl.in +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Port": {{ env "NOMAD_PORT_authorizer" }}, - "LogLevel":"debug", - "RootBeamtimesFolder":"@ASAP3_FOLDER@", - "CurrentBeamlinesFolder":"@CURRENT_BEAMLINES_FOLDER@", - "UserSecretFile":"auth_secret.key", - "AdminSecretFile":"auth_secret_admin.key", - "TokenDurationMin":600, - "Ldap": - { - "Uri" : "ldap://localhost:389", - "BaseDn" : "ou=rgy,o=desy,c=de", - "FilterTemplate" : "(cn=a3__BEAMLINE__-hosts)" - } -} diff --git a/tests/automatic/consumer/consumer_api_python/check_linux.sh b/tests/automatic/consumer/consumer_api_python/check_linux.sh index c927068ad..85fb840de 100644 --- a/tests/automatic/consumer/consumer_api_python/check_linux.sh +++ b/tests/automatic/consumer/consumer_api_python/check_linux.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash beamtime_id=test_run -source_path=`pwd`/asap3/petra3/gpfs/p01/2019/data/$beamtime_id +source_path=/tmp/asapo/asap3/petra3/gpfs/p01/2019/data/$beamtime_id data_source=detector database_name=${beamtime_id}_${data_source} token_test_run=$BT_TEST_RUN_RW_TOKEN @@ -11,22 +11,10 @@ trap Cleanup EXIT Cleanup() { set +e - nomad stop nginx >/dev/null - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop discovery >/dev/null - nomad stop broker >/dev/null - nomad stop file_transfer >/dev/null - nomad stop authorizer >/dev/null echo "db.dropDatabase()" | mongo ${database_name} >/dev/null rm $source_path/1 $source_path/1_1 } -nomad run nginx.nmd -nomad run discovery.nmd -nomad run broker.nmd -nomad run file_transfer.nmd -nomad run authorizer.nmd - mkdir -p $source_path echo -n hello1 > $source_path/1 diff --git a/tests/automatic/consumer/consumer_api_python/check_windows.bat b/tests/automatic/consumer/consumer_api_python/check_windows.bat index e584db302..68824528f 100644 --- a/tests/automatic/consumer/consumer_api_python/check_windows.bat +++ b/tests/automatic/consumer/consumer_api_python/check_windows.bat @@ -1,6 +1,6 @@ setlocal SET beamtime_id=test_run -SET source_path=%cd%\asap3\petra3\gpfs\p01\2019\data\%beamtime_id% +SET source_path=c:\tmp\asapo\asap3\petra3\gpfs\p01\2019\data\%beamtime_id% set source_path=%source_path:\=\\% SET data_source=detector @@ -9,7 +9,6 @@ SET database_name=%beamtime_id%_%data_source% SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe" set token_test_run=%BT_TEST_RUN_RW_TOKEN% -call start_services.bat for /l %%x in (1, 1, 5) do echo db.data_default.insert({"_id":%%x,"size":6,"name":"%%x","timestamp":0,"source":"none","buf_id":0,"dataset_substream":0,"meta":{"test":10}}) | %mongo_exe% %database_name% || goto :error @@ -49,9 +48,9 @@ call :clean exit /b 1 :clean -call stop_services.bat echo db.dropDatabase() | %mongo_exe% %database_name% del c:\tmp\asapo\consumer_test\files\1 del c:\tmp\asapo\consumer_test\files\1_1 +rmdir /S /Q %source_path% diff --git a/tests/automatic/consumer/next_multithread_broker/CMakeLists.txt b/tests/automatic/consumer/next_multithread_broker/CMakeLists.txt index cb89f1c4f..8225e3c88 100644 --- a/tests/automatic/consumer/next_multithread_broker/CMakeLists.txt +++ b/tests/automatic/consumer/next_multithread_broker/CMakeLists.txt @@ -12,7 +12,6 @@ target_link_libraries(${TARGET_NAME} test_common asapo-consumer) ################################ if (UNIX) - prepare_asapo() add_script_test("${TARGET_NAME}" "$<TARGET_FILE:${TARGET_NAME}>") endif() diff --git a/tests/automatic/consumer/next_multithread_broker/check_linux.sh b/tests/automatic/consumer/next_multithread_broker/check_linux.sh index d507f1e9f..02e486550 100644 --- a/tests/automatic/consumer/next_multithread_broker/check_linux.sh +++ b/tests/automatic/consumer/next_multithread_broker/check_linux.sh @@ -8,22 +8,9 @@ trap Cleanup EXIT Cleanup() { set +e - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop discovery - nomad stop authorizer - nomad stop broker - echo "db.dropDatabase()" | mongo ${database_name} + echo "db.dropDatabase()" | mongo ${database_name} } - -nomad run nginx.nmd -nomad run discovery.nmd -nomad run authorizer.nmd -nomad run broker.nmd - -sleep 1 - for i in `seq 1 10`; do echo 'db.data_default.insert({"_id":'$i',"size":100,"name":"'$i'","timestamp":0,"source":"none","buf_id":0,"dataset_substream":0,"meta":{"test":10}})' | mongo ${database_name} diff --git a/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt b/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt index 506902ea3..64ea405e7 100644 --- a/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt +++ b/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt @@ -18,8 +18,4 @@ target_link_libraries(${TARGET_NAME} test_common asapo-consumer) # Testing ################################ -prepare_asapo() - -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/authorizer_settings.json.tpl.in authorizer.json.tpl @ONLY) - add_script_test("${TARGET_NAME}" "$<TARGET_FILE:${TARGET_NAME}>" nomem) diff --git a/tests/automatic/curl_http_client/curl_http_client_command/authorizer_settings.json.tpl.in b/tests/automatic/curl_http_client/curl_http_client_command/authorizer_settings.json.tpl.in deleted file mode 100644 index 611bdefec..000000000 --- a/tests/automatic/curl_http_client/curl_http_client_command/authorizer_settings.json.tpl.in +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Port": {{ env "NOMAD_PORT_authorizer" }}, - "LogLevel":"debug", - "RootBeamtimesFolder":"@ASAP3_FOLDER@", - "CurrentBeamlinesFolder":"@CURRENT_BEAMLINES_FOLDER@", - "UserSecretFile":"auth_secret.key", - "AdminSecretFile":"auth_secret_admin.key", - "TokenDurationMin":600, - "Ldap": - { - "Uri" : "ldap://localhost:389", - "BaseDn" : "ou=rgy,o=desy,c=de", - "FilterTemplate" : "(cn=a3__BEAMLINE__-hosts)" - } -} diff --git a/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh b/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh index 406b111aa..44935e94b 100644 --- a/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh +++ b/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh @@ -5,19 +5,13 @@ set -e trap Cleanup EXIT beamtime_id=aaa -file_transfer_folder=`pwd`/asap3/petra3/gpfs/p01/2019/data/$beamtime_id +file_transfer_folder=/tmp/asapo/asap3/petra3/gpfs/p01/2019/data/$beamtime_id Cleanup() { echo cleanup - nomad stop authorizer - nomad stop file_transfer rm -rf $file_transfer_folder bbb random } -nomad run authorizer.nmd -nomad run file_transfer.nmd -sleep 1 - mkdir -p $file_transfer_folder echo -n hello > $file_transfer_folder/aaa diff --git a/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat b/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat index a9382c3c9..802493da0 100644 --- a/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat +++ b/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat @@ -1,15 +1,8 @@ setlocal SET beamtime_id=aaa -SET file_transfer_folder=%cd%\asap3\petra3\gpfs\p01\2019\data\%beamtime_id% +SET file_transfer_folder=c:\tmp\asapo\asap3\petra3\gpfs\p01\2019\data\%beamtime_id% set file_transfer_folder=%file_transfer_folder:\=\\% - - -c:\opt\consul\nomad run authorizer.nmd -c:\opt\consul\nomad run file_transfer.nmd - -ping 192.0.2.1 -n 1 -w 1000 > nul - mkdir %file_transfer_folder% echo | set /p dummyName="hello" > %file_transfer_folder%\aaa @@ -26,8 +19,6 @@ call :clean exit /b 1 :clean -c:\opt\consul\nomad stop authorizer -c:\opt\consul\nomad stop file_transfer rmdir /S /Q %file_transfer_folder% del /f bbb random diff --git a/tests/automatic/file_transfer_service/rest_api/CMakeLists.txt b/tests/automatic/file_transfer_service/rest_api/CMakeLists.txt index 7ead397a7..89ec08831 100644 --- a/tests/automatic/file_transfer_service/rest_api/CMakeLists.txt +++ b/tests/automatic/file_transfer_service/rest_api/CMakeLists.txt @@ -3,7 +3,5 @@ set(TARGET_NAME file_transfer_rest_api) ################################ # Testing ################################ -prepare_asapo() -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/authorizer_settings.json.tpl.in authorizer.json.tpl @ONLY) add_script_test("${TARGET_NAME}" "" nomem) diff --git a/tests/automatic/file_transfer_service/rest_api/authorizer_settings.json.tpl.in b/tests/automatic/file_transfer_service/rest_api/authorizer_settings.json.tpl.in deleted file mode 100644 index 611bdefec..000000000 --- a/tests/automatic/file_transfer_service/rest_api/authorizer_settings.json.tpl.in +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Port": {{ env "NOMAD_PORT_authorizer" }}, - "LogLevel":"debug", - "RootBeamtimesFolder":"@ASAP3_FOLDER@", - "CurrentBeamlinesFolder":"@CURRENT_BEAMLINES_FOLDER@", - "UserSecretFile":"auth_secret.key", - "AdminSecretFile":"auth_secret_admin.key", - "TokenDurationMin":600, - "Ldap": - { - "Uri" : "ldap://localhost:389", - "BaseDn" : "ou=rgy,o=desy,c=de", - "FilterTemplate" : "(cn=a3__BEAMLINE__-hosts)" - } -} diff --git a/tests/automatic/file_transfer_service/rest_api/check_linux.sh b/tests/automatic/file_transfer_service/rest_api/check_linux.sh index a1d034a96..fa404b575 100644 --- a/tests/automatic/file_transfer_service/rest_api/check_linux.sh +++ b/tests/automatic/file_transfer_service/rest_api/check_linux.sh @@ -4,21 +4,14 @@ set -e trap Cleanup EXIT -file_transfer_folder=`pwd`/asap3/petra3/gpfs/p01/2019/data/aaa +file_transfer_folder=/tmp/asapo/asap3/petra3/gpfs/p01/2019/data/aaa Cleanup() { echo cleanup - nomad stop authorizer - nomad stop file_transfer rm -rf $file_transfer_folder aaa big_file } -nomad run authorizer.nmd -nomad run file_transfer.nmd - -sleep 1 - mkdir -p $file_transfer_folder token=$BT_AAA_TOKEN diff --git a/tests/automatic/file_transfer_service/rest_api/check_windows.bat b/tests/automatic/file_transfer_service/rest_api/check_windows.bat index f34c5ac4a..047b29362 100644 --- a/tests/automatic/file_transfer_service/rest_api/check_windows.bat +++ b/tests/automatic/file_transfer_service/rest_api/check_windows.bat @@ -1,14 +1,8 @@ setlocal SET beamtime_id=aaa -SET file_transfer_folder=%cd%\asap3\petra3\gpfs\p01\2019\data\%beamtime_id% +SET file_transfer_folder=c:\tmp\asapo\asap3\petra3\gpfs\p01\2019\data\%beamtime_id% set file_transfer_folder=%file_transfer_folder:\=\\% - -c:\opt\consul\nomad run authorizer.nmd -c:\opt\consul\nomad run file_transfer.nmd - -ping 192.0.2.1 -n 1 -w 1000 > nul - set token=%BT_AAA_TOKEN% mkdir %file_transfer_folder% @@ -29,6 +23,4 @@ call :clean exit /b 1 :clean -c:\opt\consul\nomad stop authorizer -c:\opt\consul\nomad stop file_transfer rmdir /S /Q %file_transfer_folder% diff --git a/tests/automatic/full_chain/send_recv_streams/CMakeLists.txt b/tests/automatic/full_chain/send_recv_streams/CMakeLists.txt index 0a2ab15e8..8a6144bf2 100644 --- a/tests/automatic/full_chain/send_recv_streams/CMakeLists.txt +++ b/tests/automatic/full_chain/send_recv_streams/CMakeLists.txt @@ -9,7 +9,5 @@ set_target_properties(${TARGET_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}$<$<CONFIG:Debug>:> ) -prepare_asapo() - add_script_test("${TARGET_NAME}-tcp" "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} tcp" nomem) diff --git a/tests/automatic/full_chain/send_recv_streams/check_linux.sh b/tests/automatic/full_chain/send_recv_streams/check_linux.sh index 49aaba84a..d1e48ef1f 100644 --- a/tests/automatic/full_chain/send_recv_streams/check_linux.sh +++ b/tests/automatic/full_chain/send_recv_streams/check_linux.sh @@ -19,21 +19,8 @@ network_type=$2 Cleanup() { set +e - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop discovery - nomad stop broker - nomad stop receiver - nomad stop authorizer echo "db.dropDatabase()" | mongo ${indatabase_name} } -nomad run nginx.nmd -nomad run discovery.nmd -nomad run broker.nmd -nomad run receiver_tcp.nmd -nomad run authorizer.nmd - - $1 127.0.0.1:8400 $beamtime_id $token | tee out diff --git a/tests/automatic/full_chain/send_recv_streams/check_windows.bat b/tests/automatic/full_chain/send_recv_streams/check_windows.bat index e4c54b28b..38a7bfb1f 100644 --- a/tests/automatic/full_chain/send_recv_streams/check_windows.bat +++ b/tests/automatic/full_chain/send_recv_streams/check_windows.bat @@ -10,8 +10,6 @@ SET beamline=test SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe" -call start_services.bat - "%1" 127.0.0.1:8400 %beamtime_id% %token% goto :clean @@ -21,6 +19,5 @@ call :clean exit /b 1 :clean -call stop_services.bat echo db.dropDatabase() | %mongo_exe% %indatabase_name% diff --git a/tests/automatic/full_chain/send_recv_streams_python/CMakeLists.txt b/tests/automatic/full_chain/send_recv_streams_python/CMakeLists.txt index f592e6f8b..2029410c9 100644 --- a/tests/automatic/full_chain/send_recv_streams_python/CMakeLists.txt +++ b/tests/automatic/full_chain/send_recv_streams_python/CMakeLists.txt @@ -1,5 +1,4 @@ set(TARGET_NAME send_recv_streams_python) -prepare_asapo() if (UNIX) get_target_property(PYTHON_LIBS_CONSUMER python-lib-consumer BINARY_DIR) diff --git a/tests/automatic/full_chain/send_recv_streams_python/check_linux.sh b/tests/automatic/full_chain/send_recv_streams_python/check_linux.sh index 490f7cd40..7e9b4f3db 100644 --- a/tests/automatic/full_chain/send_recv_streams_python/check_linux.sh +++ b/tests/automatic/full_chain/send_recv_streams_python/check_linux.sh @@ -14,22 +14,10 @@ set -e trap Cleanup EXIT Cleanup() { - set +e - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop discovery - nomad stop broker - nomad stop receiver - nomad stop authorizer + set +e echo "db.dropDatabase()" | mongo ${indatabase_name} } -nomad run nginx.nmd -nomad run discovery.nmd -nomad run broker.nmd -nomad run receiver_tcp.nmd -nomad run authorizer.nmd - export PYTHONPATH=$2:$3:${PYTHONPATH} diff --git a/tests/automatic/full_chain/send_recv_streams_python/check_windows.bat b/tests/automatic/full_chain/send_recv_streams_python/check_windows.bat index 8df892351..0c0dc6a86 100644 --- a/tests/automatic/full_chain/send_recv_streams_python/check_windows.bat +++ b/tests/automatic/full_chain/send_recv_streams_python/check_windows.bat @@ -9,8 +9,6 @@ SET beamline=test SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe" -call start_services.bat - "%1" 127.0.0.1:8400 %beamtime_id% %token% goto :clean @@ -20,5 +18,4 @@ call :clean exit /b 1 :clean -call stop_services.bat echo db.dropDatabase() | %mongo_exe% %indatabase_name% diff --git a/tests/automatic/full_chain/simple_chain/CMakeLists.txt b/tests/automatic/full_chain/simple_chain/CMakeLists.txt index 84ddc34f2..39feaf30f 100644 --- a/tests/automatic/full_chain/simple_chain/CMakeLists.txt +++ b/tests/automatic/full_chain/simple_chain/CMakeLists.txt @@ -3,5 +3,5 @@ set(TARGET_NAME full_chain_simple_chain) ################################ # Testing ################################ -prepare_asapo() +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) add_script_test("${TARGET_NAME}" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME>" nomem) diff --git a/tests/automatic/full_chain/simple_chain/check_linux.sh b/tests/automatic/full_chain/simple_chain/check_linux.sh index 0baea68ff..396e7ae32 100755 --- a/tests/automatic/full_chain/simple_chain/check_linux.sh +++ b/tests/automatic/full_chain/simple_chain/check_linux.sh @@ -23,40 +23,22 @@ receiver_folder=${receiver_root_folder}/${facility}/gpfs/${beamline}/${year}/dat Cleanup() { echo cleanup rm -rf ${receiver_root_folder} - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer rm -rf out - # kill $producerid echo "db.dropDatabase()" | mongo ${beamtime_id}_detector influx -execute "drop database ${monitor_database_name}" } echo "db.${beamtime_id}_detector.insert({dummy:1})" | mongo ${beamtime_id}_detector - echo "db.dropDatabase()" | mongo ${beamtime_id}_detector - -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_tcp.nmd # Only use TCP because the consumer will only use metadata anyways -nomad run discovery.nmd -nomad run broker.nmd - -sleep 1 - token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` echo "Start producer" mkdir -p ${receiver_folder} $producer_bin localhost:8400 ${beamtime_id} 100 1000 4 0 100 -#producerid=`echo $!` echo "Start consumer in metadata only mode" -$consumer_bin ${proxy_address} ${receiver_folder} ${beamtime_id} 2 $token 5000 1 | tee out +$consumer_bin ${proxy_address} ${receiver_folder} ${beamtime_id} 2 $token 2000 1 | tee out grep "Processed 1000 file(s)" out grep -i "Using connection type: No connection" out diff --git a/tests/automatic/full_chain/simple_chain/check_windows.bat b/tests/automatic/full_chain/simple_chain/check_windows.bat index f6a7cb792..8ce75eb68 100644 --- a/tests/automatic/full_chain/simple_chain/check_windows.bat +++ b/tests/automatic/full_chain/simple_chain/check_windows.bat @@ -9,7 +9,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id%_detector.insert({dummy:1}) | %mongo_exe% %beamtime_id%_detector -call start_services.bat "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token set /P token=< token @@ -31,7 +30,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% del /f token echo db.dropDatabase() | %mongo_exe% %beamtime_id%_detector diff --git a/tests/automatic/full_chain/simple_chain_dataset/CMakeLists.txt b/tests/automatic/full_chain/simple_chain_dataset/CMakeLists.txt index c6014bf7b..2c35556c9 100644 --- a/tests/automatic/full_chain/simple_chain_dataset/CMakeLists.txt +++ b/tests/automatic/full_chain/simple_chain_dataset/CMakeLists.txt @@ -3,5 +3,5 @@ set(TARGET_NAME full_chain_simple_chain_dataset) ################################ # Testing ################################ -prepare_asapo() +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> tcp" nomem) diff --git a/tests/automatic/full_chain/simple_chain_dataset/check_linux.sh b/tests/automatic/full_chain/simple_chain_dataset/check_linux.sh index 93f9960a8..48b6d86d3 100644 --- a/tests/automatic/full_chain/simple_chain_dataset/check_linux.sh +++ b/tests/automatic/full_chain/simple_chain_dataset/check_linux.sh @@ -24,12 +24,6 @@ receiver_folder=${receiver_root_folder}/${facility}/gpfs/${beamline}/${year}/dat Cleanup() { echo cleanup rm -rf ${receiver_root_folder} - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer rm -rf out echo "db.dropDatabase()" | mongo ${beamtime_id}_detector influx -execute "drop database ${monitor_database_name}" @@ -37,14 +31,6 @@ Cleanup() { echo "db.${beamtime_id}_detector.insert({dummy:1})" | mongo ${beamtime_id}_detector -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_${network_type}.nmd -nomad run discovery.nmd -nomad run broker.nmd - -sleep 1 - token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` echo "Start producer" diff --git a/tests/automatic/full_chain/simple_chain_dataset/check_windows.bat b/tests/automatic/full_chain/simple_chain_dataset/check_windows.bat index 7057a8d9b..9e29ddb7d 100644 --- a/tests/automatic/full_chain/simple_chain_dataset/check_windows.bat +++ b/tests/automatic/full_chain/simple_chain_dataset/check_windows.bat @@ -8,8 +8,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id%_detector.insert({dummy:1}) | %mongo_exe% %beamtime_id%_detector -call start_services.bat - "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token set /P token=< token @@ -32,7 +30,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% del /f token echo db.dropDatabase() | %mongo_exe% %beamtime_id%_detector diff --git a/tests/automatic/full_chain/simple_chain_filegen/CMakeLists.txt b/tests/automatic/full_chain/simple_chain_filegen/CMakeLists.txt index 878b5192b..9b2389617 100644 --- a/tests/automatic/full_chain/simple_chain_filegen/CMakeLists.txt +++ b/tests/automatic/full_chain/simple_chain_filegen/CMakeLists.txt @@ -3,13 +3,13 @@ set(TARGET_NAME full_chain_simple_chain_filegen_producer) ################################ # Testing ################################ -prepare_asapo() if (UNIX) set (ROOT_PATH "/tmp/asapo/") else() set (ROOT_PATH "c:\\\\tmp\\\\asapo\\\\") endif() +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) configure_file(test.json.in test.json @ONLY) -add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:event-monitor-producer-bin> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> tcp" nomem) +add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:event-monitor-producer-bin> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME>" nomem) diff --git a/tests/automatic/full_chain/simple_chain_filegen/check_linux.sh b/tests/automatic/full_chain/simple_chain_filegen/check_linux.sh index 0bd69ff98..bac120a9a 100644 --- a/tests/automatic/full_chain/simple_chain_filegen/check_linux.sh +++ b/tests/automatic/full_chain/simple_chain_filegen/check_linux.sh @@ -7,7 +7,6 @@ trap Cleanup EXIT producer_bin=$1 consumer_bin=$2 asapo_tool_bin=$3 -network_type=$4 beamtime_id=asapo_test @@ -27,25 +26,11 @@ Cleanup() { kill -9 $producerid rm -rf /tmp/asapo/test_in rm -rf ${receiver_folder} - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer echo "db.dropDatabase()" | mongo ${beamtime_id}_detector } echo "db.${beamtime_id}_detector.insert({dummy:1})" | mongo ${beamtime_id}_detector -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_${network_type}.nmd -nomad run discovery.nmd -nomad run broker.nmd - -sleep 1 - token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` echo "Start producer" diff --git a/tests/automatic/full_chain/simple_chain_filegen/check_windows.bat b/tests/automatic/full_chain/simple_chain_filegen/check_windows.bat index 9d783bd6a..105fb4e92 100644 --- a/tests/automatic/full_chain/simple_chain_filegen/check_windows.bat +++ b/tests/automatic/full_chain/simple_chain_filegen/check_windows.bat @@ -1,5 +1,3 @@ - - SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe" SET beamtime_id=asapo_test SET beamline=test @@ -12,8 +10,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id%_detector.insert({dummy:1}) | %mongo_exe% %beamtime_id%_detector -call start_services.bat - "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token set /P token=< token @@ -48,7 +44,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% rmdir /S /Q c:\tmp\asapo\test_in Taskkill /IM "%producer_short_name%" /F diff --git a/tests/automatic/full_chain/simple_chain_filegen_batches/CMakeLists.txt b/tests/automatic/full_chain/simple_chain_filegen_batches/CMakeLists.txt index 810f67311..efc3d525a 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_batches/CMakeLists.txt +++ b/tests/automatic/full_chain/simple_chain_filegen_batches/CMakeLists.txt @@ -3,7 +3,6 @@ set(TARGET_NAME full_chain_simple_chain_filegen_producer_batches) ################################ # Testing ################################ -prepare_asapo() if (UNIX) set (ROOT_PATH "/tmp/asapo/") else() @@ -11,5 +10,6 @@ else() endif() configure_file(test.json.in test.json @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) -add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:event-monitor-producer-bin> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> tcp" nomem) +add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:event-monitor-producer-bin> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME>" nomem) diff --git a/tests/automatic/full_chain/simple_chain_filegen_batches/check_linux.sh b/tests/automatic/full_chain/simple_chain_filegen_batches/check_linux.sh index 553a3b90c..bd9fae0ac 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_batches/check_linux.sh +++ b/tests/automatic/full_chain/simple_chain_filegen_batches/check_linux.sh @@ -7,7 +7,6 @@ trap Cleanup EXIT producer_bin=$1 consumer_bin=$2 asapo_tool_bin=$3 -network_type=$4 beamtime_id=asapo_test @@ -28,30 +27,14 @@ Cleanup() { kill -9 $producerid rm -rf /tmp/asapo/test_in rm -rf ${receiver_folder} - - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer echo "db.dropDatabase()" | mongo ${beamtime_id}_detector rm -rf out } echo "db.${beamtime_id}_detector.insert({dummy:1})" | mongo ${beamtime_id}_detector -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_tcp.nmd -nomad run discovery.nmd -nomad run broker.nmd - -sleep 1 - token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` - mkdir /tmp/asapo/test_in/processed/test1 mkdir /tmp/asapo/test_in/processed/test2 diff --git a/tests/automatic/full_chain/simple_chain_filegen_batches/check_windows.bat b/tests/automatic/full_chain/simple_chain_filegen_batches/check_windows.bat index bc1a1402a..9b9244d62 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_batches/check_windows.bat +++ b/tests/automatic/full_chain/simple_chain_filegen_batches/check_windows.bat @@ -1,5 +1,3 @@ - - SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe" SET beamtime_id=asapo_test SET beamline=test @@ -12,8 +10,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id%_detector.insert({dummy:1}) | %mongo_exe% %beamtime_id%_detector -call start_services.bat - "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token set /P token=< token @@ -47,7 +43,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% rmdir /S /Q c:\tmp\asapo\test_in Taskkill /IM "%producer_short_name%" /F diff --git a/tests/automatic/full_chain/simple_chain_filegen_multisource/CMakeLists.txt b/tests/automatic/full_chain/simple_chain_filegen_multisource/CMakeLists.txt index 0341d8b7a..5babe60b6 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_multisource/CMakeLists.txt +++ b/tests/automatic/full_chain/simple_chain_filegen_multisource/CMakeLists.txt @@ -3,7 +3,6 @@ set(TARGET_NAME full_chain_simple_chain_filegen_producer_multisource) ################################ # Testing ################################ -prepare_asapo() if (UNIX) set (ROOT_PATH "/tmp/asapo/") else() @@ -14,5 +13,6 @@ SET (ID 1) configure_file(test.json.in test1.json @ONLY) SET (ID 2) configure_file(test.json.in test2.json @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) add_script_test("${TARGET_NAME}" "$<TARGET_FILE:event-monitor-producer-bin> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME>" nomem) diff --git a/tests/automatic/full_chain/simple_chain_filegen_multisource/check_linux.sh b/tests/automatic/full_chain/simple_chain_filegen_multisource/check_linux.sh index 9619e32b3..e75e3b5bc 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_multisource/check_linux.sh +++ b/tests/automatic/full_chain/simple_chain_filegen_multisource/check_linux.sh @@ -27,26 +27,12 @@ Cleanup() { rm -rf /tmp/asapo/test_in1 rm -rf /tmp/asapo/test_in2 rm -rf ${receiver_folder} - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer echo "db.dropDatabase()" | mongo ${beamtime_id}_detector rm -rf out } echo "db.${beamtime_id}_detector.insert({dummy:1})" | mongo ${beamtime_id}_detector -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_tcp.nmd -nomad run discovery.nmd -nomad run broker.nmd - -sleep 1 - token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` mkdir -p /tmp/asapo/test_in1/processed diff --git a/tests/automatic/full_chain/simple_chain_filegen_multisource/check_windows.bat b/tests/automatic/full_chain/simple_chain_filegen_multisource/check_windows.bat index ef7d8294e..f54592667 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_multisource/check_windows.bat +++ b/tests/automatic/full_chain/simple_chain_filegen_multisource/check_windows.bat @@ -1,5 +1,3 @@ - - SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe" SET beamtime_id=asapo_test SET beamline=test @@ -12,8 +10,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id%_detector.insert({dummy:1}) | %mongo_exe% %beamtime_id%_detector -call start_services.bat - "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token set /P token=< token @@ -55,7 +51,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% rmdir /S /Q c:\tmp\asapo\test_in1 rmdir /S /Q c:\tmp\asapo\test_in2 diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/CMakeLists.txt b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/CMakeLists.txt index 17c1adc61..3da6e5000 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/CMakeLists.txt +++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/CMakeLists.txt @@ -3,7 +3,6 @@ set(TARGET_NAME full_chain_simple_chain_withdata_fromcache) ################################ # Testing ################################ -prepare_asapo() if (UNIX) set (ROOT_PATH "/tmp/asapo/") else() @@ -11,6 +10,10 @@ else() endif() configure_file(test.json.in test.json @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/receiver_tcp.json.tpl.lin.in receiver_tcp.json.tpl @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/receiver_fabric.json.tpl.lin.in receiver_fabric.json.tpl @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) + add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:event-monitor-producer-bin> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> tcp" nomem) diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/check_linux.sh b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/check_linux.sh index 0e12523a6..9cd94ca9d 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/check_linux.sh +++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/check_linux.sh @@ -25,29 +25,26 @@ mkdir -p /tmp/asapo/test_in/processed Cleanup() { echo cleanup + set +e + if [[ $network_type == "fabric" ]]; then + nomad stop receiver + nomad run receiver_tcp.nmd + fi kill -9 $producerid rm -rf /tmp/asapo/test_in rm -rf ${receiver_folder} influx -execute "drop database ${monitor_database_name}" - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer echo "db.dropDatabase()" | mongo ${beamtime_id}_detector rm out.txt } echo "db.${beamtime_id}_detector.insert({dummy:1})" | mongo ${beamtime_id}_detector -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_${network_type}.nmd -nomad run discovery.nmd -nomad run broker.nmd -sleep 1 +if [[ $network_type == "fabric" ]]; then + nomad stop receiver + nomad run receiver_fabric.nmd +fi token=`$3 token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` @@ -71,7 +68,3 @@ grep "hello1" out.txt grep "hello2" out.txt grep "hello3" out.txt grep -i "Using connection type: $network_type" out.txt - -sleep 12 - -influx -execute "select sum(n_requests) from statistics where receiver_ds_tag !=''" -database=${monitor_database_name} -format=json | jq .results[0].series[0].values[0][1] | tee /dev/stderr | grep 3 diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/check_windows.bat b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/check_windows.bat index dfde693b8..0a54f44ba 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/check_windows.bat +++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_cache/check_windows.bat @@ -1,5 +1,3 @@ - - SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe" SET beamtime_id=asapo_test SET beamline=test @@ -12,8 +10,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id%_detector.insert({dummy:1}) | %mongo_exe% %beamtime_id%_detector -call start_services.bat - "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token set /P token=< token @@ -33,7 +29,6 @@ echo hello3 > c:\tmp\asapo\test_in\processed\test2\file2 ping 192.0.2.1 -n 1 -w 1000 > nul - REM consumer "%2" %proxy_address% %receiver_folder% %beamtime_id% 2 %token% 1000 0 > out.txt type out.txt @@ -50,7 +45,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% rmdir /S /Q c:\tmp\asapo\test_in Taskkill /IM "%producer_short_name%" /F diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/CMakeLists.txt b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/CMakeLists.txt index 4cc2403a4..ab7e4f092 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/CMakeLists.txt +++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/CMakeLists.txt @@ -4,7 +4,6 @@ set(TARGET_NAME full_chain_simple_chain_withdata_fromfile) # Testing ################################ set(RECEIVER_USE_CACHE false) -prepare_asapo() if (UNIX) set (ROOT_PATH "/tmp/asapo/") else() @@ -12,5 +11,6 @@ else() endif() configure_file(test.json.in test.json @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) -add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:event-monitor-producer-bin> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> tcp" nomem) +add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:event-monitor-producer-bin> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME>" nomem) diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/check_linux.sh b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/check_linux.sh index 748986058..59992c6e4 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/check_linux.sh +++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/check_linux.sh @@ -7,7 +7,6 @@ trap Cleanup EXIT producer_bin=$1 consumer_bin=$2 asapo_tool_bin=$3 -network_type=$4 beamtime_id=asapo_test @@ -29,11 +28,6 @@ Cleanup() { rm -rf /tmp/asapo/test_in rm -rf ${receiver_folder} influx -execute "drop database ${monitor_database_name}" - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker nomad stop authorizer echo "db.dropDatabase()" | mongo ${beamtime_id}_detector rm out.txt @@ -41,14 +35,6 @@ Cleanup() { echo "db.${beamtime_id}_detector.insert({dummy:1})" | mongo ${beamtime_id}_detector -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_${network_type}.nmd -nomad run discovery.nmd -nomad run broker.nmd - -sleep 1 - token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` diff --git a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/check_windows.bat b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/check_windows.bat index 63a5f3521..4ce03a1a7 100644 --- a/tests/automatic/full_chain/simple_chain_filegen_readdata_file/check_windows.bat +++ b/tests/automatic/full_chain/simple_chain_filegen_readdata_file/check_windows.bat @@ -12,8 +12,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id%_detector.insert({dummy:1}) | %mongo_exe% %beamtime_id%_detector -call start_services.bat - "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token set /P token=< token @@ -52,7 +50,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% rmdir /S /Q c:\tmp\asapo\test_in Taskkill /IM "%producer_short_name%" /F diff --git a/tests/automatic/full_chain/simple_chain_metadata/CMakeLists.txt b/tests/automatic/full_chain/simple_chain_metadata/CMakeLists.txt index 7149bd170..d33f53c2d 100644 --- a/tests/automatic/full_chain/simple_chain_metadata/CMakeLists.txt +++ b/tests/automatic/full_chain/simple_chain_metadata/CMakeLists.txt @@ -3,5 +3,5 @@ set(TARGET_NAME full_chain_simple_chain_metadata) ################################ # Testing ################################ -prepare_asapo() -add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> tcp" nomem) +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) +add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME>" nomem) diff --git a/tests/automatic/full_chain/simple_chain_metadata/check_linux.sh b/tests/automatic/full_chain/simple_chain_metadata/check_linux.sh index 70fc2dfe2..5142eeae8 100644 --- a/tests/automatic/full_chain/simple_chain_metadata/check_linux.sh +++ b/tests/automatic/full_chain/simple_chain_metadata/check_linux.sh @@ -23,12 +23,6 @@ receiver_folder=${receiver_root_folder}/${facility}/gpfs/${beamline}/${year}/dat Cleanup() { echo cleanup rm -rf ${receiver_root_folder} - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer rm -rf out echo "db.dropDatabase()" | mongo ${beamtime_id}_detector influx -execute "drop database ${monitor_database_name}" @@ -36,14 +30,6 @@ Cleanup() { echo "db.${beamtime_id}_detector.insert({dummy:1})" | mongo ${beamtime_id}_detector -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_tcp.nmd # Only use TCP because the consumer will only use metadata anyways -nomad run discovery.nmd -nomad run broker.nmd - -sleep 1 - token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` diff --git a/tests/automatic/full_chain/simple_chain_metadata/check_windows.bat b/tests/automatic/full_chain/simple_chain_metadata/check_windows.bat index f5121e9f4..3551790bb 100644 --- a/tests/automatic/full_chain/simple_chain_metadata/check_windows.bat +++ b/tests/automatic/full_chain/simple_chain_metadata/check_windows.bat @@ -8,8 +8,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id%_detector.insert({dummy:1}) | %mongo_exe% %beamtime_id%_detector -call start_services.bat - "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token set /P token=< token @@ -30,7 +28,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% del /f token echo db.dropDatabase() | %mongo_exe% %beamtime_id%_detector diff --git a/tests/automatic/full_chain/simple_chain_raw/CMakeLists.txt b/tests/automatic/full_chain/simple_chain_raw/CMakeLists.txt index 3bbd3e039..d054c37c2 100644 --- a/tests/automatic/full_chain/simple_chain_raw/CMakeLists.txt +++ b/tests/automatic/full_chain/simple_chain_raw/CMakeLists.txt @@ -3,21 +3,8 @@ set(TARGET_NAME full_chain_simple_chain_raw) ################################ # Testing ################################ -prepare_asapo() - -file(TO_NATIVE_PATH ${CMAKE_CURRENT_BINARY_DIR}/asap3 ASAP3_FOLDER ) -file(TO_NATIVE_PATH ${CMAKE_CURRENT_BINARY_DIR}/beamline CURRENT_BEAMLINES_FOLDER ) - -file(TO_NATIVE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/ip_bl_mapping BEAMLINES_FOLDER ) -if (WIN32) - string(REPLACE "\\" "\\\\" ASAP3_FOLDER "${ASAP3_FOLDER}") - string(REPLACE "\\" "\\\\" BEAMLINES_FOLDER "${BEAMLINES_FOLDER}") - string(REPLACE "\\" "\\\\" CURRENT_BEAMLINES_FOLDER "${CURRENT_BEAMLINES_FOLDER}") - -endif() configure_file(beamtime-metadata-11111111.json beamtime-metadata-11111111.json COPYONLY) - -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/settings.json.tpl.in authorizer.json.tpl @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) add_script_test("${TARGET_NAME}" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME>" nomem) diff --git a/tests/automatic/full_chain/simple_chain_raw/check_linux.sh b/tests/automatic/full_chain/simple_chain_raw/check_linux.sh index 176da746d..109485d11 100644 --- a/tests/automatic/full_chain/simple_chain_raw/check_linux.sh +++ b/tests/automatic/full_chain/simple_chain_raw/check_linux.sh @@ -9,43 +9,27 @@ beamtime_id=11111111 monitor_database_name=db_test proxy_address=127.0.0.1:8400 -mkdir -p asap3/petra3/gpfs/p07/2019/data/11111111 -cp beamtime-metadata* asap3/petra3/gpfs/p07/2019/data/11111111 -mkdir -p beamline/p07/current -cp beamtime-metadata* beamline/p07/current/ + +mkdir -p /tmp/asapo/asap3/petra3/gpfs/p07/2019/data/11111111 +cp beamtime-metadata* /tmp/asapo/asap3/petra3/gpfs/p07/2019/data/11111111 +mkdir -p /tmp/asapo/beamline/p07/current +cp beamtime-metadata* /tmp/asapo/beamline/p07/current/ Cleanup() { echo cleanup - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer - rm -rf out asap3 beamline + rm -rf out /tmp/asapo/asap3 /tmp/asapo/beamline echo "db.dropDatabase()" | mongo ${beamtime_id}_detector influx -execute "drop database ${monitor_database_name}" } -#echo "db.data_${beamtime_id}_detector.insert({dummy:1})" | mongo data_${beamtime_id}_detector echo "db.dropDatabase()" | mongo ${beamtime_id}_detector - -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_tcp.nmd -nomad run discovery.nmd -nomad run broker.nmd - -sleep 1 - token=`$3 token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` #producer $1 localhost:8400 ${beamtime_id} 100 10 4 100 100 - -$2 ${proxy_address} "__" ${beamtime_id} 2 $token 5000 1 > out +$2 ${proxy_address} "__" ${beamtime_id} 2 $token 2000 1 > out cat out cat out | grep "Processed 10 file(s)" -ls beamline/p07/current/raw/1 | tee /dev/stderr | grep 1 +ls /tmp/asapo/beamline/p07/current/raw/1 | tee /dev/stderr | grep 1 diff --git a/tests/automatic/full_chain/simple_chain_raw/check_windows.bat b/tests/automatic/full_chain/simple_chain_raw/check_windows.bat index c60ca7696..2a62d959a 100644 --- a/tests/automatic/full_chain/simple_chain_raw/check_windows.bat +++ b/tests/automatic/full_chain/simple_chain_raw/check_windows.bat @@ -1,20 +1,16 @@ SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe" SET beamtime_id=11111111 -mkdir asap3\petra3\gpfs\p07\2019\data\11111111 -mkdir beamline\p07\current -copy beamtime-metadata* beamline\p07\current\ /y -copy beamtime-metadata* asap3\petra3\gpfs\p07\2019\data\11111111\ /y +mkdir c:\tmp\asapo\asap3\petra3\gpfs\p07\2019\data\11111111 +mkdir c:\tmp\asapo\beamline\p07\current +copy beamtime-metadata* c:\tmp\asapo\beamline\p07\current\ /y +copy beamtime-metadata* c:\tmp\asapo\asap3\petra3\gpfs\p07\2019\data\11111111\ /y set proxy_address="127.0.0.1:8400" -call start_services.bat - "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token - set /P token=< token - REM producer mkdir %receiver_folder% start /B "" "%1" %proxy_address% %beamtime_id% 100 10 4 100 100 @@ -24,7 +20,7 @@ REM consumer "%2" %proxy_address% "_" %beamtime_id% 2 %token% 5000 1 > out.txt type out.txt findstr /i /l /c:"Processed 10 file(s)" out.txt || goto :error -if not exist beamline\p07\current\raw\1 goto :error +if not exist c:\tmp\asapo\beamline\p07\current\raw\1 goto :error goto :clean @@ -33,9 +29,8 @@ call :clean exit /b 1 :clean -call stop_services.bat -rmdir /S /Q asap3 -rmdir /S /Q beamline +rmdir /S /Q c:\tmp\asapo\asap3 +rmdir /S /Q c:\tmp\asapo\beamline del /f token del /f out.txt echo db.dropDatabase() | %mongo_exe% %beamtime_id%_detector diff --git a/tests/automatic/full_chain/simple_chain_raw/settings.json.tpl.in b/tests/automatic/full_chain/simple_chain_raw/settings.json.tpl.in deleted file mode 100644 index 130d17e47..000000000 --- a/tests/automatic/full_chain/simple_chain_raw/settings.json.tpl.in +++ /dev/null @@ -1,14 +0,0 @@ -{ - "Port": {{ env "NOMAD_PORT_authorizer" }}, - "LogLevel":"debug", - "RootBeamtimesFolder":"@ASAP3_FOLDER@", - "CurrentBeamlinesFolder":"@CURRENT_BEAMLINES_FOLDER@", - "UserSecretFile":"auth_secret.key", - "AdminSecretFile":"auth_secret_admin.key", - "Ldap": - { - "Uri" : "ldap://localhost:389", - "BaseDn" : "ou=rgy,o=desy,c=de", - "FilterTemplate" : "(cn=a3__BEAMLINE__-hosts)" - } -} \ No newline at end of file diff --git a/tests/automatic/full_chain/simple_chain_usermeta_python/CMakeLists.txt b/tests/automatic/full_chain/simple_chain_usermeta_python/CMakeLists.txt index 827a81a9c..719adce9e 100644 --- a/tests/automatic/full_chain/simple_chain_usermeta_python/CMakeLists.txt +++ b/tests/automatic/full_chain/simple_chain_usermeta_python/CMakeLists.txt @@ -3,7 +3,6 @@ set(TARGET_NAME full_chain_usermeta_python) ################################ # Testing ################################ -prepare_asapo() if (UNIX) get_target_property(PYTHON_LIBS python-lib-consumer BINARY_DIR) @@ -11,5 +10,5 @@ else() get_target_property(PYTHON_LIBS asapo_consumer BINARY_DIR) endif() - +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) add_script_test("${TARGET_NAME}" "$<TARGET_FILE:dummy-data-producer> $<TARGET_PROPERTY:asapo,EXENAME> ${CMAKE_CURRENT_SOURCE_DIR} ${PYTHON_LIBS} ${Python_EXECUTABLE}" nomem) diff --git a/tests/automatic/full_chain/simple_chain_usermeta_python/check_linux.sh b/tests/automatic/full_chain/simple_chain_usermeta_python/check_linux.sh index 129b8298c..0a5e90992 100644 --- a/tests/automatic/full_chain/simple_chain_usermeta_python/check_linux.sh +++ b/tests/automatic/full_chain/simple_chain_usermeta_python/check_linux.sh @@ -22,12 +22,6 @@ receiver_folder=${receiver_root_folder}/${facility}/gpfs/${beamline}/${year}/dat Cleanup() { echo cleanup rm -rf ${receiver_root_folder} - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer rm -rf out echo "db.dropDatabase()" | mongo ${beamtime_id}_detector influx -execute "drop database ${monitor_database_name}" @@ -35,14 +29,6 @@ Cleanup() { echo "db.${beamtime_id}_detector.insert({dummy:1})" | mongo ${beamtime_id}_detector -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_tcp.nmd # Only use TCP because the consumer will only use metadata anyways -nomad run discovery.nmd -nomad run broker.nmd - -sleep 2 - token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` diff --git a/tests/automatic/full_chain/simple_chain_usermeta_python/check_windows.bat b/tests/automatic/full_chain/simple_chain_usermeta_python/check_windows.bat index 86d039d55..338aa65ee 100644 --- a/tests/automatic/full_chain/simple_chain_usermeta_python/check_windows.bat +++ b/tests/automatic/full_chain/simple_chain_usermeta_python/check_windows.bat @@ -8,8 +8,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id%_detector.insert({dummy:1}) | %mongo_exe% %beamtime_id%_detector -call start_services.bat - "%2" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token set /P token=< token @@ -34,7 +32,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% del /f token echo db.dropDatabase() | %mongo_exe% %beamtime_id%_detector diff --git a/tests/automatic/full_chain/two_beamlines/CMakeLists.txt b/tests/automatic/full_chain/two_beamlines/CMakeLists.txt index b4b621b3d..6337cc472 100644 --- a/tests/automatic/full_chain/two_beamlines/CMakeLists.txt +++ b/tests/automatic/full_chain/two_beamlines/CMakeLists.txt @@ -3,11 +3,13 @@ set(TARGET_NAME full_chain_two_beamlines) ################################ # Testing ################################ -prepare_asapo() add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> tcp" nomem) +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/receiver_tcp.json.tpl.lin.in receiver_tcp.json.tpl @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) if (ENABLE_LIBFABRIC) if (ENABLE_LIBFABRIC_LOCALHOST) + configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/receiver_fabric.json.tpl.lin.in receiver_fabric.json.tpl @ONLY) add_script_test("${TARGET_NAME}-fabric" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> fabric" nomem) else () message(WARNING "Disabled automated LibFabric of '${TARGET_NAME}-fabric' test because 'ENABLE_LIBFABRIC_LOCALHOST' is not enabled.") diff --git a/tests/automatic/full_chain/two_beamlines/check_linux.sh b/tests/automatic/full_chain/two_beamlines/check_linux.sh index f43ddad7b..ee0fc635e 100644 --- a/tests/automatic/full_chain/two_beamlines/check_linux.sh +++ b/tests/automatic/full_chain/two_beamlines/check_linux.sh @@ -26,18 +26,14 @@ year=2019 receiver_folder1=${receiver_root_folder}/${facility}/gpfs/${beamline1}/${year}/data/${beamtime_id1} receiver_folder2=${receiver_root_folder}/${facility}/gpfs/${beamline2}/${year}/data/${beamtime_id2} - - Cleanup() { echo cleanup + set +e + if [[ $network_type == "fabric" ]]; then + nomad stop receiver + nomad run receiver_tcp.nmd + fi rm -rf ${receiver_root_folder} - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer -# kill $producerid echo "db.dropDatabase()" | mongo ${beamtime_id1}_${data_source} echo "db.dropDatabase()" | mongo ${beamtime_id2}_${data_source} influx -execute "drop database ${monitor_database_name}" @@ -46,13 +42,10 @@ Cleanup() { echo "db.${beamtime_id1}_${data_source}.insert({dummy:1})" | mongo ${beamtime_id1}_${data_source} echo "db.${beamtime_id2}_${data_source}.insert({dummy:1})" | mongo ${beamtime_id2}_${data_source} -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_${network_type}.nmd -nomad run discovery.nmd -nomad run broker.nmd - -sleep 3 +if [[ $network_type == "fabric" ]]; then + nomad stop receiver + nomad run receiver_fabric.nmd +fi token1=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id1` token2=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id2` diff --git a/tests/automatic/full_chain/two_beamlines/check_windows.bat b/tests/automatic/full_chain/two_beamlines/check_windows.bat index 1d7390636..968f7e4fe 100644 --- a/tests/automatic/full_chain/two_beamlines/check_windows.bat +++ b/tests/automatic/full_chain/two_beamlines/check_windows.bat @@ -18,8 +18,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id1%_%data_source%.insert({dummy:1}) | %mongo_exe% %beamtime_id1%_%data_source% echo db.%beamtime_id2%_%data_source%.insert({dummy:1}) | %mongo_exe% %beamtime_id2%_%data_source% -call start_services.bat - "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id1% > token set /P token1=< token "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id2% > token @@ -52,7 +50,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% del /f token1 del /f token2 diff --git a/tests/automatic/full_chain/two_streams/CMakeLists.txt b/tests/automatic/full_chain/two_streams/CMakeLists.txt index bde0f7e39..85eb3a5e9 100644 --- a/tests/automatic/full_chain/two_streams/CMakeLists.txt +++ b/tests/automatic/full_chain/two_streams/CMakeLists.txt @@ -3,5 +3,5 @@ set(TARGET_NAME full_chain_two_streams) ################################ # Testing ################################ -prepare_asapo() +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> tcp" nomem) diff --git a/tests/automatic/full_chain/two_streams/check_linux.sh b/tests/automatic/full_chain/two_streams/check_linux.sh index 835de3037..0021c1798 100644 --- a/tests/automatic/full_chain/two_streams/check_linux.sh +++ b/tests/automatic/full_chain/two_streams/check_linux.sh @@ -27,12 +27,6 @@ receiver_folder=${receiver_root_folder}/${facility}/gpfs/${beamline}/${year}/dat Cleanup() { echo cleanup rm -rf ${receiver_root_folder} - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer echo "db.dropDatabase()" | mongo ${beamtime_id}_${stream1} echo "db.dropDatabase()" | mongo ${beamtime_id}_${stream2} } @@ -40,14 +34,6 @@ Cleanup() { echo "db.${beamtime_id}_${stream1}.insert({dummy:1})" | mongo ${beamtime_id}_${stream1} echo "db.${beamtime_id}_${stream2}.insert({dummy:1})" | mongo ${beamtime_id}_${stream2} -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_tcp.nmd -nomad run discovery.nmd -nomad run broker.nmd - -sleep 3 - token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` echo "Start producers" diff --git a/tests/automatic/full_chain/two_streams/check_windows.bat b/tests/automatic/full_chain/two_streams/check_windows.bat index 5e3b68c2c..ee45bc72f 100644 --- a/tests/automatic/full_chain/two_streams/check_windows.bat +++ b/tests/automatic/full_chain/two_streams/check_windows.bat @@ -13,8 +13,6 @@ set proxy_address="127.0.0.1:8400" echo db.%beamtime_id%_%stream1%.insert({dummy:1}) | %mongo_exe% %beamtime_id%_%stream1% echo db.%beamtime_id%_%stream2%.insert({dummy:1}) | %mongo_exe% %beamtime_id%_%stream2% -call start_services.bat - "%3" token -endpoint http://127.0.0.1:8400/asapo-authorizer -secret admin_token.key -types read %beamtime_id% > token set /P token=< token @@ -40,7 +38,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% del /f token1 del /f token2 diff --git a/tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt b/tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt index 9c50d1887..22840923a 100644 --- a/tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt +++ b/tests/automatic/high_avail/broker_mongo_restart/CMakeLists.txt @@ -3,5 +3,5 @@ set(TARGET_NAME broker_mongo_restart) ################################ # Testing ################################ -prepare_asapo() +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) add_script_test("${TARGET_NAME}-tcp" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> tcp" nomem) diff --git a/tests/automatic/high_avail/broker_mongo_restart/check_linux.sh b/tests/automatic/high_avail/broker_mongo_restart/check_linux.sh index 89a8247cb..302423c24 100755 --- a/tests/automatic/high_avail/broker_mongo_restart/check_linux.sh +++ b/tests/automatic/high_avail/broker_mongo_restart/check_linux.sh @@ -49,12 +49,6 @@ Cleanup() { set +e echo cleanup rm -rf ${receiver_root_folder} - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer kill $producerid kill $workerid echo "db.dropDatabase()" | mongo --port 27015 ${beamtime_id}_detector || echo "db.dropDatabase()" | mongo --port 27016 ${beamtime_id}_detector @@ -71,15 +65,6 @@ sed -i 's/info/debug/g' broker.json.tpl start_mongo 27016 wait_mongo 27016 - -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_tcp.nmd -nomad run discovery.nmd -nomad run broker.nmd - -sleep 1 - token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` diff --git a/tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt b/tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt index bebdc30dc..0c7eba429 100644 --- a/tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt +++ b/tests/automatic/high_avail/receiver_mongo_restart/CMakeLists.txt @@ -3,5 +3,5 @@ set(TARGET_NAME receiver-mongo-restart) ################################ # Testing ################################ -prepare_asapo() +configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/admin_token.key admin_token.key COPYONLY) add_script_test("${TARGET_NAME}" "$<TARGET_FILE:dummy-data-producer>" nomem) diff --git a/tests/automatic/high_avail/receiver_mongo_restart/check_linux.sh b/tests/automatic/high_avail/receiver_mongo_restart/check_linux.sh index 11199a46a..87734e95d 100644 --- a/tests/automatic/high_avail/receiver_mongo_restart/check_linux.sh +++ b/tests/automatic/high_avail/receiver_mongo_restart/check_linux.sh @@ -41,14 +41,10 @@ receiver_folder=${receiver_root_folder}/${facility}/gpfs/${beamline}/${year}/dat Cleanup() { echo cleanup + set +e rm -rf ${receiver_root_folder} - nomad stop receiver - nomad stop discovery - nomad stop authorizer - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - echo "db.dropDatabase()" | mongo --port 27016 ${beamtime_id}_detector - kill_mongo + echo "db.dropDatabase()" | mongo --port 27016 ${beamtime_id}_detector + kill_mongo } start_mongo @@ -60,17 +56,8 @@ echo "db.${beamtime_id}_detector.insert({dummy:1})" | mongo --port 27016 ${beamt sed -i 's/27017/27016/g' discovery.json.tpl - -nomad run authorizer.nmd -nomad run nginx.nmd -nomad run receiver_tcp.nmd -nomad run discovery.nmd - mkdir -p ${receiver_folder} - -sleep 1 - nfiles=1000 $1 localhost:8400 ${beamtime_id} 100 $nfiles 1 0 200 & diff --git a/tests/automatic/high_avail/services_restart/CMakeLists.txt b/tests/automatic/high_avail/services_restart/CMakeLists.txt index 4b27ebf5b..cb554fba0 100644 --- a/tests/automatic/high_avail/services_restart/CMakeLists.txt +++ b/tests/automatic/high_avail/services_restart/CMakeLists.txt @@ -4,5 +4,5 @@ set(TARGET_NAME service_restart) # Testing ################################ prepare_asapo() -add_script_test("${TARGET_NAME}-all-tcp" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> broker 1000 998 tcp" nomem) -add_script_test("${TARGET_NAME}-all-but-broker-tcp" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> receiver 1000 1000 tcp" nomem) +add_script_test("${TARGET_NAME}-all" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> broker 1000 998" nomem) +add_script_test("${TARGET_NAME}-all-but-broker" "$<TARGET_FILE:dummy-data-producer> $<TARGET_FILE:getnext> $<TARGET_PROPERTY:asapo,EXENAME> receiver 1000 1000" nomem) diff --git a/tests/automatic/high_avail/services_restart/check_linux.sh b/tests/automatic/high_avail/services_restart/check_linux.sh index a9e011210..d8324f4da 100644 --- a/tests/automatic/high_avail/services_restart/check_linux.sh +++ b/tests/automatic/high_avail/services_restart/check_linux.sh @@ -7,7 +7,7 @@ trap Cleanup EXIT producer_bin=$1 consumer_bin=$2 asapo_tool_bin=$3 -network_type=$7 + beamtime_id=asapo_test @@ -24,25 +24,10 @@ mkdir -p ${receiver_folder} Cleanup() { echo cleanup rm -rf ${receiver_folder} - nomad stop nginx - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill - nomad stop receiver - nomad stop discovery - nomad stop broker - nomad stop authorizer echo "db.dropDatabase()" | mongo ${beamtime_id}_detector influx -execute "drop database ${monitor_database_name}" } -#sed -i 's/info/debug/g' broker.json.tpl - -nomad run nginx.nmd -nomad run authorizer.nmd -nomad run receiver_tcp.nmd -nomad run discovery.nmd -nomad run broker.nmd - -sleep 1 token=`$asapo_tool_bin token -endpoint http://localhost:8400/asapo-authorizer -secret admin_token.key -types read $beamtime_id` @@ -52,7 +37,7 @@ echo "Start producer" $producer_bin localhost:8400 ${beamtime_id} 100 $5 4 0 100 & #producerid=`echo $!` -echo "Start consumer in $network_type mode" +echo "Start consumer" $consumer_bin ${proxy_address} dummy_path ${beamtime_id} 2 $token 30000 1 &> output.txt & sleep 1 @@ -67,10 +52,10 @@ nomad stop receiver nomad run nginx.nmd nomad run authorizer.nmd nomad run discovery.nmd -nomad run receiver_$7.nmd +nomad run receiver_tcp.nmd if [[ "$4" == "receiver" ]]; then - nomad run $4_$7.nmd + nomad run $4_tcp.nmd else nomad run $4.nmd fi diff --git a/tests/automatic/producer/aai/CMakeLists.txt b/tests/automatic/producer/aai/CMakeLists.txt index 2330026be..27749acb8 100644 --- a/tests/automatic/producer/aai/CMakeLists.txt +++ b/tests/automatic/producer/aai/CMakeLists.txt @@ -3,9 +3,6 @@ set(TARGET_NAME producer_python_aai) ################################ # Testing ################################ -prepare_asapo() - -prepare_asapo() if (UNIX) get_target_property(PYTHON_LIBS python-lib-producer BINARY_DIR) @@ -13,14 +10,7 @@ else() get_target_property(PYTHON_LIBS asapo_producer BINARY_DIR) endif() -file(TO_NATIVE_PATH ${CMAKE_CURRENT_BINARY_DIR}/beamline CURRENT_BEAMLINES_FOLDER ) - -if (WIN32) - string(REPLACE "\\" "\\\\" CURRENT_BEAMLINES_FOLDER "${CURRENT_BEAMLINES_FOLDER}") -endif() - -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/settings.json.tpl.in authorizer.json.tpl @ONLY) -configure_file(beamtime-metadata-11111111.json beamline/p07/current/beamtime-metadata-11111111.json COPYONLY) +configure_file(beamtime-metadata-11111111.json beamtime-metadata-11111111.json) file(TO_NATIVE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/producer_aai.py TEST_SCRIPT ) diff --git a/tests/automatic/producer/aai/check_linux.sh b/tests/automatic/producer/aai/check_linux.sh index d5aeba38c..e1da7ce2e 100644 --- a/tests/automatic/producer/aai/check_linux.sh +++ b/tests/automatic/producer/aai/check_linux.sh @@ -18,32 +18,26 @@ token=$BLP07_W_TOKEN Cleanup() { echo cleanup rm -rf ${receiver_root_folder} - nomad stop receiver >/dev/null - nomad stop discovery >/dev/null - nomad stop authorizer >/dev/null - nomad stop nginx >/dev/null - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill > /dev/null - echo "db.dropDatabase()" | mongo ${beamtime_id}_${data_source} >/dev/null - echo "db.dropDatabase()" | mongo ${beamtime_id2}_${data_source} >/dev/null + echo "db.dropDatabase()" | mongo ${beamtime_id}_${data_source} >/dev/null + echo "db.dropDatabase()" | mongo ${beamtime_id2}_${data_source} >/dev/null + rm -rf $beamline_dir } export PYTHONPATH=$2:${PYTHONPATH} -nomad run authorizer.nmd >/dev/null -nomad run nginx.nmd >/dev/null -nomad run receiver_tcp.nmd >/dev/null -nomad run discovery.nmd >/dev/null - mkdir -p ${receiver_folder} ${receiver_folder2} sleep 1 echo test > file1 +beamline_dir='/tmp/asapo/beamline/p07/current/' +mkdir -p $beamline_dir +cp beamtime-metadata-11111111.json $beamline_dir -$1 $3 $beamline $token $data_source "127.0.0.1:8400" > out || cat out +$1 $3 $beamline $token $data_source "127.0.0.1:8400" ${beamline_dir}"beamtime-metadata-11111111.json" > out || cat out cat out -cat out | grep "successfuly sent" | wc -l | grep 3 -cat out | grep "reauthorization\|Broken" | wc -l | grep 1 -cat out | grep "duplicated" | wc -l | grep 2 +cat out | grep "successfuly sent" | wc -l | tee /dev/stderr | grep 3 +cat out | grep "reauthorization\|Broken" | wc -l | tee /dev/stderr | grep 1 +cat out | grep "duplicated" | wc -l | tee /dev/stderr | grep 2 diff --git a/tests/automatic/producer/aai/check_windows.bat b/tests/automatic/producer/aai/check_windows.bat index fa0c3b902..5a5741ddf 100644 --- a/tests/automatic/producer/aai/check_windows.bat +++ b/tests/automatic/producer/aai/check_windows.bat @@ -9,11 +9,11 @@ SET receiver_folder2="%receiver_root_folder%\test_facility\gpfs\%beamline%\2019\ SET dbname=%beamtime_id%_%data_source% SET dbname2=%beamtime_id2%_%data_source% SET token=%BLP07_W_TOKEN% +SET beamline_dir=/c:\tmp\asapo\beamline\p07\current\ -echo db.%dbname%.insert({dummy:1})" | %mongo_exe% %dbname% -call start_services.bat +echo db.%dbname%.insert({dummy:1})" | %mongo_exe% %dbname% mkdir %receiver_folder% mkdir %receiver_folder2% @@ -24,7 +24,10 @@ ping 192.0.2.1 -n 1 -w 1000 > nul set PYTHONPATH=%2 -"%1" "%3" %beamline% %token% %data_source% "127.0.0.1:8400" > out +mkdir %beamline_dir% +copy beamtime-metadata-11111111.json %beamline_dir% /y + +"%1" "%3" %beamline% %token% %data_source% "127.0.0.1:8400"%beamline_dir%beamtime-metadata-11111111.json > out type out set NUM=0 @@ -44,9 +47,9 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% rmdir /S /Q %receiver_root_folder2% +rmdir /S /Q %beamline_dir% echo db.dropDatabase() | %mongo_exe% %dbname% echo db.dropDatabase() | %mongo_exe% %dbname2% diff --git a/tests/automatic/producer/aai/producer_aai.py b/tests/automatic/producer/aai/producer_aai.py index 8828b87ab..da575d7c7 100644 --- a/tests/automatic/producer/aai/producer_aai.py +++ b/tests/automatic/producer/aai/producer_aai.py @@ -12,6 +12,7 @@ beamline = sys.argv[1] token = sys.argv[2] data_source = sys.argv[3] endpoint = sys.argv[4] +fname = sys.argv[5] nthreads = 1 @@ -42,7 +43,6 @@ producer.send_file(1, local_path = "./file1", exposed_path = "processed/"+data_s producer.wait_requests_finished(10000) -fname = 'beamline/p07/current/beamtime-metadata-11111111.json' with open(fname) as json_file: data = json.load(json_file) data['beamtimeId']='22222222' diff --git a/tests/automatic/producer/aai/settings.json.tpl.in b/tests/automatic/producer/aai/settings.json.tpl.in deleted file mode 100644 index a98ad02c7..000000000 --- a/tests/automatic/producer/aai/settings.json.tpl.in +++ /dev/null @@ -1,9 +0,0 @@ -{ - "Port": {{ env "NOMAD_PORT_authorizer" }}, - "LogLevel":"debug", - "CurrentBeamlinesFolder":"@CURRENT_BEAMLINES_FOLDER@", - "UserSecretFile":"auth_secret.key", - "AdminSecretFile":"auth_secret_admin.key" -} - - diff --git a/tests/automatic/producer/python_api/CMakeLists.txt b/tests/automatic/producer/python_api/CMakeLists.txt index 8e8282c91..cec3071ed 100644 --- a/tests/automatic/producer/python_api/CMakeLists.txt +++ b/tests/automatic/producer/python_api/CMakeLists.txt @@ -3,10 +3,6 @@ set(TARGET_NAME producer_python_api) ################################ # Testing ################################ -prepare_asapo() - -prepare_asapo() - if (UNIX) get_target_property(PYTHON_LIBS python-lib-producer BINARY_DIR) else() diff --git a/tests/automatic/producer/python_api/check_linux.sh b/tests/automatic/producer/python_api/check_linux.sh index 957e6ff0d..4d677c18a 100644 --- a/tests/automatic/producer/python_api/check_linux.sh +++ b/tests/automatic/producer/python_api/check_linux.sh @@ -16,11 +16,6 @@ receiver_folder=${receiver_root_folder}/${facility}/gpfs/${beamline}/${year}/dat Cleanup() { echo cleanup rm -rf ${receiver_root_folder} - nomad stop receiver >/dev/null - nomad stop discovery >/dev/null - nomad stop authorizer >/dev/null - nomad stop nginx >/dev/null - nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill > /dev/null echo "db.dropDatabase()" | mongo ${beamtime_id}_${data_source} >/dev/null } @@ -28,17 +23,10 @@ export PYTHONPATH=$2:${PYTHONPATH} echo "db.${beamtime_id}_${data_source}.insert({dummy:1})" | mongo ${beamtime_id}_${data_source} >/dev/null -nomad run authorizer.nmd >/dev/null -nomad run nginx.nmd >/dev/null -nomad run receiver_tcp.nmd >/dev/null -nomad run discovery.nmd >/dev/null - mkdir -p ${receiver_folder} echo test > file1 -sleep 10 - $1 $3 $data_source $beamtime_id "127.0.0.1:8400" &> out || cat out cat out echo count successfully send, expect 15 diff --git a/tests/automatic/producer/python_api/check_windows.bat b/tests/automatic/producer/python_api/check_windows.bat index 844e51b37..8ae2e3382 100644 --- a/tests/automatic/producer/python_api/check_windows.bat +++ b/tests/automatic/producer/python_api/check_windows.bat @@ -8,8 +8,6 @@ SET dbname=%beamtime_id%_%data_source% echo db.%dbname%.insert({dummy:1})" | %mongo_exe% %dbname% -call start_services.bat - mkdir %receiver_folder% echo test > file1 @@ -43,7 +41,6 @@ call :clean exit /b 1 :clean -call stop_services.bat rmdir /S /Q %receiver_root_folder% echo db.dropDatabase() | %mongo_exe% %dbname% -- GitLab