diff --git a/producer/event_monitor_producer/src/main_eventmon.cpp b/producer/event_monitor_producer/src/main_eventmon.cpp index 1be2be99e214b411eedc4a6921db84f213ec93f0..18d23aa95bc38c0a6935f3e3f55804a10c36179e 100644 --- a/producer/event_monitor_producer/src/main_eventmon.cpp +++ b/producer/event_monitor_producer/src/main_eventmon.cpp @@ -113,7 +113,7 @@ int main (int argc, char* argv[]) { } continue; } - event_header.file_id = i++; + event_header.file_id = ++i; producer->SendFile(event_header, GetEventMonConfig()->root_monitored_folder + asapo::kPathSeparator + event_header.file_name, ProcessAfterSend); } diff --git a/producer/event_monitor_producer/src/single_folder_watch_windows.cpp b/producer/event_monitor_producer/src/single_folder_watch_windows.cpp index a0aa050c846906c83d8a097e9578729a0901ccd1..8f0010fab57301ec404deca9075403ed1cd946de 100644 --- a/producer/event_monitor_producer/src/single_folder_watch_windows.cpp +++ b/producer/event_monitor_producer/src/single_folder_watch_windows.cpp @@ -13,8 +13,7 @@ SingleFolderWatch::SingleFolderWatch(std::string root_folder, std::string folder root_folder_{std::move(root_folder)}, folder_{std::move(folder)}, buffer_{new char[kBufLen]}, -event_list_{event_list} -{ +event_list_{event_list} { } Error SingleFolderWatch::Init() { diff --git a/tests/automatic/full_chain/CMakeLists.txt b/tests/automatic/full_chain/CMakeLists.txt index 49d58db5778bfacf1c47dc634d3dcecee55b89aa..dc2c2f74469c9936efc323b31598339cc80ae09b 100644 --- a/tests/automatic/full_chain/CMakeLists.txt +++ b/tests/automatic/full_chain/CMakeLists.txt @@ -1,2 +1,3 @@ add_subdirectory(simple_chain) add_subdirectory(two_beamlines) +add_subdirectory(simple_chain_filegen) diff --git a/tests/automatic/full_chain/simple_chain_filegen/CMakeLists.txt b/tests/automatic/full_chain/simple_chain_filegen/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..2df30fa61e0c07b7d80307d4f08a0f1013395ac3 --- /dev/null +++ b/tests/automatic/full_chain/simple_chain_filegen/CMakeLists.txt @@ -0,0 +1,15 @@ +set(TARGET_NAME full_chain_simple_chain) + +################################ +# Testing +################################ +prepare_asapo() +if (UNIX) + set (ROOT_PATH "/tmp/asapo/") +else() + set (ROOT_PATH "c:\\\\tmp\\\\asapo\\\\") +endif() + +configure_file(test.json.in test.json @ONLY) + +add_script_test("${TARGET_NAME}" "$<TARGET_FILE:event-monitor-producer-bin> $<TARGET_FILE:getnext_broker> $<TARGET_PROPERTY:asapo,EXENAME>" nomem) diff --git a/tests/automatic/full_chain/simple_chain_filegen/check_linux.sh b/tests/automatic/full_chain/simple_chain_filegen/check_linux.sh new file mode 100644 index 0000000000000000000000000000000000000000..a5d741499e85b67ea9fe0c8f8ed72d8f303d7c01 --- /dev/null +++ b/tests/automatic/full_chain/simple_chain_filegen/check_linux.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -e + +trap Cleanup EXIT + +beamtime_id=asapo_test +token=`$3 token -secret broker_secret.key $beamtime_id` + +monitor_database_name=db_test +proxy_address=127.0.0.1:8400 + +beamline=test +receiver_root_folder=/tmp/asapo/receiver/files +receiver_folder=${receiver_root_folder}/${beamline}/${beamtime_id} + +mkdir -p /tmp/asapo/test_in/test1/ +mkdir -p /tmp/asapo/test_in/test2/ + +Cleanup() { + echo cleanup + kill $producerid + rm -rf /tmp/asapo/test_in/test1 + rm -rf /tmp/asapo/test_in/test2 + nomad stop nginx + nomad stop receiver + nomad stop discovery + nomad stop broker + nomad stop authorizer + echo "db.dropDatabase()" | mongo ${beamtime_id} +} + +echo "db.${beamtime_id}.insert({dummy:1})" | mongo ${beamtime_id} + +nomad run nginx.nmd +nomad run authorizer.nmd +nomad run receiver.nmd +nomad run discovery.nmd +nomad run broker.nmd + +sleep 1 + +#producer +mkdir -p ${receiver_folder} +$1 test.json & +producerid=`echo $!` + +sleep 1 + +echo hello > /tmp/asapo/test_in/test1/file1 +echo hello > /tmp/asapo/test_in/test1/file2 +echo hello > /tmp/asapo/test_in/test2/file2 + +$2 ${proxy_address} ${beamtime_id} 2 $token 1000 | grep "Processed 3 file(s)" diff --git a/tests/automatic/full_chain/simple_chain_filegen/check_windows.bat b/tests/automatic/full_chain/simple_chain_filegen/check_windows.bat new file mode 100644 index 0000000000000000000000000000000000000000..b3d1a1a8dee7596f76b81619cf751626ba973a2d --- /dev/null +++ b/tests/automatic/full_chain/simple_chain_filegen/check_windows.bat @@ -0,0 +1,62 @@ +SET mongo_exe="c:\Program Files\MongoDB\Server\3.6\bin\mongo.exe" +SET beamtime_id=asapo_test +SET beamline=test +SET receiver_root_folder=c:\tmp\asapo\receiver\files +SET receiver_folder="%receiver_root_folder%\%beamline%\%beamtime_id%" + +set producer_short_name="%~nx1" + + +"%3" token -secret broker_secret.key %beamtime_id% > token +set /P token=< token + +set proxy_address="127.0.0.1:8400" + +echo db.%beamtime_id%.insert({dummy:1}) | %mongo_exe% %beamtime_id% + +c:\opt\consul\nomad run receiver.nmd +c:\opt\consul\nomad run authorizer.nmd +c:\opt\consul\nomad run discovery.nmd +c:\opt\consul\nomad run broker.nmd +c:\opt\consul\nomad run nginx.nmd + +ping 1.0.0.0 -n 10 -w 100 > nul + +REM producer +mkdir c:\tmp\asapo\test_in\test1 +mkdir c:\tmp\asapo\test_in\test2 +start /B "" "%1" test.json +ping 1.0.0.0 -n 1 -w 100 > nul + +echo hello > c:\tmp\asapo\test_in\test1\file1 +echo hello > c:\tmp\asapo\test_in\test1\file2 +echo hello > c:\tmp\asapo\test_in\test2\file2 + +ping 1.0.0.0 -n 1 -w 100 > nul + + +REM worker +"%2" %proxy_address% %beamtime_id% 2 %token% 1000 | findstr /c:"Processed 3 file(s)" || goto :error + + +goto :clean + +:error +call :clean +exit /b 1 + +:clean +c:\opt\consul\nomad stop receiver +c:\opt\consul\nomad stop discovery +c:\opt\consul\nomad stop broker +c:\opt\consul\nomad stop authorizer +c:\opt\consul\nomad stop nginx +rmdir /S /Q %receiver_root_folder% +rmdir /S /Q c:\tmp\asapo\test_in\test1 +rmdir /S /Q c:\tmp\asapo\test_in\test2 +Taskkill /IM "%producer_short_name%" /F + +del /f token +echo db.dropDatabase() | %mongo_exe% %beamtime_id% + + diff --git a/tests/automatic/full_chain/simple_chain_filegen/test.json.in b/tests/automatic/full_chain/simple_chain_filegen/test.json.in new file mode 100644 index 0000000000000000000000000000000000000000..f072140a4ea55aa15aae487ce7d61cd80219fe31 --- /dev/null +++ b/tests/automatic/full_chain/simple_chain_filegen/test.json.in @@ -0,0 +1,12 @@ +{ + "AsapoEndpoint":"localhost:8400", + "Tag":"test_tag", + "BeamtimeID":"asapo_test", + "Mode":"tcp", + "NThreads":1, + "LogLevel":"debug", + "RootMonitoredFolder":"@ROOT_PATH@test_in", + "MonitoredSubFolders":["test1","test2"], + "IgnoreExtentions":["tmp"], + "RemoveAfterSend":true +} diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen.nomad b/tests/manual/tests_via_nomad/asapo-test_filegen.nomad new file mode 100644 index 0000000000000000000000000000000000000000..b3fcb094f344b94cf30b6895e2cdc6d5770c5fdc --- /dev/null +++ b/tests/manual/tests_via_nomad/asapo-test_filegen.nomad @@ -0,0 +1,71 @@ +job "asapo-filegen" { + datacenters = [ + "dc1"] + + type = "batch" + + group "windows" { + + constraint { + attribute = "${attr.kernel.name}" + value = "windows" + } + + count = 1 + + task "filegen" { + driver = "raw_exec" + + config { + command = "local/filegen_win.exe" + args = [ + "1","1M","10","c:/tmp/asapo/test_in/test_folder/file_win"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/filegen_win.exe" + mode = "file" + destination = "local/filegen_win.exe" + } + } + + + } + #windows + + group "linux" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + constraint { + attribute = "${meta.location}" + value = "petra3" + } + + count = 1 + + task "filegen" { + driver = "raw_exec" + + config { + command = "local/filegen_linux" + args = [ + "1","1M","10","/tmp/asapo/test_in/test_folder/file_lin_"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/filegen_linux" + mode = "file" + destination = "local/filegen_linux" + } + } + + } + #linux +} + + + diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_producer.nomad b/tests/manual/tests_via_nomad/asapo-test_filemon_producer.nomad new file mode 100644 index 0000000000000000000000000000000000000000..b10e845fba8d967613210c7c26177960e295d693 --- /dev/null +++ b/tests/manual/tests_via_nomad/asapo-test_filemon_producer.nomad @@ -0,0 +1,118 @@ +job "asapo-produceronly" { + datacenters = [ + "dc1"] + + type = "batch" + + group "producer-windows" { + + constraint { + attribute = "${attr.kernel.name}" + value = "windows" + } + + count = 1 + task "producer" { + driver = "raw_exec" + + config { + command = "local/asapo-eventmon-producer.exe" + args = [ + "local/test.json"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer.exe" + mode = "file" + destination = "local/asapo-eventmon-producer.exe" + } + + template { + data = <<EOH +{ + "AsapoEndpoint":"psana002:8400", + "Tag":"test_tag", + "BeamtimeID":"asapo_test1", + "Mode":"tcp", + "NThreads":1, + "LogLevel":"debug", + "RootMonitoredFolder":"c:\\tmp\\asapo\\test_in", + "MonitoredSubFolders":["test_folder"], + "IgnoreExtentions":["tmp"], + "RemoveAfterSend":true +} + EOH + destination = "local/test.json" + } + # resources { + # cpu = 5000 + # memory = 128 + # network { + # mbits = 10000 + # } + } # producer task + } + #windows + + group "producer-linux" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + constraint { + attribute = "${meta.location}" + value = "petra3" + } + + count = 1 + + task "producer" { + driver = "raw_exec" + + config { + command = "local/asapo-eventmon-producer" + args = [ + "local/test.json"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer" + mode = "file" + destination = "local/asapo-eventmon-producer" + } + + template { + data = <<EOH +{ + "AsapoEndpoint":"psana002:8400", + "Tag":"test_tag", + "BeamtimeID":"asapo_test2", + "Mode":"filesystem", + "NThreads":1, + "LogLevel":"debug", + "RootMonitoredFolder":"/tmp/asapo/test_in", + "MonitoredSubFolders":["test_folder"], + "IgnoreExtentions":["tmp"], + "RemoveAfterSend":true +} + EOH + destination = "local/test.json" + } + + # resources { + # cpu = 5000 + # memory = 128 + # network { + # mbits = 10000 + # } + # } + + } # task producer + } + #linux +} + + + diff --git a/tests/manual/tests_via_nomad/asapo-test_produceronly.nomad b/tests/manual/tests_via_nomad/asapo-test_produceronly.nomad new file mode 100644 index 0000000000000000000000000000000000000000..3ac5548e87d80d9013ecfc18ee110bfe23ea7464 --- /dev/null +++ b/tests/manual/tests_via_nomad/asapo-test_produceronly.nomad @@ -0,0 +1,118 @@ +job "asapo-produceronly" { + datacenters = [ + "dc1"] + + type = "batch" + + group "producer-windows" { + + constraint { + attribute = "${attr.kernel.name}" + value = "windows" + } + + count = 1 + task "producer" { + driver = "raw_exec" + + config { + command = "local/asapo-eventmon-producer.exe" + args = [ + "local/test.json"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer.exe" + mode = "file" + destination = "local/asapo-eventmon-producer.exe" + } + + template { + data = <<EOH +{ + "AsapoEndpoint":"c:\\tmp\\asapo\\test_out", + "Tag":"test_tag", + "BeamtimeID":"asapo_test", + "Mode":"filesystem", + "NThreads":1, + "LogLevel":"debug", + "RootMonitoredFolder":"c:\\tmp\\asapo\\test_in", + "MonitoredSubFolders":["test_folder"], + "IgnoreExtentions":["tmp"], + "RemoveAfterSend":true +} + EOH + destination = "local/test.json" + } + # resources { + # cpu = 5000 + # memory = 128 + # network { + # mbits = 10000 + # } + } # producer task + } + #windows + + group "producer-linux" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + constraint { + attribute = "${meta.location}" + value = "petra3" + } + + count = 1 + + task "producer" { + driver = "raw_exec" + + config { + command = "local/asapo-eventmon-producer" + args = [ + "local/test.json"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer" + mode = "file" + destination = "local/asapo-eventmon-producer" + } + + template { + data = <<EOH +{ + "AsapoEndpoint":"/tmp/asapo/test_out", + "Tag":"test_tag", + "BeamtimeID":"asapo_test", + "Mode":"filesystem", + "NThreads":1, + "LogLevel":"debug", + "RootMonitoredFolder":"/tmp/asapo/test_in", + "MonitoredSubFolders":["test_folder"], + "IgnoreExtentions":["tmp"], + "RemoveAfterSend":true +} + EOH + destination = "local/test.json" + } + + # resources { + # cpu = 5000 + # memory = 128 + # network { + # mbits = 10000 + # } + # } + + } # task producer + } + #linux +} + + + diff --git a/tests/manual/tests_via_nomad/copy.sh b/tests/manual/tests_via_nomad/copy.sh new file mode 100755 index 0000000000000000000000000000000000000000..e8f30e43d758e2c5348ed89984e20620119920d4 --- /dev/null +++ b/tests/manual/tests_via_nomad/copy.sh @@ -0,0 +1 @@ +scp * psana002:/bldocuments/support/asapo/config/nomad_jobs/