From cf6ce9d662d542a7402ef4490a3ec68453ac1256 Mon Sep 17 00:00:00 2001 From: Sergey Yakubov <sergey.yakubov@desy.de> Date: Fri, 2 Aug 2019 16:06:15 +0200 Subject: [PATCH] update nomad tests --- ....in => asapo-test_filegen_worker.nomad.in} | 0 ... => asapo-test_filegen_worker_1M.nomad.in} | 0 ...sapo-test_filegen_worker_1M_batch.nomad.in | 199 ++++++++++++++++++ ...est_filegen_worker_1M_multisource.nomad.in | 138 ++++++++++++ .../asapo-test_filegen_worker_batch.nomad.in | 199 ++++++++++++++++++ .../asapo-test_filemon.nomad.in | 4 +- .../asapo-test_filemon_batch.nomad.in | 6 +- .../asapo-test_filemon_multisource.nomad.in | 129 ++++++++++++ tests/manual/tests_via_nomad/run_test.sh | 7 - .../run_test_dummy_producer.sh | 4 - .../tests_via_nomad/run_test_fullchain.sh | 12 ++ .../run_test_fullchain_batch.sh | 12 ++ .../run_test_fullchain_multisource.sh | 12 ++ .../tests_via_nomad/run_test_linuxonly.sh | 7 - 14 files changed, 706 insertions(+), 23 deletions(-) rename tests/manual/tests_via_nomad/{asapo-test_filegen_filemon.nomad.in => asapo-test_filegen_worker.nomad.in} (100%) rename tests/manual/tests_via_nomad/{asapo-test_filegen_filemon_1M.nomad.in => asapo-test_filegen_worker_1M.nomad.in} (100%) create mode 100644 tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_batch.nomad.in create mode 100644 tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_multisource.nomad.in create mode 100644 tests/manual/tests_via_nomad/asapo-test_filegen_worker_batch.nomad.in create mode 100644 tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in delete mode 100755 tests/manual/tests_via_nomad/run_test.sh delete mode 100755 tests/manual/tests_via_nomad/run_test_dummy_producer.sh create mode 100755 tests/manual/tests_via_nomad/run_test_fullchain.sh create mode 100755 tests/manual/tests_via_nomad/run_test_fullchain_batch.sh create mode 100755 tests/manual/tests_via_nomad/run_test_fullchain_multisource.sh delete mode 100755 tests/manual/tests_via_nomad/run_test_linuxonly.sh diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_filemon.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filegen_worker.nomad.in similarity index 100% rename from tests/manual/tests_via_nomad/asapo-test_filegen_filemon.nomad.in rename to tests/manual/tests_via_nomad/asapo-test_filegen_worker.nomad.in diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_filemon_1M.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M.nomad.in similarity index 100% rename from tests/manual/tests_via_nomad/asapo-test_filegen_filemon_1M.nomad.in rename to tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M.nomad.in diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_batch.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_batch.nomad.in new file mode 100644 index 000000000..2ce7d3824 --- /dev/null +++ b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_batch.nomad.in @@ -0,0 +1,199 @@ +job "asapo-test" { + datacenters = [ + "dc1"] + + type = "batch" + + group "filegen-windows" { + + constraint { + attribute = "${attr.kernel.name}" + value = "windows" + } + + count = 1 + + task "filegen" { + driver = "raw_exec" + + config { + command = "local/filegen_win.exe" + args = [ + "1", + "1M", + "100000", + "1100", + "u:/asapo/test_folder/file_win"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/filegen_win.exe" + mode = "file" + destination = "local/filegen_win.exe" + } + } + + + } + #windows + + group "filegen-linux" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + constraint { + attribute = "${meta.location}" + value = "petra3" + } + + count = 1 + + task "filegen" { + driver = "raw_exec" + + config { + command = "local/filegen_linux" + args = [ + "1", + "1M", + "100000", + "1200", + "/run/user/data/file_lin"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/filegen_linux" + mode = "file" + destination = "local/filegen_linux" + } + } + + } + + + group "worker-linux1" { + + restart { + attempts = 0 + } + + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + # constraint { + # attribute = "${meta.location}" + # operator = "!=" + # value = "petra3" + # } + + count = 1 + + task "worker-linux" { + driver = "raw_exec" + template { + data = <<EOH + WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}" + EOH + destination = "secrets/file.env" + env = true + } + + config { + command = "local/getnext_broker" + args = [ + "psana002:8400", + "/bldocuments/support/asapo/data/test1/asapo_test1", + "asapo_test1", + "16", + "oTsKsj8i6WcW_gVzeIFvZCtSfMErjDELJEyAI23n7Ik=", + "30000", + "${WORKER_READ_META_ONLY}", + "1"] + } + +# resources { +# cpu = 5000 +# memory = 128 +# network { +# mbits = 10000 +# } +# } + + artifact { + source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@" + mode = "file" + destination = "local/getnext_broker" + } + } + + } + # worker-linux1 + + + group "worker-linux2" { + + restart { + attempts = 0 + } + + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + # constraint { + # attribute = "${meta.location}" + # operator = "!=" + # value = "petra3" + # } + + count = 1 + + task "worker-linux" { + driver = "raw_exec" + template { + data = <<EOH + WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}" + EOH + destination = "secrets/file.env" + env = true + } + + config { + command = "local/getnext_broker" + args = [ + "psana002:8400", + "/bldocuments/support/asapo/data/test2/asapo_test2", + "asapo_test2", + "16", + "yzgAcLmijSLWIm8dBiGNCbc0i42u5HSm-zR6FRqo__Y=", + "30000", + "${WORKER_READ_META_ONLY}", + "1"] + } +# resources { +# cpu = 5000 +# memory = 128 +# network { +# mbits = 10000 +# } +# } + + artifact { + source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@" + mode = "file" + destination = "local/getnext_broker" + } + } + + } + # worker-linux2 + +} diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_multisource.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_multisource.nomad.in new file mode 100644 index 000000000..26623e83b --- /dev/null +++ b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_multisource.nomad.in @@ -0,0 +1,138 @@ +job "asapo-test" { + datacenters = [ + "dc1"] + + type = "batch" + + group "filegen-windows" { + + constraint { + attribute = "${attr.kernel.name}" + value = "windows" + } + + count = 1 + + task "filegen" { + driver = "raw_exec" + + config { + command = "local/filegen_win.exe" + args = [ + "1", + "1M", + "100000", + "1100", + "u:/asapo/test_folder/file_win"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/filegen_win.exe" + mode = "file" + destination = "local/filegen_win.exe" + } + } + + + } + #windows + + group "filegen-linux" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + constraint { + attribute = "${meta.location}" + value = "petra3" + } + + count = 1 + + task "filegen" { + driver = "raw_exec" + + config { + command = "local/filegen_linux" + args = [ + "1", + "1M", + "100000", + "1200", + "/run/user/data/file_lin"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/filegen_linux" + mode = "file" + destination = "local/filegen_linux" + } + } + + } + + + group "worker" { + + restart { + attempts = 0 + } + + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + # constraint { + # attribute = "${meta.location}" + # operator = "!=" + # value = "petra3" + # } + + count = 1 + + task "worker-linux" { + driver = "raw_exec" + template { + data = <<EOH + WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}" + EOH + destination = "secrets/file.env" + env = true + } + + config { + command = "local/getnext_broker" + args = [ + "psana002:8400", + "/bldocuments/support/asapo/data/test1/asapo_test1", + "asapo_test", + "16", + "KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc=", + "30000", + "${WORKER_READ_META_ONLY}", + "1"] + } + +# resources { +# cpu = 5000 +# memory = 128 +# network { +# mbits = 10000 +# } +# } + + artifact { + source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@" + mode = "file" + destination = "local/getnext_broker" + } + } + + } + # worker + +} diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_worker_batch.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_batch.nomad.in new file mode 100644 index 000000000..1298b228b --- /dev/null +++ b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_batch.nomad.in @@ -0,0 +1,199 @@ +job "asapo-test" { + datacenters = [ + "dc1"] + + type = "batch" + + group "filegen-windows" { + + constraint { + attribute = "${attr.kernel.name}" + value = "windows" + } + + count = 1 + + task "filegen" { + driver = "raw_exec" + + config { + command = "local/filegen_win.exe" + args = [ + "1", + "10M", + "10000", + "120", + "u:/asapo/test_folder/file_win"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/filegen_win.exe" + mode = "file" + destination = "local/filegen_win.exe" + } + } + + + } + #windows + + group "filegen-linux" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + constraint { + attribute = "${meta.location}" + value = "petra3" + } + + count = 1 + + task "filegen" { + driver = "raw_exec" + + config { + command = "local/filegen_linux" + args = [ + "1", + "10M", + "10000", + "120", + "/run/user/data/file_lin"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/filegen_linux" + mode = "file" + destination = "local/filegen_linux" + } + } + + } + + + group "worker-linux1" { + + restart { + attempts = 0 + } + + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + # constraint { + # attribute = "${meta.location}" + # operator = "!=" + # value = "petra3" + # } + + count = 1 + + task "worker-linux" { + driver = "raw_exec" + template { + data = <<EOH + WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}" + EOH + destination = "secrets/file.env" + env = true + } + + config { + command = "local/getnext_broker" + args = [ + "psana002:8400", + "/bldocuments/support/asapo/data/test1/asapo_test1", + "asapo_test1", + "16", + "oTsKsj8i6WcW_gVzeIFvZCtSfMErjDELJEyAI23n7Ik=", + "30000", + "${WORKER_READ_META_ONLY}", + "1"] + } + +# resources { +# cpu = 5000 +# memory = 128 +# network { +# mbits = 10000 +# } +# } + + artifact { + source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@" + mode = "file" + destination = "local/getnext_broker" + } + } + + } + # worker-linux1 + + + group "worker-linux2" { + + restart { + attempts = 0 + } + + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + # constraint { + # attribute = "${meta.location}" + # operator = "!=" + # value = "petra3" + # } + + count = 1 + + task "worker-linux" { + driver = "raw_exec" + template { + data = <<EOH + WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}" + EOH + destination = "secrets/file.env" + env = true + } + + config { + command = "local/getnext_broker" + args = [ + "psana002:8400", + "/bldocuments/support/asapo/data/test2/asapo_test2", + "asapo_test2", + "16", + "yzgAcLmijSLWIm8dBiGNCbc0i42u5HSm-zR6FRqo__Y=", + "30000", + "${WORKER_READ_META_ONLY}", + "1"] + } +# resources { +# cpu = 5000 +# memory = 128 +# network { +# mbits = 10000 +# } +# } + + artifact { + source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@" + mode = "file" + destination = "local/getnext_broker" + } + } + + } + # worker-linux2 + +} diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in index 5e69201b5..250a26803 100644 --- a/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in +++ b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in @@ -35,7 +35,7 @@ job "asapo-filemon" { "BeamtimeID":"asapo_test1", "Mode":"tcp", "NThreads":8, - "LogLevel":"debug", + "LogLevel":"info", "RootMonitoredFolder":"u:\\asapo", "MonitoredSubFolders":["test_folder"], "IgnoreExtentions":["tmp"], @@ -94,7 +94,7 @@ job "asapo-filemon" { "BeamtimeID":"asapo_test2", "Mode":"tcp", "NThreads":8, - "LogLevel":"debug", + "LogLevel":"info", "RootMonitoredFolder":"/run/user", "MonitoredSubFolders":["data"], "IgnoreExtentions":["tmp"], diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in index 37df9e4d8..85e5872e9 100644 --- a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in +++ b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in @@ -1,4 +1,4 @@ -job "asapo-filemon" { +job "asapo-filemon_batch" { datacenters = [ "dc1"] @@ -35,7 +35,7 @@ job "asapo-filemon" { "BeamtimeID":"asapo_test1", "Mode":"tcp", "NThreads":8, - "LogLevel":"debug", + "LogLevel":"info", "RootMonitoredFolder":"u:\\asapo", "MonitoredSubFolders":["test_folder"], "IgnoreExtentions":["tmp"], @@ -95,7 +95,7 @@ job "asapo-filemon" { "BeamtimeID":"asapo_test2", "Mode":"tcp", "NThreads":8, - "LogLevel":"debug", + "LogLevel":"info", "RootMonitoredFolder":"/run/user", "MonitoredSubFolders":["data"], "IgnoreExtentions":["tmp"], diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in new file mode 100644 index 000000000..2a6159367 --- /dev/null +++ b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in @@ -0,0 +1,129 @@ +job "asapo-filemon_multisource" { + datacenters = [ + "dc1"] + + type = "batch" + + group "producer-windows" { + + constraint { + attribute = "${attr.kernel.name}" + value = "windows" + } + + count = 1 + task "producer" { + driver = "raw_exec" + + config { + command = "local/asapo-eventmon-producer.exe" + args = [ + "local/test.json"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@.exe" + mode = "file" + destination = "local/asapo-eventmon-producer.exe" + } + + template { + data = <<EOH +{ + "AsapoEndpoint":"psana002:8400", + "Tag":"test_tag", + "BeamtimeID":"asapo_test", + "Mode":"tcp", + "NThreads":8, + "LogLevel":"info", + "RootMonitoredFolder":"u:\\asapo", + "MonitoredSubFolders":["test_folder"], + "IgnoreExtentions":["tmp"], + "RemoveAfterSend":true, + "Subset": { + "Mode":"multisource", + "SourceId": 1, + "NSources":2 + } +} + EOH + destination = "local/test.json" + } + # resources { + # cpu = 5000 + # memory = 128 + # network { + # mbits = 10000 + # } + } # producer task + } + #windows + + group "producer-linux" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + constraint { + attribute = "${meta.location}" + value = "petra3" + } + + count = 1 + + task "producer" { + driver = "raw_exec" + + config { + command = "local/asapo-eventmon-producer" + args = [ + "local/test.json"] + } + + artifact { + source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@" + mode = "file" + destination = "local/asapo-eventmon-producer" + } + + template { + data = <<EOH +{ + "AsapoEndpoint":"psana002:8400", + "Tag":"test_tag", + "BeamtimeID":"asapo_test", + "Mode":"tcp", + "NThreads":8, + "LogLevel":"info", + "RootMonitoredFolder":"/run/user", + "MonitoredSubFolders":["data"], + "IgnoreExtentions":["tmp"], + "RemoveAfterSend":true, + "Subset": { + "Mode":"multisource", + "SourceId": 2, + "NSources":2 + } + +} + EOH + destination = "local/test.json" + } + + # resources { + # cpu = 5000 + # memory = 128 + # network { + # mbits = 10000 + # } + # } + + } # task producer + } + #linux +} + + + diff --git a/tests/manual/tests_via_nomad/run_test.sh b/tests/manual/tests_via_nomad/run_test.sh deleted file mode 100755 index 0460ca5e6..000000000 --- a/tests/manual/tests_via_nomad/run_test.sh +++ /dev/null @@ -1,7 +0,0 @@ -. ./clean_after_tests.sh -nomad stop asapo-filemon-producer -nomad run asapo-test_filemon_producer_toreceiver.nomad -sleep 1 -nomad stop asapo-test -nomad run asapo-test_filegen_filemon.nomad - diff --git a/tests/manual/tests_via_nomad/run_test_dummy_producer.sh b/tests/manual/tests_via_nomad/run_test_dummy_producer.sh deleted file mode 100755 index dda2a1b3b..000000000 --- a/tests/manual/tests_via_nomad/run_test_dummy_producer.sh +++ /dev/null @@ -1,4 +0,0 @@ -. ./clean_after_tests.sh -nomad stop asapo-test -nomad run asapo-test_dummy_producer_linux_noworker.nomad - diff --git a/tests/manual/tests_via_nomad/run_test_fullchain.sh b/tests/manual/tests_via_nomad/run_test_fullchain.sh new file mode 100755 index 000000000..9f842d002 --- /dev/null +++ b/tests/manual/tests_via_nomad/run_test_fullchain.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +nomad stop asapo-filemon +nomad stop asapo-filemon_batch +nomad stop asapo-filemon_multisource +nomad run asapo-test_filemon_batch.nomad +sleep 1 +nomad stop asapo-test +. ./clean_after_tests.sh + +nomad run asapo-test_filegen_worker.nomad + diff --git a/tests/manual/tests_via_nomad/run_test_fullchain_batch.sh b/tests/manual/tests_via_nomad/run_test_fullchain_batch.sh new file mode 100755 index 000000000..459aa9f66 --- /dev/null +++ b/tests/manual/tests_via_nomad/run_test_fullchain_batch.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +nomad stop asapo-filemon +nomad stop asapo-filemon_batch +nomad stop asapo-filemon_multisource +nomad run asapo-test_filemon_batch.nomad +sleep 1 +nomad stop asapo-test +. ./clean_after_tests.sh + +nomad run asapo-test_filegen_worker_batch.nomad + diff --git a/tests/manual/tests_via_nomad/run_test_fullchain_multisource.sh b/tests/manual/tests_via_nomad/run_test_fullchain_multisource.sh new file mode 100755 index 000000000..8a5a8687c --- /dev/null +++ b/tests/manual/tests_via_nomad/run_test_fullchain_multisource.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +nomad stop asapo-filemon +nomad stop asapo-filemon_batch +nomad stop asapo-filemon_multisource +nomad run asapo-test_filemon_multisource.nomad +sleep 1 +nomad stop asapo-test +. ./clean_after_tests.sh + +nomad run asapo-test_filegen_worker_1M_multisource.nomad + diff --git a/tests/manual/tests_via_nomad/run_test_linuxonly.sh b/tests/manual/tests_via_nomad/run_test_linuxonly.sh deleted file mode 100755 index 039793552..000000000 --- a/tests/manual/tests_via_nomad/run_test_linuxonly.sh +++ /dev/null @@ -1,7 +0,0 @@ -. ./clean_after_tests.sh -nomad stop asapo-filemon-producer -nomad run asapo-test_filemon_producer_toreceiver.nomad -sleep 1 -nomad stop asapo-test -nomad run asapo-test_filegen_filemon_linuxonly.nomad - -- GitLab