diff --git a/tests/manual/performance_broker_receiver/getlast_broker.cpp b/tests/manual/performance_broker_receiver/getlast_broker.cpp
index d339a65fc9487d2ecf391da6663168f821be743a..29ee0e4a0a08e8b7d3010f5060f1f7d1aae5d07c 100644
--- a/tests/manual/performance_broker_receiver/getlast_broker.cpp
+++ b/tests/manual/performance_broker_receiver/getlast_broker.cpp
@@ -6,17 +6,16 @@
#include <chrono>
#include <iomanip>
#include <numeric>
+#include <mutex>
#include "asapo_worker.h"
-#include <mutex>
+using std::chrono::system_clock;
+using asapo::Error;
std::string group_id = "";
std::mutex lock;
-using std::chrono::system_clock;
-using asapo::Error;
-
struct Params {
std::string server;
std::string file_path;
@@ -25,6 +24,7 @@ struct Params {
int timeout_ms;
int nthreads;
bool read_data;
+ bool datasets;
};
void WaitThreads(std::vector<std::thread>* threads) {
@@ -42,47 +42,57 @@ int ProcessError(const Error& err) {
std::vector<std::thread> StartThreads(const Params& params,
std::vector<int>* nfiles,
std::vector<int>* errors,
- std::vector<int>* nbuf) {
- auto exec_next = [¶ms, nfiles, errors, nbuf](int i) {
- asapo::FileInfo fi;
- Error err;
- auto broker = asapo::DataBrokerFactory::CreateServerBroker(params.server, params.file_path, params.beamtime_id,
- params.token, &err);
- broker->SetTimeout(params.timeout_ms);
- asapo::FileData data;
-
- lock.lock();
-
- if (group_id.empty()) {
- group_id = broker->GenerateNewGroupId(&err);
- if (err) {
- (*errors)[i] += ProcessError(err);
- return;
- }
- }
-
- lock.unlock();
-
- auto start = system_clock::now();
- while (std::chrono::duration_cast<std::chrono::milliseconds>(system_clock::now() - start).count() <
- params.timeout_ms) {
- err = broker->GetLast(&fi, group_id, params.read_data ? &data : nullptr);
- if (err == nullptr) {
- (*nbuf)[i] += fi.buf_id == 0 ? 0 : 1;
- if (params.read_data && (*nfiles)[i] < 10 && fi.size < 10) {
- if (data != nullptr) {
- data[9] = 0;
- std::cout << "Received: " << reinterpret_cast<char const*>(data.get()) << std::endl;
- }
- }
- } else {
- (*errors)[i] += ProcessError(err);
- if (err == asapo::IOErrorTemplates::kTimeout) {
- break;
- }
- }
- (*nfiles)[i]++;
- }
+ std::vector<int>* nbuf,
+ std::vector<int>* nfiles_total) {
+ auto exec_next = [¶ms, nfiles, errors, nbuf, nfiles_total](int i) {
+ asapo::FileInfo fi;
+ Error err;
+ auto broker = asapo::DataBrokerFactory::CreateServerBroker(params.server, params.file_path, params.beamtime_id,
+ params.token, &err);
+ broker->SetTimeout((uint64_t) params.timeout_ms);
+ asapo::FileData data;
+
+ lock.lock();
+
+ if (group_id.empty()) {
+ group_id = broker->GenerateNewGroupId(&err);
+ if (err) {
+ (*errors)[i] += ProcessError(err);
+ return;
+ }
+ }
+
+ lock.unlock();
+
+ auto start = system_clock::now();
+ while (std::chrono::duration_cast<std::chrono::milliseconds>(system_clock::now() - start).count() <
+ params.timeout_ms) {
+ if (params.datasets) {
+ auto dataset = broker->GetLastDataset(group_id, &err);
+ if (err == nullptr) {
+ for (auto& fi : dataset.content) {
+ (*nbuf)[i] += fi.buf_id == 0 ? 0 : 1;
+ (*nfiles_total)[i]++;
+ }
+ }
+ } else {
+ err = broker->GetLast(&fi, group_id, params.read_data ? &data : nullptr);
+ if (err == nullptr) {
+ (*nbuf)[i] += fi.buf_id == 0 ? 0 : 1;
+ if (params.read_data && (*nfiles)[i] < 10 && fi.size < 10) {
+ data[9] = 0;
+ std::cout << "Received: " << reinterpret_cast<char const*>(data.get()) << std::endl;
+ }
+ }
+ }
+ if (err) {
+ (*errors)[i] += ProcessError(err);
+ if (err == asapo::IOErrorTemplates::kTimeout) {
+ break;
+ }
+ }
+ (*nfiles)[i]++;
+ }
};
std::vector<std::thread> threads;
@@ -92,21 +102,22 @@ std::vector<std::thread> StartThreads(const Params& params,
return threads;
}
-int ReadAllData(const Params& params, uint64_t* duration_ms, int* nerrors, int* nbuf) {
+int ReadAllData(const Params& params, uint64_t* duration_ms, int* nerrors, int* nbuf, int* nfiles_total) {
asapo::FileInfo fi;
system_clock::time_point t1 = system_clock::now();
std::vector<int> nfiles(params.nthreads, 0);
std::vector<int> errors(params.nthreads, 0);
std::vector<int> nfiles_frombuf(params.nthreads, 0);
+ std::vector<int> nfiles_total_in_datasets(params.nthreads, 0);
- auto threads = StartThreads(params, &nfiles, &errors, &nfiles_frombuf);
+ auto threads = StartThreads(params, &nfiles, &errors, &nfiles_frombuf, &nfiles_total_in_datasets);
WaitThreads(&threads);
int n_total = std::accumulate(nfiles.begin(), nfiles.end(), 0);
*nerrors = std::accumulate(errors.begin(), errors.end(), 0);
*nbuf = std::accumulate(nfiles_frombuf.begin(), nfiles_frombuf.end(), 0);
-
+ *nfiles_total = std::accumulate(nfiles_total_in_datasets.begin(), nfiles_total_in_datasets.end(), 0);
system_clock::time_point t2 = system_clock::now();
auto duration_read = std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1);
@@ -116,14 +127,15 @@ int ReadAllData(const Params& params, uint64_t* duration_ms, int* nerrors, int*
int main(int argc, char* argv[]) {
asapo::ExitAfterPrintVersionIfNeeded("GetLast Broker Example", argc, argv);
- if (argc != 8) {
+ Params params;
+ params.datasets = false;
+ if (argc != 8 && argc != 9) {
std::cout << "Usage: " + std::string{argv[0]}
- + " <server> <files_path> <run_name> <nthreads> <token> <timeout ms> <metaonly>"
+ + " <server> <files_path> <run_name> <nthreads> <token> <timeout ms> <metaonly> [use datasets]"
<<
std::endl;
exit(EXIT_FAILURE);
}
- Params params;
params.server = std::string{argv[1]};
params.file_path = std::string{argv[2]};
params.beamtime_id = std::string{argv[3]};
@@ -131,14 +143,18 @@ int main(int argc, char* argv[]) {
params.token = std::string{argv[5]};
params.timeout_ms = atoi(argv[6]);
params.read_data = atoi(argv[7]) != 1;
-
+ if (argc == 9) {
+ params.datasets = atoi(argv[8]) == 1;
+ }
uint64_t duration_ms;
- int nerrors, nbuf;
- auto nfiles = ReadAllData(params, &duration_ms, &nerrors, &nbuf);
-
- std::cout << "Processed " << nfiles << " file(s)" << std::endl;
+ int nerrors, nbuf, nfiles_total;
+ auto nfiles = ReadAllData(params, &duration_ms, &nerrors, &nbuf, &nfiles_total);
+ std::cout << "Processed " << nfiles << (params.datasets ? " dataset(s)" : " file(s)") << std::endl;
+ if (params.datasets) {
+ std::cout << " with " << nfiles_total << " file(s)" << std::endl;
+ }
std::cout << "Successfully: " << nfiles - nerrors << std::endl;
- if (params.read_data) {
+ if (params.read_data && !params.datasets) {
std::cout << " from memory buffer: " << nbuf << std::endl;
std::cout << " from filesystem: " << nfiles - nerrors - nbuf << std::endl;
}
diff --git a/tests/manual/tests_via_nomad/asapo-test_dummy_producer_only.nomad.in b/tests/manual/tests_via_nomad/asapo-test_dummy_producer_only.nomad.in
new file mode 100644
index 0000000000000000000000000000000000000000..0effcca2a01a2244364f6eaf6b0550c5b43f5e0c
--- /dev/null
+++ b/tests/manual/tests_via_nomad/asapo-test_dummy_producer_only.nomad.in
@@ -0,0 +1,211 @@
+job "asapo-test" {
+ datacenters = [
+ "dc1"]
+
+ type = "batch"
+
+ group "producer-windows" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "windows"
+ }
+
+ count = 0
+ task "producer" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/dummy-data-producer.exe"
+ args = [
+ "psana002:8400",
+ "asapo_test1",
+ "100",
+ "1000000",
+ "8",
+ "0",
+ "1000"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/dummy-data-producer-@ASAPO_VERSION@.exe"
+ mode = "file"
+ destination = "local/dummy-data-producer.exe"
+ }
+
+ # resources {
+ # cpu = 5000
+ # memory = 128
+ # network {
+ # mbits = 10000
+ # }
+ }
+
+ }
+ #windows
+
+ group "producer-linux" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ constraint {
+ attribute = "${meta.location}"
+ value = "petra3"
+ }
+
+ count = 1
+
+ task "producer" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/dummy-data-producer"
+ args = [
+ "psana002:8400",
+ "asapo_test2",
+ "100",
+ "1000",
+ "8",
+ "0",
+ "1000",
+ "1000"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/dummy-data-producer-@ASAPO_VERSION@"
+ mode = "file"
+ destination = "local/dummy-data-producer"
+ }
+
+ # resources {
+ # cpu = 5000
+ # memory = 128
+ # network {
+ # mbits = 10000
+ # }
+ # }
+
+ }
+
+ }
+ #linux
+
+ group "worker-linux1" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ # constraint {
+ # attribute = "${meta.location}"
+ # operator = "!="
+ # value = "petra3"
+ # }
+
+ count = 0
+
+
+ task "worker-linux" {
+ driver = "raw_exec"
+
+ template {
+ data = <<EOH
+ WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+ EOH
+ destination = "secrets/file.env"
+ env = true
+ }
+
+ config {
+ command = "local/getnext_broker"
+ args = [
+ "psana002:8400",
+ "/bldocuments/support/asapo/data/test1/asapo_test1",
+ "asapo_test1",
+ "16",
+ "oTsKsj8i6WcW_gVzeIFvZCtSfMErjDELJEyAI23n7Ik=",
+ "30000",
+ "${WORKER_READ_META_ONLY}"]
+ }
+
+ resources {
+ cpu = 5000
+ memory = 128
+ network {
+ mbits = 10000
+ }
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+ mode = "file"
+ destination = "local/getnext_broker"
+ }
+ }
+
+ }
+ # worker-linux1
+
+
+ group "worker-linux2" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ # constraint {
+ # attribute = "${meta.location}"
+ # operator = "!="
+ # value = "petra3"
+ # }
+
+
+ count = 0
+
+ task "worker-linux" {
+ driver = "raw_exec"
+
+ template {
+ data = <<EOH
+ WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+ EOH
+ destination = "secrets/file.env"
+ env = true
+ }
+
+ config {
+ command = "local/getnext_broker"
+ args = [
+ "psana002:8400",
+ "/bldocuments/support/asapo/data/test2/asapo_test2",
+ "asapo_test2",
+ "16",
+ "yzgAcLmijSLWIm8dBiGNCbc0i42u5HSm-zR6FRqo__Y=",
+ "30000",
+ "${WORKER_READ_META_ONLY}"]
+ }
+ resources {
+ cpu = 5000
+ memory = 128
+ network {
+ mbits = 10000
+ }
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+ mode = "file"
+ destination = "local/getnext_broker"
+
+ }
+ }
+
+ }
+
+}
+# worker-linux2
diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_filemon.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filegen_worker.nomad.in
similarity index 100%
rename from tests/manual/tests_via_nomad/asapo-test_filegen_filemon.nomad.in
rename to tests/manual/tests_via_nomad/asapo-test_filegen_worker.nomad.in
diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_filemon_1M.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M.nomad.in
similarity index 100%
rename from tests/manual/tests_via_nomad/asapo-test_filegen_filemon_1M.nomad.in
rename to tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M.nomad.in
diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_batch.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_batch.nomad.in
new file mode 100644
index 0000000000000000000000000000000000000000..2ce7d38242e70bd5ce9ba73f651ca04edbe339ce
--- /dev/null
+++ b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_batch.nomad.in
@@ -0,0 +1,199 @@
+job "asapo-test" {
+ datacenters = [
+ "dc1"]
+
+ type = "batch"
+
+ group "filegen-windows" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "windows"
+ }
+
+ count = 1
+
+ task "filegen" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/filegen_win.exe"
+ args = [
+ "1",
+ "1M",
+ "100000",
+ "1100",
+ "u:/asapo/test_folder/file_win"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/filegen_win.exe"
+ mode = "file"
+ destination = "local/filegen_win.exe"
+ }
+ }
+
+
+ }
+ #windows
+
+ group "filegen-linux" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ constraint {
+ attribute = "${meta.location}"
+ value = "petra3"
+ }
+
+ count = 1
+
+ task "filegen" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/filegen_linux"
+ args = [
+ "1",
+ "1M",
+ "100000",
+ "1200",
+ "/run/user/data/file_lin"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/filegen_linux"
+ mode = "file"
+ destination = "local/filegen_linux"
+ }
+ }
+
+ }
+
+
+ group "worker-linux1" {
+
+ restart {
+ attempts = 0
+ }
+
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ # constraint {
+ # attribute = "${meta.location}"
+ # operator = "!="
+ # value = "petra3"
+ # }
+
+ count = 1
+
+ task "worker-linux" {
+ driver = "raw_exec"
+ template {
+ data = <<EOH
+ WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+ EOH
+ destination = "secrets/file.env"
+ env = true
+ }
+
+ config {
+ command = "local/getnext_broker"
+ args = [
+ "psana002:8400",
+ "/bldocuments/support/asapo/data/test1/asapo_test1",
+ "asapo_test1",
+ "16",
+ "oTsKsj8i6WcW_gVzeIFvZCtSfMErjDELJEyAI23n7Ik=",
+ "30000",
+ "${WORKER_READ_META_ONLY}",
+ "1"]
+ }
+
+# resources {
+# cpu = 5000
+# memory = 128
+# network {
+# mbits = 10000
+# }
+# }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+ mode = "file"
+ destination = "local/getnext_broker"
+ }
+ }
+
+ }
+ # worker-linux1
+
+
+ group "worker-linux2" {
+
+ restart {
+ attempts = 0
+ }
+
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ # constraint {
+ # attribute = "${meta.location}"
+ # operator = "!="
+ # value = "petra3"
+ # }
+
+ count = 1
+
+ task "worker-linux" {
+ driver = "raw_exec"
+ template {
+ data = <<EOH
+ WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+ EOH
+ destination = "secrets/file.env"
+ env = true
+ }
+
+ config {
+ command = "local/getnext_broker"
+ args = [
+ "psana002:8400",
+ "/bldocuments/support/asapo/data/test2/asapo_test2",
+ "asapo_test2",
+ "16",
+ "yzgAcLmijSLWIm8dBiGNCbc0i42u5HSm-zR6FRqo__Y=",
+ "30000",
+ "${WORKER_READ_META_ONLY}",
+ "1"]
+ }
+# resources {
+# cpu = 5000
+# memory = 128
+# network {
+# mbits = 10000
+# }
+# }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+ mode = "file"
+ destination = "local/getnext_broker"
+ }
+ }
+
+ }
+ # worker-linux2
+
+}
diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_multisource.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_multisource.nomad.in
new file mode 100644
index 0000000000000000000000000000000000000000..26623e83bf7473b7e0292ef3b68b8c5d2f716e38
--- /dev/null
+++ b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_multisource.nomad.in
@@ -0,0 +1,138 @@
+job "asapo-test" {
+ datacenters = [
+ "dc1"]
+
+ type = "batch"
+
+ group "filegen-windows" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "windows"
+ }
+
+ count = 1
+
+ task "filegen" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/filegen_win.exe"
+ args = [
+ "1",
+ "1M",
+ "100000",
+ "1100",
+ "u:/asapo/test_folder/file_win"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/filegen_win.exe"
+ mode = "file"
+ destination = "local/filegen_win.exe"
+ }
+ }
+
+
+ }
+ #windows
+
+ group "filegen-linux" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ constraint {
+ attribute = "${meta.location}"
+ value = "petra3"
+ }
+
+ count = 1
+
+ task "filegen" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/filegen_linux"
+ args = [
+ "1",
+ "1M",
+ "100000",
+ "1200",
+ "/run/user/data/file_lin"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/filegen_linux"
+ mode = "file"
+ destination = "local/filegen_linux"
+ }
+ }
+
+ }
+
+
+ group "worker" {
+
+ restart {
+ attempts = 0
+ }
+
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ # constraint {
+ # attribute = "${meta.location}"
+ # operator = "!="
+ # value = "petra3"
+ # }
+
+ count = 1
+
+ task "worker-linux" {
+ driver = "raw_exec"
+ template {
+ data = <<EOH
+ WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+ EOH
+ destination = "secrets/file.env"
+ env = true
+ }
+
+ config {
+ command = "local/getnext_broker"
+ args = [
+ "psana002:8400",
+ "/bldocuments/support/asapo/data/test1/asapo_test1",
+ "asapo_test",
+ "16",
+ "KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc=",
+ "30000",
+ "${WORKER_READ_META_ONLY}",
+ "1"]
+ }
+
+# resources {
+# cpu = 5000
+# memory = 128
+# network {
+# mbits = 10000
+# }
+# }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+ mode = "file"
+ destination = "local/getnext_broker"
+ }
+ }
+
+ }
+ # worker
+
+}
diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_worker_batch.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_batch.nomad.in
new file mode 100644
index 0000000000000000000000000000000000000000..1298b228b1cfe5713c7c68f73f782cb76ec277b0
--- /dev/null
+++ b/tests/manual/tests_via_nomad/asapo-test_filegen_worker_batch.nomad.in
@@ -0,0 +1,199 @@
+job "asapo-test" {
+ datacenters = [
+ "dc1"]
+
+ type = "batch"
+
+ group "filegen-windows" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "windows"
+ }
+
+ count = 1
+
+ task "filegen" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/filegen_win.exe"
+ args = [
+ "1",
+ "10M",
+ "10000",
+ "120",
+ "u:/asapo/test_folder/file_win"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/filegen_win.exe"
+ mode = "file"
+ destination = "local/filegen_win.exe"
+ }
+ }
+
+
+ }
+ #windows
+
+ group "filegen-linux" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ constraint {
+ attribute = "${meta.location}"
+ value = "petra3"
+ }
+
+ count = 1
+
+ task "filegen" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/filegen_linux"
+ args = [
+ "1",
+ "10M",
+ "10000",
+ "120",
+ "/run/user/data/file_lin"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/filegen_linux"
+ mode = "file"
+ destination = "local/filegen_linux"
+ }
+ }
+
+ }
+
+
+ group "worker-linux1" {
+
+ restart {
+ attempts = 0
+ }
+
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ # constraint {
+ # attribute = "${meta.location}"
+ # operator = "!="
+ # value = "petra3"
+ # }
+
+ count = 1
+
+ task "worker-linux" {
+ driver = "raw_exec"
+ template {
+ data = <<EOH
+ WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+ EOH
+ destination = "secrets/file.env"
+ env = true
+ }
+
+ config {
+ command = "local/getnext_broker"
+ args = [
+ "psana002:8400",
+ "/bldocuments/support/asapo/data/test1/asapo_test1",
+ "asapo_test1",
+ "16",
+ "oTsKsj8i6WcW_gVzeIFvZCtSfMErjDELJEyAI23n7Ik=",
+ "30000",
+ "${WORKER_READ_META_ONLY}",
+ "1"]
+ }
+
+# resources {
+# cpu = 5000
+# memory = 128
+# network {
+# mbits = 10000
+# }
+# }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+ mode = "file"
+ destination = "local/getnext_broker"
+ }
+ }
+
+ }
+ # worker-linux1
+
+
+ group "worker-linux2" {
+
+ restart {
+ attempts = 0
+ }
+
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ # constraint {
+ # attribute = "${meta.location}"
+ # operator = "!="
+ # value = "petra3"
+ # }
+
+ count = 1
+
+ task "worker-linux" {
+ driver = "raw_exec"
+ template {
+ data = <<EOH
+ WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+ EOH
+ destination = "secrets/file.env"
+ env = true
+ }
+
+ config {
+ command = "local/getnext_broker"
+ args = [
+ "psana002:8400",
+ "/bldocuments/support/asapo/data/test2/asapo_test2",
+ "asapo_test2",
+ "16",
+ "yzgAcLmijSLWIm8dBiGNCbc0i42u5HSm-zR6FRqo__Y=",
+ "30000",
+ "${WORKER_READ_META_ONLY}",
+ "1"]
+ }
+# resources {
+# cpu = 5000
+# memory = 128
+# network {
+# mbits = 10000
+# }
+# }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+ mode = "file"
+ destination = "local/getnext_broker"
+ }
+ }
+
+ }
+ # worker-linux2
+
+}
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_toreceiver.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in
similarity index 96%
rename from tests/manual/tests_via_nomad/asapo-test_filemon_producer_toreceiver.nomad.in
rename to tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in
index b9c374bcde07d325cdcb0418980790225dfca88a..250a268036d383197c72106e42ca991914c4b781 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_toreceiver.nomad.in
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in
@@ -1,4 +1,4 @@
-job "asapo-filemon-producer" {
+job "asapo-filemon" {
datacenters = [
"dc1"]
@@ -35,7 +35,7 @@ job "asapo-filemon-producer" {
"BeamtimeID":"asapo_test1",
"Mode":"tcp",
"NThreads":8,
- "LogLevel":"debug",
+ "LogLevel":"info",
"RootMonitoredFolder":"u:\\asapo",
"MonitoredSubFolders":["test_folder"],
"IgnoreExtentions":["tmp"],
@@ -94,7 +94,7 @@ job "asapo-filemon-producer" {
"BeamtimeID":"asapo_test2",
"Mode":"tcp",
"NThreads":8,
- "LogLevel":"debug",
+ "LogLevel":"info",
"RootMonitoredFolder":"/run/user",
"MonitoredSubFolders":["data"],
"IgnoreExtentions":["tmp"],
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in
new file mode 100644
index 0000000000000000000000000000000000000000..85e5872e9029e5ff696225e11f520b9781ebc6af
--- /dev/null
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in
@@ -0,0 +1,127 @@
+job "asapo-filemon_batch" {
+ datacenters = [
+ "dc1"]
+
+ type = "batch"
+
+ group "producer-windows" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "windows"
+ }
+
+ count = 1
+ task "producer" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/asapo-eventmon-producer.exe"
+ args = [
+ "local/test.json"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@.exe"
+ mode = "file"
+ destination = "local/asapo-eventmon-producer.exe"
+ }
+
+ template {
+ data = <<EOH
+{
+ "AsapoEndpoint":"psana002:8400",
+ "Tag":"test_tag",
+ "BeamtimeID":"asapo_test1",
+ "Mode":"tcp",
+ "NThreads":8,
+ "LogLevel":"info",
+ "RootMonitoredFolder":"u:\\asapo",
+ "MonitoredSubFolders":["test_folder"],
+ "IgnoreExtentions":["tmp"],
+ "RemoveAfterSend":true,
+ "Subset": {
+ "Mode":"batch",
+ "BatchSize": {{ keyOrDefault "monitor_batch_size" "3" }}
+ }
+}
+ EOH
+ destination = "local/test.json"
+ }
+ # resources {
+ # cpu = 5000
+ # memory = 128
+ # network {
+ # mbits = 10000
+ # }
+ } # producer task
+ }
+ #windows
+
+ group "producer-linux" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ constraint {
+ attribute = "${meta.location}"
+ value = "petra3"
+ }
+
+ count = 1
+
+ task "producer" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/asapo-eventmon-producer"
+ args = [
+ "local/test.json"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@"
+ mode = "file"
+ destination = "local/asapo-eventmon-producer"
+ }
+
+ template {
+ data = <<EOH
+{
+ "AsapoEndpoint":"psana002:8400",
+ "Tag":"test_tag",
+ "BeamtimeID":"asapo_test2",
+ "Mode":"tcp",
+ "NThreads":8,
+ "LogLevel":"info",
+ "RootMonitoredFolder":"/run/user",
+ "MonitoredSubFolders":["data"],
+ "IgnoreExtentions":["tmp"],
+ "RemoveAfterSend":true,
+ "Subset": {
+ "Mode":"batch",
+ "BatchSize": {{ keyOrDefault "monitor_batch_size" "3" }}
+ }
+
+}
+ EOH
+ destination = "local/test.json"
+ }
+
+ # resources {
+ # cpu = 5000
+ # memory = 128
+ # network {
+ # mbits = 10000
+ # }
+ # }
+
+ } # task producer
+ }
+ #linux
+}
+
+
+
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in
new file mode 100644
index 0000000000000000000000000000000000000000..2a6159367306c3d349492fd3d740dbdadc7d89b0
--- /dev/null
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in
@@ -0,0 +1,129 @@
+job "asapo-filemon_multisource" {
+ datacenters = [
+ "dc1"]
+
+ type = "batch"
+
+ group "producer-windows" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "windows"
+ }
+
+ count = 1
+ task "producer" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/asapo-eventmon-producer.exe"
+ args = [
+ "local/test.json"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@.exe"
+ mode = "file"
+ destination = "local/asapo-eventmon-producer.exe"
+ }
+
+ template {
+ data = <<EOH
+{
+ "AsapoEndpoint":"psana002:8400",
+ "Tag":"test_tag",
+ "BeamtimeID":"asapo_test",
+ "Mode":"tcp",
+ "NThreads":8,
+ "LogLevel":"info",
+ "RootMonitoredFolder":"u:\\asapo",
+ "MonitoredSubFolders":["test_folder"],
+ "IgnoreExtentions":["tmp"],
+ "RemoveAfterSend":true,
+ "Subset": {
+ "Mode":"multisource",
+ "SourceId": 1,
+ "NSources":2
+ }
+}
+ EOH
+ destination = "local/test.json"
+ }
+ # resources {
+ # cpu = 5000
+ # memory = 128
+ # network {
+ # mbits = 10000
+ # }
+ } # producer task
+ }
+ #windows
+
+ group "producer-linux" {
+
+ constraint {
+ attribute = "${attr.kernel.name}"
+ value = "linux"
+ }
+
+ constraint {
+ attribute = "${meta.location}"
+ value = "petra3"
+ }
+
+ count = 1
+
+ task "producer" {
+ driver = "raw_exec"
+
+ config {
+ command = "local/asapo-eventmon-producer"
+ args = [
+ "local/test.json"]
+ }
+
+ artifact {
+ source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@"
+ mode = "file"
+ destination = "local/asapo-eventmon-producer"
+ }
+
+ template {
+ data = <<EOH
+{
+ "AsapoEndpoint":"psana002:8400",
+ "Tag":"test_tag",
+ "BeamtimeID":"asapo_test",
+ "Mode":"tcp",
+ "NThreads":8,
+ "LogLevel":"info",
+ "RootMonitoredFolder":"/run/user",
+ "MonitoredSubFolders":["data"],
+ "IgnoreExtentions":["tmp"],
+ "RemoveAfterSend":true,
+ "Subset": {
+ "Mode":"multisource",
+ "SourceId": 2,
+ "NSources":2
+ }
+
+}
+ EOH
+ destination = "local/test.json"
+ }
+
+ # resources {
+ # cpu = 5000
+ # memory = 128
+ # network {
+ # mbits = 10000
+ # }
+ # }
+
+ } # task producer
+ }
+ #linux
+}
+
+
+
diff --git a/tests/manual/tests_via_nomad/run_test.sh b/tests/manual/tests_via_nomad/run_test.sh
deleted file mode 100755
index 0460ca5e6c5b8f45ffe9f822f5df7235a517a513..0000000000000000000000000000000000000000
--- a/tests/manual/tests_via_nomad/run_test.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-. ./clean_after_tests.sh
-nomad stop asapo-filemon-producer
-nomad run asapo-test_filemon_producer_toreceiver.nomad
-sleep 1
-nomad stop asapo-test
-nomad run asapo-test_filegen_filemon.nomad
-
diff --git a/tests/manual/tests_via_nomad/run_test_dummy_producer.sh b/tests/manual/tests_via_nomad/run_test_dummy_producer.sh
deleted file mode 100755
index dda2a1b3b5603b4df33af06a0a74277c7d1aac7c..0000000000000000000000000000000000000000
--- a/tests/manual/tests_via_nomad/run_test_dummy_producer.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-. ./clean_after_tests.sh
-nomad stop asapo-test
-nomad run asapo-test_dummy_producer_linux_noworker.nomad
-
diff --git a/tests/manual/tests_via_nomad/run_test_fullchain.sh b/tests/manual/tests_via_nomad/run_test_fullchain.sh
new file mode 100755
index 0000000000000000000000000000000000000000..9f842d002cbd45639a67c4501126051741e0fb81
--- /dev/null
+++ b/tests/manual/tests_via_nomad/run_test_fullchain.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+nomad stop asapo-filemon
+nomad stop asapo-filemon_batch
+nomad stop asapo-filemon_multisource
+nomad run asapo-test_filemon_batch.nomad
+sleep 1
+nomad stop asapo-test
+. ./clean_after_tests.sh
+
+nomad run asapo-test_filegen_worker.nomad
+
diff --git a/tests/manual/tests_via_nomad/run_test_fullchain_batch.sh b/tests/manual/tests_via_nomad/run_test_fullchain_batch.sh
new file mode 100755
index 0000000000000000000000000000000000000000..459aa9f665b846bd9b2d9aa079786081650bd552
--- /dev/null
+++ b/tests/manual/tests_via_nomad/run_test_fullchain_batch.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+nomad stop asapo-filemon
+nomad stop asapo-filemon_batch
+nomad stop asapo-filemon_multisource
+nomad run asapo-test_filemon_batch.nomad
+sleep 1
+nomad stop asapo-test
+. ./clean_after_tests.sh
+
+nomad run asapo-test_filegen_worker_batch.nomad
+
diff --git a/tests/manual/tests_via_nomad/run_test_fullchain_multisource.sh b/tests/manual/tests_via_nomad/run_test_fullchain_multisource.sh
new file mode 100755
index 0000000000000000000000000000000000000000..8a5a8687cc32dba70cd68321b2371168b70bbfe4
--- /dev/null
+++ b/tests/manual/tests_via_nomad/run_test_fullchain_multisource.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+nomad stop asapo-filemon
+nomad stop asapo-filemon_batch
+nomad stop asapo-filemon_multisource
+nomad run asapo-test_filemon_multisource.nomad
+sleep 1
+nomad stop asapo-test
+. ./clean_after_tests.sh
+
+nomad run asapo-test_filegen_worker_1M_multisource.nomad
+
diff --git a/tests/manual/tests_via_nomad/run_test_linuxonly.sh b/tests/manual/tests_via_nomad/run_test_linuxonly.sh
deleted file mode 100755
index 0397935522e74561609a497efbf7bd7f7f803698..0000000000000000000000000000000000000000
--- a/tests/manual/tests_via_nomad/run_test_linuxonly.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-. ./clean_after_tests.sh
-nomad stop asapo-filemon-producer
-nomad run asapo-test_filemon_producer_toreceiver.nomad
-sleep 1
-nomad stop asapo-test
-nomad run asapo-test_filegen_filemon_linuxonly.nomad
-