Merge pull request #41 in ASAPO/asapo from feature_datasets to develop

* commit 'cf6ce9d6':
  update nomad tests
  update getlat test for datasets
  producer only nomad
  updated nomad tests

tests/manual/performance_broker_receiver/getlast_broker.cpp

@@ -6,17 +6,16 @@
 #include <chrono>
 #include <iomanip>
 #include <numeric>
+#include <mutex>
 
 #include "asapo_worker.h"
 
-#include <mutex>
+using std::chrono::system_clock;
+using asapo::Error;
 
 std::string group_id = "";
 std::mutex lock;
 
-using std::chrono::system_clock;
-using asapo::Error;
-
 struct Params {
     std::string server;
     std::string file_path;
@@ -25,6 +24,7 @@ struct Params {
     int timeout_ms;
     int nthreads;
     bool read_data;
+    bool datasets;
 };
 
 void WaitThreads(std::vector<std::thread>* threads) {
@@ -42,47 +42,57 @@ int ProcessError(const Error& err) {
 std::vector<std::thread> StartThreads(const Params& params,
                                       std::vector<int>* nfiles,
                                       std::vector<int>* errors,
-                                      std::vector<int>* nbuf) {
-    auto exec_next = [&params, nfiles, errors, nbuf](int i) {
-        asapo::FileInfo fi;
-        Error err;
-        auto broker = asapo::DataBrokerFactory::CreateServerBroker(params.server, params.file_path, params.beamtime_id,
-                      params.token, &err);
-        broker->SetTimeout(params.timeout_ms);
-        asapo::FileData data;
-
-        lock.lock();
-
-        if (group_id.empty()) {
-            group_id = broker->GenerateNewGroupId(&err);
-            if (err) {
-                (*errors)[i] += ProcessError(err);
-                return;
-            }
-        }
-
-        lock.unlock();
-
-        auto start = system_clock::now();
-        while (std::chrono::duration_cast<std::chrono::milliseconds>(system_clock::now() - start).count() <
-                params.timeout_ms) {
-            err = broker->GetLast(&fi, group_id, params.read_data ? &data : nullptr);
-            if (err == nullptr) {
-                (*nbuf)[i] += fi.buf_id == 0 ? 0 : 1;
-                if (params.read_data && (*nfiles)[i] < 10 && fi.size < 10) {
-                    if (data != nullptr) {
-                        data[9] = 0;
-                        std::cout << "Received: " << reinterpret_cast<char const*>(data.get()) << std::endl;
-                    }
-                }
-            } else {
-                (*errors)[i] += ProcessError(err);
-                if (err == asapo::IOErrorTemplates::kTimeout) {
-                    break;
-                }
-            }
-            (*nfiles)[i]++;
-        }
+                                      std::vector<int>* nbuf,
+                                      std::vector<int>* nfiles_total) {
+    auto exec_next = [&params, nfiles, errors, nbuf, nfiles_total](int i) {
+      asapo::FileInfo fi;
+      Error err;
+      auto broker = asapo::DataBrokerFactory::CreateServerBroker(params.server, params.file_path, params.beamtime_id,
+                                                                 params.token, &err);
+      broker->SetTimeout((uint64_t) params.timeout_ms);
+      asapo::FileData data;
+
+      lock.lock();
+
+      if (group_id.empty()) {
+          group_id = broker->GenerateNewGroupId(&err);
+          if (err) {
+              (*errors)[i] += ProcessError(err);
+              return;
+          }
+      }
+
+      lock.unlock();
+
+      auto start = system_clock::now();
+      while (std::chrono::duration_cast<std::chrono::milliseconds>(system_clock::now() - start).count() <
+          params.timeout_ms) {
+          if (params.datasets) {
+              auto dataset = broker->GetLastDataset(group_id, &err);
+              if (err == nullptr) {
+                  for (auto& fi : dataset.content) {
+                      (*nbuf)[i] += fi.buf_id == 0 ? 0 : 1;
+                      (*nfiles_total)[i]++;
+                  }
+              }
+          } else {
+              err = broker->GetLast(&fi, group_id, params.read_data ? &data : nullptr);
+              if (err == nullptr) {
+                  (*nbuf)[i] += fi.buf_id == 0 ? 0 : 1;
+                  if (params.read_data && (*nfiles)[i] < 10 && fi.size < 10) {
+                      data[9] = 0;
+                      std::cout << "Received: " << reinterpret_cast<char const*>(data.get()) << std::endl;
+                  }
+              }
+          }
+          if (err) {
+              (*errors)[i] += ProcessError(err);
+              if (err == asapo::IOErrorTemplates::kTimeout) {
+                  break;
+              }
+          }
+          (*nfiles)[i]++;
+      }
     };
 
     std::vector<std::thread> threads;
@@ -92,21 +102,22 @@ std::vector<std::thread> StartThreads(const Params& params,
     return threads;
 }
 
-int ReadAllData(const Params& params, uint64_t* duration_ms, int* nerrors, int* nbuf) {
+int ReadAllData(const Params& params, uint64_t* duration_ms, int* nerrors, int* nbuf, int* nfiles_total) {
     asapo::FileInfo fi;
     system_clock::time_point t1 = system_clock::now();
 
     std::vector<int> nfiles(params.nthreads, 0);
     std::vector<int> errors(params.nthreads, 0);
     std::vector<int> nfiles_frombuf(params.nthreads, 0);
+    std::vector<int> nfiles_total_in_datasets(params.nthreads, 0);
 
-    auto threads = StartThreads(params, &nfiles, &errors, &nfiles_frombuf);
+    auto threads = StartThreads(params, &nfiles, &errors, &nfiles_frombuf, &nfiles_total_in_datasets);
     WaitThreads(&threads);
 
     int n_total = std::accumulate(nfiles.begin(), nfiles.end(), 0);
     *nerrors = std::accumulate(errors.begin(), errors.end(), 0);
     *nbuf = std::accumulate(nfiles_frombuf.begin(), nfiles_frombuf.end(), 0);
-
+    *nfiles_total = std::accumulate(nfiles_total_in_datasets.begin(), nfiles_total_in_datasets.end(), 0);
 
     system_clock::time_point t2 = system_clock::now();
     auto duration_read = std::chrono::duration_cast<std::chrono::milliseconds>(t2 - t1);
@@ -116,14 +127,15 @@ int ReadAllData(const Params& params, uint64_t* duration_ms, int* nerrors, int*
 
 int main(int argc, char* argv[]) {
     asapo::ExitAfterPrintVersionIfNeeded("GetLast Broker Example", argc, argv);
-    if (argc != 8) {
+    Params params;
+    params.datasets = false;
+    if (argc != 8 && argc != 9) {
         std::cout << "Usage: " + std::string{argv[0]}
-                  + " <server> <files_path> <run_name> <nthreads> <token> <timeout ms> <metaonly>"
+            + " <server> <files_path> <run_name> <nthreads> <token> <timeout ms> <metaonly> [use datasets]"
                   <<
                   std::endl;
         exit(EXIT_FAILURE);
     }
-    Params params;
     params.server = std::string{argv[1]};
     params.file_path = std::string{argv[2]};
     params.beamtime_id = std::string{argv[3]};
@@ -131,14 +143,18 @@ int main(int argc, char* argv[]) {
     params.token = std::string{argv[5]};
     params.timeout_ms = atoi(argv[6]);
     params.read_data = atoi(argv[7]) != 1;
-
+    if (argc == 9) {
+        params.datasets = atoi(argv[8]) == 1;
+    }
     uint64_t duration_ms;
-    int nerrors, nbuf;
-    auto nfiles = ReadAllData(params, &duration_ms, &nerrors, &nbuf);
-
-    std::cout << "Processed " << nfiles << " file(s)" << std::endl;
+    int nerrors, nbuf, nfiles_total;
+    auto nfiles = ReadAllData(params, &duration_ms, &nerrors, &nbuf, &nfiles_total);
+    std::cout << "Processed " << nfiles << (params.datasets ? " dataset(s)" : " file(s)") << std::endl;
+    if (params.datasets) {
+        std::cout << "  with " << nfiles_total << " file(s)" << std::endl;
+    }
     std::cout << "Successfully: " << nfiles - nerrors << std::endl;
-    if (params.read_data) {
+    if (params.read_data && !params.datasets) {
         std::cout << "  from memory buffer: " << nbuf << std::endl;
         std::cout << "  from filesystem: " << nfiles - nerrors - nbuf << std::endl;
     }

tests/manual/tests_via_nomad/asapo-test_dummy_producer_only.nomad.in

+job "asapo-test" {
+  datacenters = [
+    "dc1"]
+
+  type = "batch"
+
+  group "producer-windows" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "windows"
+    }
+
+    count = 0
+    task "producer" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/dummy-data-producer.exe"
+        args = [
+          "psana002:8400",
+          "asapo_test1",
+          "100",
+          "1000000",
+          "8",
+          "0",
+          "1000"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/dummy-data-producer-@ASAPO_VERSION@.exe"
+        mode = "file"
+        destination = "local/dummy-data-producer.exe"
+      }
+
+      #      resources {
+      #        cpu = 5000
+      #        memory = 128
+      #        network {
+      #          mbits = 10000
+      #        }
+    }
+
+  }
+  #windows
+
+  group "producer-linux" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    constraint {
+      attribute = "${meta.location}"
+      value = "petra3"
+    }
+
+    count = 1
+
+    task "producer" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/dummy-data-producer"
+        args = [
+          "psana002:8400",
+          "asapo_test2",
+          "100",
+          "1000",
+          "8",
+          "0",
+          "1000",
+          "1000"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/dummy-data-producer-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/dummy-data-producer"
+      }
+
+      #      resources {
+      #        cpu = 5000
+      #        memory = 128
+      #        network {
+      #          mbits = 10000
+      #        }
+      #      }
+
+    }
+
+  }
+  #linux
+
+  group "worker-linux1" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 0
+
+
+    task "worker-linux" {
+      driver = "raw_exec"
+
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test1/asapo_test1",
+          "asapo_test1",
+          "16",
+          "oTsKsj8i6WcW_gVzeIFvZCtSfMErjDELJEyAI23n7Ik=",
+          "30000",
+          "${WORKER_READ_META_ONLY}"]
+      }
+
+      resources {
+        cpu = 5000
+        memory = 128
+        network {
+          mbits = 10000
+        }
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker-linux1
+
+
+  group "worker-linux2" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+
+	count = 0
+
+    task "worker-linux" {
+      driver = "raw_exec"
+
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test2/asapo_test2",
+          "asapo_test2",
+          "16",
+          "yzgAcLmijSLWIm8dBiGNCbc0i42u5HSm-zR6FRqo__Y=",
+          "30000",
+          "${WORKER_READ_META_ONLY}"]
+      }
+      resources {
+        cpu = 5000
+        memory = 128
+        network {
+          mbits = 10000
+        }
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+
+      }
+    }
+
+  }
+
+}
+# worker-linux2

tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_batch.nomad.in

+job "asapo-test" {
+  datacenters = [
+    "dc1"]
+
+  type = "batch"
+
+  group "filegen-windows" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "windows"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_win.exe"
+        args = [
+          "1",
+          "1M",
+          "100000",
+          "1100",
+          "u:/asapo/test_folder/file_win"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_win.exe"
+        mode = "file"
+        destination = "local/filegen_win.exe"
+      }
+    }
+
+
+  }
+  #windows
+
+  group "filegen-linux" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    constraint {
+      attribute = "${meta.location}"
+      value = "petra3"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_linux"
+        args = [
+          "1",
+          "1M",
+          "100000",
+          "1200",
+          "/run/user/data/file_lin"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_linux"
+        mode = "file"
+        destination = "local/filegen_linux"
+      }
+    }
+
+  }
+
+
+  group "worker-linux1" {
+
+    restart {
+      attempts = 0
+    }
+
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 1
+
+    task "worker-linux" {
+      driver = "raw_exec"
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test1/asapo_test1",
+          "asapo_test1",
+          "16",
+          "oTsKsj8i6WcW_gVzeIFvZCtSfMErjDELJEyAI23n7Ik=",
+          "30000",
+          "${WORKER_READ_META_ONLY}",
+          "1"]
+      }
+
+#      resources {
+#        cpu = 5000
+#        memory = 128
+#        network {
+#          mbits = 10000
+#        }
+#      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker-linux1
+
+
+  group "worker-linux2" {
+
+    restart {
+      attempts = 0
+    }
+
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 1
+
+    task "worker-linux" {
+      driver = "raw_exec"
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test2/asapo_test2",
+          "asapo_test2",
+          "16",
+          "yzgAcLmijSLWIm8dBiGNCbc0i42u5HSm-zR6FRqo__Y=",
+          "30000",
+          "${WORKER_READ_META_ONLY}",
+          "1"]
+      }
+#      resources {
+#        cpu = 5000
+#        memory = 128
+#        network {
+#          mbits = 10000
+#        }
+#      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker-linux2
+
+}

tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_multisource.nomad.in

+job "asapo-test" {
+  datacenters = [
+    "dc1"]
+
+  type = "batch"
+
+  group "filegen-windows" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "windows"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_win.exe"
+        args = [
+          "1",
+          "1M",
+          "100000",
+          "1100",
+          "u:/asapo/test_folder/file_win"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_win.exe"
+        mode = "file"
+        destination = "local/filegen_win.exe"
+      }
+    }
+
+
+  }
+  #windows
+
+  group "filegen-linux" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    constraint {
+      attribute = "${meta.location}"
+      value = "petra3"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_linux"
+        args = [
+          "1",
+          "1M",
+          "100000",
+          "1200",
+          "/run/user/data/file_lin"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_linux"
+        mode = "file"
+        destination = "local/filegen_linux"
+      }
+    }
+
+  }
+
+
+  group "worker" {
+
+    restart {
+      attempts = 0
+    }
+
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 1
+
+    task "worker-linux" {
+      driver = "raw_exec"
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test1/asapo_test1",
+          "asapo_test",
+          "16",
+          "KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc=",
+          "30000",
+          "${WORKER_READ_META_ONLY}",
+          "1"]
+      }
+
+#      resources {
+#        cpu = 5000
+#        memory = 128
+#        network {
+#          mbits = 10000
+#        }
+#      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker
+
+}

tests/manual/tests_via_nomad/asapo-test_filegen_worker_batch.nomad.in

+job "asapo-test" {
+  datacenters = [
+    "dc1"]
+
+  type = "batch"
+
+  group "filegen-windows" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "windows"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_win.exe"
+        args = [
+          "1",
+          "10M",
+          "10000",
+          "120",
+          "u:/asapo/test_folder/file_win"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_win.exe"
+        mode = "file"
+        destination = "local/filegen_win.exe"
+      }
+    }
+
+
+  }
+  #windows
+
+  group "filegen-linux" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    constraint {
+      attribute = "${meta.location}"
+      value = "petra3"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_linux"
+        args = [
+          "1",
+          "10M",
+          "10000",
+          "120",
+          "/run/user/data/file_lin"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_linux"
+        mode = "file"
+        destination = "local/filegen_linux"
+      }
+    }
+
+  }
+
+
+  group "worker-linux1" {
+
+    restart {
+      attempts = 0
+    }
+
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 1
+
+    task "worker-linux" {
+      driver = "raw_exec"
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test1/asapo_test1",
+          "asapo_test1",
+          "16",
+          "oTsKsj8i6WcW_gVzeIFvZCtSfMErjDELJEyAI23n7Ik=",
+          "30000",
+          "${WORKER_READ_META_ONLY}",
+          "1"]
+      }
+
+#      resources {
+#        cpu = 5000
+#        memory = 128
+#        network {
+#          mbits = 10000
+#        }
+#      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker-linux1
+
+
+  group "worker-linux2" {
+
+    restart {
+      attempts = 0
+    }
+
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 1
+
+    task "worker-linux" {
+      driver = "raw_exec"
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test2/asapo_test2",
+          "asapo_test2",
+          "16",
+          "yzgAcLmijSLWIm8dBiGNCbc0i42u5HSm-zR6FRqo__Y=",
+          "30000",
+          "${WORKER_READ_META_ONLY}",
+          "1"]
+      }
+#      resources {
+#        cpu = 5000
+#        memory = 128
+#        network {
+#          mbits = 10000
+#        }
+#      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker-linux2
+
+}

tests/manual/tests_via_nomad/asapo-test_filemon_producer_toreceiver.nomad.in → tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in

-job "asapo-filemon-producer" {
+job "asapo-filemon" {
   datacenters = [
     "dc1"]
 
@@ -35,7 +35,7 @@ job "asapo-filemon-producer" {
  "BeamtimeID":"asapo_test1",
  "Mode":"tcp",
  "NThreads":8,
- "LogLevel":"debug",
+ "LogLevel":"info",
  "RootMonitoredFolder":"u:\\asapo",
  "MonitoredSubFolders":["test_folder"],
  "IgnoreExtentions":["tmp"],
@@ -94,7 +94,7 @@ job "asapo-filemon-producer" {
  "BeamtimeID":"asapo_test2",
  "Mode":"tcp",
  "NThreads":8,
- "LogLevel":"debug",
+ "LogLevel":"info",
  "RootMonitoredFolder":"/run/user",
  "MonitoredSubFolders":["data"],
  "IgnoreExtentions":["tmp"],

tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in

+job "asapo-filemon_batch" {
+  datacenters = [
+    "dc1"]
+
+  type = "batch"
+
+  group "producer-windows" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "windows"
+    }
+
+    count = 1
+    task "producer" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/asapo-eventmon-producer.exe"
+        args = [
+          "local/test.json"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@.exe"
+        mode = "file"
+        destination = "local/asapo-eventmon-producer.exe"
+      }
+
+      template {
+        data = <<EOH
+{
+ "AsapoEndpoint":"psana002:8400",
+ "Tag":"test_tag",
+ "BeamtimeID":"asapo_test1",
+ "Mode":"tcp",
+ "NThreads":8,
+ "LogLevel":"info",
+ "RootMonitoredFolder":"u:\\asapo",
+ "MonitoredSubFolders":["test_folder"],
+ "IgnoreExtentions":["tmp"],
+ "RemoveAfterSend":true,
+ "Subset": {
+  	"Mode":"batch",
+  	"BatchSize": {{ keyOrDefault "monitor_batch_size" "3" }}
+ }
+}
+        EOH
+        destination = "local/test.json"
+      }
+      #      resources {
+      #        cpu = 5000
+      #        memory = 128
+      #        network {
+      #          mbits = 10000
+      #        }
+    } # producer task
+  }
+  #windows
+
+  group "producer-linux" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    constraint {
+      attribute = "${meta.location}"
+      value = "petra3"
+    }
+
+    count = 1
+
+    task "producer" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/asapo-eventmon-producer"
+        args = [
+          "local/test.json"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/asapo-eventmon-producer"
+      }
+
+      template {
+        data = <<EOH
+{
+ "AsapoEndpoint":"psana002:8400",
+ "Tag":"test_tag",
+ "BeamtimeID":"asapo_test2",
+ "Mode":"tcp",
+ "NThreads":8,
+ "LogLevel":"info",
+ "RootMonitoredFolder":"/run/user",
+ "MonitoredSubFolders":["data"],
+ "IgnoreExtentions":["tmp"],
+ "RemoveAfterSend":true,
+ "Subset": {
+  	"Mode":"batch",
+  	"BatchSize": {{ keyOrDefault "monitor_batch_size" "3" }}
+ }
+
+}
+        EOH
+        destination = "local/test.json"
+      }
+
+      #      resources {
+      #        cpu = 5000
+      #        memory = 128
+      #        network {
+      #          mbits = 10000
+      #        }
+      #      }
+
+    } # task producer
+  }
+  #linux
+}
+
+
+

tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in

+job "asapo-filemon_multisource" {
+  datacenters = [
+    "dc1"]
+
+  type = "batch"
+
+  group "producer-windows" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "windows"
+    }
+
+    count = 1
+    task "producer" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/asapo-eventmon-producer.exe"
+        args = [
+          "local/test.json"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@.exe"
+        mode = "file"
+        destination = "local/asapo-eventmon-producer.exe"
+      }
+
+      template {
+        data = <<EOH
+{
+ "AsapoEndpoint":"psana002:8400",
+ "Tag":"test_tag",
+ "BeamtimeID":"asapo_test",
+ "Mode":"tcp",
+ "NThreads":8,
+ "LogLevel":"info",
+ "RootMonitoredFolder":"u:\\asapo",
+ "MonitoredSubFolders":["test_folder"],
+ "IgnoreExtentions":["tmp"],
+ "RemoveAfterSend":true,
+ "Subset": {
+  	"Mode":"multisource",
+  	"SourceId": 1,
+  	"NSources":2
+ }
+}
+        EOH
+        destination = "local/test.json"
+      }
+      #      resources {
+      #        cpu = 5000
+      #        memory = 128
+      #        network {
+      #          mbits = 10000
+      #        }
+    } # producer task
+  }
+  #windows
+
+  group "producer-linux" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    constraint {
+      attribute = "${meta.location}"
+      value = "petra3"
+    }
+
+    count = 1
+
+    task "producer" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/asapo-eventmon-producer"
+        args = [
+          "local/test.json"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/asapo-eventmon-producer"
+      }
+
+      template {
+        data = <<EOH
+{
+ "AsapoEndpoint":"psana002:8400",
+ "Tag":"test_tag",
+ "BeamtimeID":"asapo_test",
+ "Mode":"tcp",
+ "NThreads":8,
+ "LogLevel":"info",
+ "RootMonitoredFolder":"/run/user",
+ "MonitoredSubFolders":["data"],
+ "IgnoreExtentions":["tmp"],
+ "RemoveAfterSend":true,
+ "Subset": {
+  	"Mode":"multisource",
+  	"SourceId": 2,
+  	"NSources":2
+ }
+
+}
+        EOH
+        destination = "local/test.json"
+      }
+
+      #      resources {
+      #        cpu = 5000
+      #        memory = 128
+      #        network {
+      #          mbits = 10000
+      #        }
+      #      }
+
+    } # task producer
+  }
+  #linux
+}
+
+
+

tests/manual/tests_via_nomad/run_test.sh

-. ./clean_after_tests.sh
-nomad stop asapo-filemon-producer
-nomad run asapo-test_filemon_producer_toreceiver.nomad
-sleep 1
-nomad stop asapo-test
-nomad run asapo-test_filegen_filemon.nomad
-

tests/manual/tests_via_nomad/run_test_dummy_producer.sh

-. ./clean_after_tests.sh
-nomad stop asapo-test
-nomad run  asapo-test_dummy_producer_linux_noworker.nomad
-

tests/manual/tests_via_nomad/run_test_fullchain.sh

+#!/usr/bin/env bash
+
+nomad stop asapo-filemon
+nomad stop asapo-filemon_batch
+nomad stop asapo-filemon_multisource
+nomad run asapo-test_filemon_batch.nomad
+sleep 1
+nomad stop asapo-test
+. ./clean_after_tests.sh
+
+nomad run asapo-test_filegen_worker.nomad
+

tests/manual/tests_via_nomad/run_test_fullchain_batch.sh

+#!/usr/bin/env bash
+
+nomad stop asapo-filemon
+nomad stop asapo-filemon_batch
+nomad stop asapo-filemon_multisource
+nomad run asapo-test_filemon_batch.nomad
+sleep 1
+nomad stop asapo-test
+. ./clean_after_tests.sh
+
+nomad run asapo-test_filegen_worker_batch.nomad
+

tests/manual/tests_via_nomad/run_test_fullchain_multisource.sh

+#!/usr/bin/env bash
+
+nomad stop asapo-filemon
+nomad stop asapo-filemon_batch
+nomad stop asapo-filemon_multisource
+nomad run asapo-test_filemon_multisource.nomad
+sleep 1
+nomad stop asapo-test
+. ./clean_after_tests.sh
+
+nomad run asapo-test_filegen_worker_1M_multisource.nomad
+

tests/manual/tests_via_nomad/run_test_linuxonly.sh

-. ./clean_after_tests.sh
-nomad stop asapo-filemon-producer
-nomad run asapo-test_filemon_producer_toreceiver.nomad
-sleep 1
-nomad stop asapo-test
-nomad run asapo-test_filegen_filemon_linuxonly.nomad
-