update nomad tests

tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_batch.nomad.in

+job "asapo-test" {
+  datacenters = [
+    "dc1"]
+
+  type = "batch"
+
+  group "filegen-windows" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "windows"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_win.exe"
+        args = [
+          "1",
+          "1M",
+          "100000",
+          "1100",
+          "u:/asapo/test_folder/file_win"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_win.exe"
+        mode = "file"
+        destination = "local/filegen_win.exe"
+      }
+    }
+
+
+  }
+  #windows
+
+  group "filegen-linux" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    constraint {
+      attribute = "${meta.location}"
+      value = "petra3"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_linux"
+        args = [
+          "1",
+          "1M",
+          "100000",
+          "1200",
+          "/run/user/data/file_lin"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_linux"
+        mode = "file"
+        destination = "local/filegen_linux"
+      }
+    }
+
+  }
+
+
+  group "worker-linux1" {
+
+    restart {
+      attempts = 0
+    }
+
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 1
+
+    task "worker-linux" {
+      driver = "raw_exec"
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test1/asapo_test1",
+          "asapo_test1",
+          "16",
+          "oTsKsj8i6WcW_gVzeIFvZCtSfMErjDELJEyAI23n7Ik=",
+          "30000",
+          "${WORKER_READ_META_ONLY}",
+          "1"]
+      }
+
+#      resources {
+#        cpu = 5000
+#        memory = 128
+#        network {
+#          mbits = 10000
+#        }
+#      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker-linux1
+
+
+  group "worker-linux2" {
+
+    restart {
+      attempts = 0
+    }
+
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 1
+
+    task "worker-linux" {
+      driver = "raw_exec"
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test2/asapo_test2",
+          "asapo_test2",
+          "16",
+          "yzgAcLmijSLWIm8dBiGNCbc0i42u5HSm-zR6FRqo__Y=",
+          "30000",
+          "${WORKER_READ_META_ONLY}",
+          "1"]
+      }
+#      resources {
+#        cpu = 5000
+#        memory = 128
+#        network {
+#          mbits = 10000
+#        }
+#      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker-linux2
+
+}

tests/manual/tests_via_nomad/asapo-test_filegen_worker_1M_multisource.nomad.in

+job "asapo-test" {
+  datacenters = [
+    "dc1"]
+
+  type = "batch"
+
+  group "filegen-windows" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "windows"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_win.exe"
+        args = [
+          "1",
+          "1M",
+          "100000",
+          "1100",
+          "u:/asapo/test_folder/file_win"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_win.exe"
+        mode = "file"
+        destination = "local/filegen_win.exe"
+      }
+    }
+
+
+  }
+  #windows
+
+  group "filegen-linux" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    constraint {
+      attribute = "${meta.location}"
+      value = "petra3"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_linux"
+        args = [
+          "1",
+          "1M",
+          "100000",
+          "1200",
+          "/run/user/data/file_lin"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_linux"
+        mode = "file"
+        destination = "local/filegen_linux"
+      }
+    }
+
+  }
+
+
+  group "worker" {
+
+    restart {
+      attempts = 0
+    }
+
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 1
+
+    task "worker-linux" {
+      driver = "raw_exec"
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test1/asapo_test1",
+          "asapo_test",
+          "16",
+          "KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc=",
+          "30000",
+          "${WORKER_READ_META_ONLY}",
+          "1"]
+      }
+
+#      resources {
+#        cpu = 5000
+#        memory = 128
+#        network {
+#          mbits = 10000
+#        }
+#      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker
+
+}

tests/manual/tests_via_nomad/asapo-test_filegen_worker_batch.nomad.in

+job "asapo-test" {
+  datacenters = [
+    "dc1"]
+
+  type = "batch"
+
+  group "filegen-windows" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "windows"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_win.exe"
+        args = [
+          "1",
+          "10M",
+          "10000",
+          "120",
+          "u:/asapo/test_folder/file_win"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_win.exe"
+        mode = "file"
+        destination = "local/filegen_win.exe"
+      }
+    }
+
+
+  }
+  #windows
+
+  group "filegen-linux" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    constraint {
+      attribute = "${meta.location}"
+      value = "petra3"
+    }
+
+    count = 1
+
+    task "filegen" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/filegen_linux"
+        args = [
+          "1",
+          "10M",
+          "10000",
+          "120",
+          "/run/user/data/file_lin"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/filegen_linux"
+        mode = "file"
+        destination = "local/filegen_linux"
+      }
+    }
+
+  }
+
+
+  group "worker-linux1" {
+
+    restart {
+      attempts = 0
+    }
+
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 1
+
+    task "worker-linux" {
+      driver = "raw_exec"
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test1/asapo_test1",
+          "asapo_test1",
+          "16",
+          "oTsKsj8i6WcW_gVzeIFvZCtSfMErjDELJEyAI23n7Ik=",
+          "30000",
+          "${WORKER_READ_META_ONLY}",
+          "1"]
+      }
+
+#      resources {
+#        cpu = 5000
+#        memory = 128
+#        network {
+#          mbits = 10000
+#        }
+#      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker-linux1
+
+
+  group "worker-linux2" {
+
+    restart {
+      attempts = 0
+    }
+
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    #    constraint {
+    #      attribute = "${meta.location}"
+    #      operator = "!="
+    #      value = "petra3"
+    #    }
+
+    count = 1
+
+    task "worker-linux" {
+      driver = "raw_exec"
+    template {
+     data = <<EOH
+        WORKER_READ_META_ONLY = "{{ keyOrDefault "worker_read_meta_only" "1" }}"
+        EOH
+     destination = "secrets/file.env"
+     env         = true
+    }
+
+      config {
+        command = "local/getnext_broker"
+        args = [
+          "psana002:8400",
+          "/bldocuments/support/asapo/data/test2/asapo_test2",
+          "asapo_test2",
+          "16",
+          "yzgAcLmijSLWIm8dBiGNCbc0i42u5HSm-zR6FRqo__Y=",
+          "30000",
+          "${WORKER_READ_META_ONLY}",
+          "1"]
+      }
+#      resources {
+#        cpu = 5000
+#        memory = 128
+#        network {
+#          mbits = 10000
+#        }
+#      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/getnext_broker"
+      }
+    }
+
+  }
+  # worker-linux2
+
+}

tests/manual/tests_via_nomad/asapo-test_filemon.nomad.in

@@ -35,7 +35,7 @@ job "asapo-filemon" {
  "BeamtimeID":"asapo_test1",
  "Mode":"tcp",
  "NThreads":8,
- "LogLevel":"debug",
+ "LogLevel":"info",
  "RootMonitoredFolder":"u:\\asapo",
  "MonitoredSubFolders":["test_folder"],
  "IgnoreExtentions":["tmp"],
@@ -94,7 +94,7 @@ job "asapo-filemon" {
  "BeamtimeID":"asapo_test2",
  "Mode":"tcp",
  "NThreads":8,
- "LogLevel":"debug",
+ "LogLevel":"info",
  "RootMonitoredFolder":"/run/user",
  "MonitoredSubFolders":["data"],
  "IgnoreExtentions":["tmp"],

tests/manual/tests_via_nomad/asapo-test_filemon_batch.nomad.in

-job "asapo-filemon" {
+job "asapo-filemon_batch" {
   datacenters = [
     "dc1"]
 
@@ -35,7 +35,7 @@ job "asapo-filemon" {
  "BeamtimeID":"asapo_test1",
  "Mode":"tcp",
  "NThreads":8,
- "LogLevel":"debug",
+ "LogLevel":"info",
  "RootMonitoredFolder":"u:\\asapo",
  "MonitoredSubFolders":["test_folder"],
  "IgnoreExtentions":["tmp"],
@@ -95,7 +95,7 @@ job "asapo-filemon" {
  "BeamtimeID":"asapo_test2",
  "Mode":"tcp",
  "NThreads":8,
- "LogLevel":"debug",
+ "LogLevel":"info",
  "RootMonitoredFolder":"/run/user",
  "MonitoredSubFolders":["data"],
  "IgnoreExtentions":["tmp"],

tests/manual/tests_via_nomad/asapo-test_filemon_multisource.nomad.in

+job "asapo-filemon_multisource" {
+  datacenters = [
+    "dc1"]
+
+  type = "batch"
+
+  group "producer-windows" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "windows"
+    }
+
+    count = 1
+    task "producer" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/asapo-eventmon-producer.exe"
+        args = [
+          "local/test.json"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@.exe"
+        mode = "file"
+        destination = "local/asapo-eventmon-producer.exe"
+      }
+
+      template {
+        data = <<EOH
+{
+ "AsapoEndpoint":"psana002:8400",
+ "Tag":"test_tag",
+ "BeamtimeID":"asapo_test",
+ "Mode":"tcp",
+ "NThreads":8,
+ "LogLevel":"info",
+ "RootMonitoredFolder":"u:\\asapo",
+ "MonitoredSubFolders":["test_folder"],
+ "IgnoreExtentions":["tmp"],
+ "RemoveAfterSend":true,
+ "Subset": {
+  	"Mode":"multisource",
+  	"SourceId": 1,
+  	"NSources":2
+ }
+}
+        EOH
+        destination = "local/test.json"
+      }
+      #      resources {
+      #        cpu = 5000
+      #        memory = 128
+      #        network {
+      #          mbits = 10000
+      #        }
+    } # producer task
+  }
+  #windows
+
+  group "producer-linux" {
+
+    constraint {
+      attribute = "${attr.kernel.name}"
+      value = "linux"
+    }
+
+    constraint {
+      attribute = "${meta.location}"
+      value = "petra3"
+    }
+
+    count = 1
+
+    task "producer" {
+      driver = "raw_exec"
+
+      config {
+        command = "local/asapo-eventmon-producer"
+        args = [
+          "local/test.json"]
+      }
+
+      artifact {
+        source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@"
+        mode = "file"
+        destination = "local/asapo-eventmon-producer"
+      }
+
+      template {
+        data = <<EOH
+{
+ "AsapoEndpoint":"psana002:8400",
+ "Tag":"test_tag",
+ "BeamtimeID":"asapo_test",
+ "Mode":"tcp",
+ "NThreads":8,
+ "LogLevel":"info",
+ "RootMonitoredFolder":"/run/user",
+ "MonitoredSubFolders":["data"],
+ "IgnoreExtentions":["tmp"],
+ "RemoveAfterSend":true,
+ "Subset": {
+  	"Mode":"multisource",
+  	"SourceId": 2,
+  	"NSources":2
+ }
+
+}
+        EOH
+        destination = "local/test.json"
+      }
+
+      #      resources {
+      #        cpu = 5000
+      #        memory = 128
+      #        network {
+      #          mbits = 10000
+      #        }
+      #      }
+
+    } # task producer
+  }
+  #linux
+}
+
+
+

tests/manual/tests_via_nomad/run_test.sh

-. ./clean_after_tests.sh
-nomad stop asapo-filemon-producer
-nomad run asapo-test_filemon_producer_toreceiver.nomad
-sleep 1
-nomad stop asapo-test
-nomad run asapo-test_filegen_filemon.nomad
-

tests/manual/tests_via_nomad/run_test_dummy_producer.sh

-. ./clean_after_tests.sh
-nomad stop asapo-test
-nomad run  asapo-test_dummy_producer_linux_noworker.nomad
-

tests/manual/tests_via_nomad/run_test_fullchain.sh

+#!/usr/bin/env bash
+
+nomad stop asapo-filemon
+nomad stop asapo-filemon_batch
+nomad stop asapo-filemon_multisource
+nomad run asapo-test_filemon_batch.nomad
+sleep 1
+nomad stop asapo-test
+. ./clean_after_tests.sh
+
+nomad run asapo-test_filegen_worker.nomad
+

tests/manual/tests_via_nomad/run_test_fullchain_batch.sh

+#!/usr/bin/env bash
+
+nomad stop asapo-filemon
+nomad stop asapo-filemon_batch
+nomad stop asapo-filemon_multisource
+nomad run asapo-test_filemon_batch.nomad
+sleep 1
+nomad stop asapo-test
+. ./clean_after_tests.sh
+
+nomad run asapo-test_filegen_worker_batch.nomad
+

tests/manual/tests_via_nomad/run_test_fullchain_multisource.sh

+#!/usr/bin/env bash
+
+nomad stop asapo-filemon
+nomad stop asapo-filemon_batch
+nomad stop asapo-filemon_multisource
+nomad run asapo-test_filemon_multisource.nomad
+sleep 1
+nomad stop asapo-test
+. ./clean_after_tests.sh
+
+nomad run asapo-test_filegen_worker_1M_multisource.nomad
+

tests/manual/tests_via_nomad/run_test_linuxonly.sh

-. ./clean_after_tests.sh
-nomad stop asapo-filemon-producer
-nomad run asapo-test_filemon_producer_toreceiver.nomad
-sleep 1
-nomad stop asapo-test
-nomad run asapo-test_filegen_filemon_linuxonly.nomad
-