Merge pull request #58 in ASAPO/asapo from feature_virtualized-deployment to develop

* commit '31324b4e':
  use infiniband ip for receiver data server
  fix network error code
  advertise ip for receiver, update config files
  fix thread unsafe calls in io
  start fluentd after elasticsearch
  change fluentd config
  increase timeout for get_next, exit on any error, update deploy scripts
  add user vars file, variables for n of receivers and brokers
  increase timout for tests
  fix
  flag to use ip over ib, flag to set lightweight service nodes
  more work on deployment
  started deploying on Maxwell

tests/automatic/full_chain/two_streams/check_windows.bat

@@ -31,11 +31,11 @@ start /B "" "%1" %proxy_address% %beamtime_id%%%%stream2% 100 900 4 0 100
 ping 1.0.0.0 -n 1 -w 100 > nul
 
 REM consumer
-"%2" %proxy_address% %receiver_folder% %beamtime_id%%%%stream1% 2 %token% 10000  0 > out1.txt
+"%2" %proxy_address% %receiver_folder% %beamtime_id%%%%stream1% 2 %token% 12000  0 > out1.txt
 type out1.txt
 findstr /i /l /c:"Processed 1000 file(s)"  out1.txt || goto :error
 
-"%2" %proxy_address% %receiver_folder% %beamtime_id%%%%stream2% 2 %token% 10000  0 > out2.txt
+"%2" %proxy_address% %receiver_folder% %beamtime_id%%%%stream2% 2 %token% 12000  0 > out2.txt
 type out2.txt
 findstr /i /l /c:"Processed 900 file(s)"  out2.txt || goto :error
 

tests/automatic/mongo_db/insert_dataset/insert_dataset_mongodb.cpp

@@ -46,7 +46,7 @@ int main(int argc, char* argv[]) {
     fi.source = "host:1234";
     fi.id = 1;
 
-    uint64_t subset_size=2;
+    uint64_t subset_size = 2;
 
     if (args.keyword != "Notconnected") {
         db.Connect("127.0.0.1", "data", "test");

tests/automatic/settings/receiver.json.tpl.lin.in

 {
+  "AdvertiseIP": "127.0.0.1",
   "PerformanceDbServer":"localhost:8086",
   "PerformanceDbName": "db_test",
   "DatabaseServer":"auto",

tests/automatic/settings/receiver.json.tpl.win.in

 {
+  "AdvertiseIP": "127.0.0.1",
   "PerformanceDbServer":"localhost:8086",
   "PerformanceDbName": "db_test",
   "DatabaseServer":"auto",

tests/automatic/system_io/ip_tcp_network/client_serv/ip_tcp_network.cpp

@@ -16,7 +16,8 @@ using asapo::M_AssertEq;
 
 using namespace std::chrono;
 
-static const std::unique_ptr<asapo::IO> io(asapo::GenerateDefaultIO());
+static const std::unique_ptr<asapo::IO>
+    io(asapo::GenerateDefaultIO());
 static const std::string kListenAddress = "127.0.0.1:60123";
 static std::promise<void> kThreadStarted;
 static const int kNumberOfChecks = 2;
@@ -158,7 +159,7 @@ int main(int argc, char* argv[]) {
 
     std::cout << "[META] Check unknown host" << std::endl;
     io->CreateAndConnectIPTCPSocket("some-host-that-might-not-exists.aa:1234", &err);
-    if(asapo::IOErrorTemplates::kUnableToResolveHostname != err) {
+    if(err != asapo::IOErrorTemplates::kAddressNotValid) {
         ExitIfErrIsNotOk(&err, 303);
     }
 

tests/manual/python_tests/producer/receiver.json.tpl

 {
+  "AdvertiseIP": "127.0.0.1",
   "PerformanceDbServer":"localhost:8086",
   "PerformanceDbName": "db_test",
   "DatabaseServer":"localhost:27017",

tests/manual/receiver_debug_local/discovery.json

+{
+  "Mode": "static",
+  "Receiver": {
+    "StaticEndpoints": [
+      "localhost:22001"
+    ],
+    "MaxConnections": 32
+  },
+  "Broker": {
+    "StaticEndpoint": "localhost:5005"
+  },
+  "Mongo": {
+    "StaticEndpoint": "127.0.0.1:27017"
+  },
+  "Port": 5900,
+  "LogLevel":"debug"
+}
+
+

tests/manual/receiver_debug_local/discovery.json.tpl

@@ -9,6 +9,9 @@
   "Broker": {
     "StaticEndpoint": "localhost:5005"
   },
+  "Mongo": {
+    "StaticEndpoint": "127.0.0.1:27017"
+  },
   "Port": {{ env "NOMAD_PORT_discovery" }},
   "LogLevel":"debug"
 }

tests/manual/receiver_debug_local/nginx_kill.nmd

+job "nginx_kill" {
+  datacenters = ["dc1"]
+
+  type = "batch"
+
+  group "nginx_kill" {
+    count = 1
+
+    task "nginx_kill" {
+      driver = "raw_exec"
+      config {
+        command = "killall",
+        args =  ["nginx"]
+      }
+   }
+  }
+}

tests/manual/receiver_debug_local/start_dummyproducer.sh

@@ -2,13 +2,13 @@
 
 beamtime_id=asapo_test
 
-nfiles=10
+nfiles=1000
 timeout=100
-fsize=100
-mode=0 #tcp
-nthreads=4
+fsize=10000
+mode=10 #tcp & no file write
+nthreads=32
 
 exec=/home/yakubov/projects/asapo/cmake-build-debug/examples/producer/dummy-data-producer/dummy-data-producer
 
 
-$exec localhost:8400 ${beamtime_id} $fsize $nfiles  $nthreads $mode  $timeout
\ No newline at end of file
+$exec localhost:8400 ${beamtime_id} $fsize $nfiles  $nthreads $mode  $timeout

tests/manual/receiver_debug_local/start_receiver.sh

+#!/usr/bin/env bash
+
+
+exec=/home/yakubov/projects/asapo/cmake-build-debug/receiver/receiver
+
+
+$exec receiver.json