From 603ee94d61d6ab48c4cf87461721784724567db6 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Wed, 6 Mar 2019 12:31:58 +0100
Subject: [PATCH] update nomad config
---
CMakeLists.txt | 4 +--
CMakeModules/prepare_version.cmake | 10 +++++++
config/nomad/receiver.nmd.in | 1 +
deploy/nomad_jobs/asapo-brokers.nmd.in | 2 +-
deploy/nomad_jobs/asapo-receivers.nmd.in | 4 ++-
deploy/nomad_jobs/asapo-services.nmd.in | 4 +--
deploy/nomad_jobs/copy.sh.in | 3 +++
deploy/nomad_jobs/receiver.json.tpl | 4 +--
.../settings/receiver.json.tpl.lin.in | 2 +-
.../settings/receiver.json.tpl.win.in | 2 +-
.../asapo-test_dummy_producer.nomad | 8 +++---
.../asapo-test_filegen_filemon.nomad | 4 +--
.../asapo-test_filemon_producer_tolocal.nomad | 4 +--
...apo-test_filemon_producer_toreceiver.nomad | 4 +--
.../tests_via_nomad/clean_after_tests.sh | 27 +++++++++++++++++++
tests/manual/tests_via_nomad/run_test.sh | 7 +++++
.../run_test_dummy_producer.sh | 4 +++
.../tests_via_nomad/run_test_linuxonly.sh | 7 +++++
18 files changed, 81 insertions(+), 20 deletions(-)
create mode 100755 deploy/nomad_jobs/copy.sh.in
create mode 100755 tests/manual/tests_via_nomad/clean_after_tests.sh
create mode 100755 tests/manual/tests_via_nomad/run_test.sh
create mode 100755 tests/manual/tests_via_nomad/run_test_dummy_producer.sh
create mode 100755 tests/manual/tests_via_nomad/run_test_linuxonly.sh
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b1159c604..36ccfba2f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -47,11 +47,12 @@ message (STATUS "cURL include: ${CURL_INCLUDE_DIRS}")
# format sources
include(astyle)
+include(prepare_version)
+
include(testing_cpp)
include(prepare_asapo)
-
if(BUILD_WORKER_TOOLS)
set (BUILD_MONGODB_CLIENTLIB ON)
endif()
@@ -89,4 +90,3 @@ if(BUILD_DOCS)
include(doxygen)
endif()
-include(prepare_version)
\ No newline at end of file
diff --git a/CMakeModules/prepare_version.cmake b/CMakeModules/prepare_version.cmake
index f0704fcad..901ef4c10 100644
--- a/CMakeModules/prepare_version.cmake
+++ b/CMakeModules/prepare_version.cmake
@@ -1,6 +1,16 @@
execute_process(COMMAND git describe --tags --dirty OUTPUT_VARIABLE VERSION)
string(STRIP ${VERSION} VERSION)
+
+execute_process(COMMAND git rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE BRANCH)
+string(STRIP ${BRANCH} BRANCH)
+
+if (${BRANCH} STREQUAL "develop")
+ SET (ASAPO_VERSION ${BRANCH}.${VERSION})
+else()
+ SET (ASAPO_VERSION ${BRANCH}.latest)
+endif()
+
string(TIMESTAMP TIMESTAMP "%H:%M:%S %d.%m.%Y UTC" UTC)
configure_file(${PROJECT_SOURCE_DIR}/common/cpp/include/common/version.h.in ${PROJECT_SOURCE_DIR}/common/cpp/include/common/version.h @ONLY)
diff --git a/config/nomad/receiver.nmd.in b/config/nomad/receiver.nmd.in
index a27372d18..282208f29 100644
--- a/config/nomad/receiver.nmd.in
+++ b/config/nomad/receiver.nmd.in
@@ -19,6 +19,7 @@ job "receiver" {
memory = 256 # 256MB
network {
port "recv" {}
+ port "recv_ds" {}
}
}
diff --git a/deploy/nomad_jobs/asapo-brokers.nmd.in b/deploy/nomad_jobs/asapo-brokers.nmd.in
index 4c5a3040d..c3080085d 100644
--- a/deploy/nomad_jobs/asapo-brokers.nmd.in
+++ b/deploy/nomad_jobs/asapo-brokers.nmd.in
@@ -24,7 +24,7 @@ job "asapo-brokers" {
config {
network_mode = "host"
dns_servers = ["127.0.0.1"]
- image = "yakser/asapo-broker:develop"
+ image = "yakser/asapo-broker:@ASAPO_VERSION@"
force_pull = true
volumes = ["local/config.json:/var/lib/broker/config.json"]
logging {
diff --git a/deploy/nomad_jobs/asapo-receivers.nmd.in b/deploy/nomad_jobs/asapo-receivers.nmd.in
index 4159d9791..65f433215 100644
--- a/deploy/nomad_jobs/asapo-receivers.nmd.in
+++ b/deploy/nomad_jobs/asapo-receivers.nmd.in
@@ -24,7 +24,7 @@ job "asapo-receivers" {
config {
network_mode = "host"
dns_servers = ["127.0.0.1"]
- image = "yakser/asapo-receiver:develop"
+ image = "yakser/asapo-receiver:@ASAPO_VERSION@"
force_pull = true
volumes = ["local/config.json:/var/lib/receiver/config.json",
"/bldocuments/support/asapo/data:/var/lib/receiver/data"]
@@ -40,7 +40,9 @@ job "asapo-receivers" {
resources {
network {
port "recv" {}
+ port "recv_ds" {}
}
+ memory = 40000
}
service {
diff --git a/deploy/nomad_jobs/asapo-services.nmd.in b/deploy/nomad_jobs/asapo-services.nmd.in
index 3be75045c..b8d4bdd2a 100644
--- a/deploy/nomad_jobs/asapo-services.nmd.in
+++ b/deploy/nomad_jobs/asapo-services.nmd.in
@@ -12,7 +12,7 @@ job "asapo-services" {
config {
network_mode = "host"
dns_servers = ["127.0.0.1"]
- image = "yakser/asapo-authorizer:develop"
+ image = "yakser/asapo-authorizer:@ASAPO_VERSION@"
force_pull = true
volumes = ["local/config.json:/var/lib/authorizer/config.json",
"/bldocuments/support/asapo/beamtime_beamline_mapping.txt:/var/lib/authorizer/beamtime_beamline_mapping.txt",
@@ -71,7 +71,7 @@ job "asapo-services" {
config {
network_mode = "host"
dns_servers = ["127.0.0.1"]
- image = "yakser/asapo-discovery:develop"
+ image = "yakser/asapo-discovery:@ASAPO_VERSION@"
force_pull = true
volumes = ["local/config.json:/var/lib/discovery/config.json"]
logging {
diff --git a/deploy/nomad_jobs/copy.sh.in b/deploy/nomad_jobs/copy.sh.in
new file mode 100755
index 000000000..d6809604a
--- /dev/null
+++ b/deploy/nomad_jobs/copy.sh.in
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+scp * psana002:@NOMAD_INSTALL@/
diff --git a/deploy/nomad_jobs/receiver.json.tpl b/deploy/nomad_jobs/receiver.json.tpl
index b5021a78c..3795e8561 100644
--- a/deploy/nomad_jobs/receiver.json.tpl
+++ b/deploy/nomad_jobs/receiver.json.tpl
@@ -7,11 +7,11 @@
"ListenPort": {{ env "NOMAD_PORT_recv" }},
"DataServer": {
"NThreads": 2,
- "ListenPort": 23123
+ "ListenPort": {{ env "NOMAD_PORT_recv_ds" }}
},
"DataCache": {
"Use": true,
- "SizeGB": 1,
+ "SizeGB": 30,
"ReservedShare": 10
},
"Tag": "{{ env "NOMAD_ADDR_recv" }}",
diff --git a/tests/automatic/settings/receiver.json.tpl.lin.in b/tests/automatic/settings/receiver.json.tpl.lin.in
index e662a64df..cfe057742 100644
--- a/tests/automatic/settings/receiver.json.tpl.lin.in
+++ b/tests/automatic/settings/receiver.json.tpl.lin.in
@@ -4,7 +4,7 @@
"BrokerDbAddress":"localhost:27017",
"DataServer": {
"NThreads": 2,
- "ListenPort": 23123
+ "ListenPort": {{ env "NOMAD_PORT_recv_ds" }}
},
"DataCache": {
"Use": @RECEIVER_USE_CACHE@,
diff --git a/tests/automatic/settings/receiver.json.tpl.win.in b/tests/automatic/settings/receiver.json.tpl.win.in
index 511c2cda7..39ad1b7c1 100644
--- a/tests/automatic/settings/receiver.json.tpl.win.in
+++ b/tests/automatic/settings/receiver.json.tpl.win.in
@@ -7,7 +7,7 @@
"ListenPort": {{ env "NOMAD_PORT_recv" }},
"DataServer": {
"NThreads": 2,
- "ListenPort": 23123
+ "ListenPort": {{ env "NOMAD_PORT_recv_ds" }}
},
"DataCache": {
"Use": @RECEIVER_USE_CACHE@,
diff --git a/tests/manual/tests_via_nomad/asapo-test_dummy_producer.nomad b/tests/manual/tests_via_nomad/asapo-test_dummy_producer.nomad
index 2a4f017ee..20b6ca2fc 100644
--- a/tests/manual/tests_via_nomad/asapo-test_dummy_producer.nomad
+++ b/tests/manual/tests_via_nomad/asapo-test_dummy_producer.nomad
@@ -28,7 +28,7 @@ job "asapo-test" {
}
artifact {
- source = "http://nims.desy.de/extra/asapo/dummy-data-producer.exe"
+ source = "http://nims.desy.de/extra/asapo/dummy-data-producer-@ASAPO_VERSION@.exe"
mode = "file"
destination = "local/dummy-data-producer.exe"
@@ -75,7 +75,7 @@ job "asapo-test" {
}
artifact {
- source = "http://nims.desy.de/extra/asapo/dummy-data-producer"
+ source = "http://nims.desy.de/extra/asapo/dummy-data-producer-@ASAPO_VERSION@"
}
# resources {
@@ -128,7 +128,7 @@ job "asapo-test" {
}
artifact {
- source = "http://nims.desy.de/extra/asapo/getnext_broker"
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
}
}
@@ -172,7 +172,7 @@ job "asapo-test" {
}
artifact {
- source = "http://nims.desy.de/extra/asapo/getnext_broker"
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
}
}
diff --git a/tests/manual/tests_via_nomad/asapo-test_filegen_filemon.nomad b/tests/manual/tests_via_nomad/asapo-test_filegen_filemon.nomad
index 00dedc361..bcc7a9344 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filegen_filemon.nomad
+++ b/tests/manual/tests_via_nomad/asapo-test_filegen_filemon.nomad
@@ -111,7 +111,7 @@ job "asapo-test" {
# }
artifact {
- source = "http://nims.desy.de/extra/asapo/getnext_broker"
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
}
}
@@ -155,7 +155,7 @@ job "asapo-test" {
# }
artifact {
- source = "http://nims.desy.de/extra/asapo/getnext_broker"
+ source = "http://nims.desy.de/extra/asapo/getnext_broker-@ASAPO_VERSION@"
}
}
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad
index 3ac5548e8..ed3c4f477 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_tolocal.nomad
@@ -22,7 +22,7 @@ job "asapo-produceronly" {
}
artifact {
- source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer.exe"
+ source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@.exe"
mode = "file"
destination = "local/asapo-eventmon-producer.exe"
}
@@ -78,7 +78,7 @@ job "asapo-produceronly" {
}
artifact {
- source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer"
+ source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@"
mode = "file"
destination = "local/asapo-eventmon-producer"
}
diff --git a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_toreceiver.nomad b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_toreceiver.nomad
index b4356226d..a716b6384 100644
--- a/tests/manual/tests_via_nomad/asapo-test_filemon_producer_toreceiver.nomad
+++ b/tests/manual/tests_via_nomad/asapo-test_filemon_producer_toreceiver.nomad
@@ -22,7 +22,7 @@ job "asapo-filemon-producer" {
}
artifact {
- source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer.exe"
+ source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@.exe"
mode = "file"
destination = "local/asapo-eventmon-producer.exe"
}
@@ -78,7 +78,7 @@ job "asapo-filemon-producer" {
}
artifact {
- source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer"
+ source = "http://nims.desy.de/extra/asapo/asapo-eventmon-producer-@ASAPO_VERSION@"
mode = "file"
destination = "local/asapo-eventmon-producer"
}
diff --git a/tests/manual/tests_via_nomad/clean_after_tests.sh b/tests/manual/tests_via_nomad/clean_after_tests.sh
new file mode 100755
index 000000000..1eeed43f3
--- /dev/null
+++ b/tests/manual/tests_via_nomad/clean_after_tests.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+mongoaddr=`dig +short @127.0.0.1 mongo.service.asapo | head -1`
+
+database="asapo_test"
+
+cont=$(ssh $mongoaddr bash -c "'docker ps | grep mongo'")
+
+image=`echo "${cont##* }"`
+
+rm -rf /bldocuments/support/asapo/data/test/asapo_test
+rm -rf /bldocuments/support/asapo/data/test1/asapo_test1
+rm -rf /bldocuments/support/asapo/data/test2/asapo_test2
+
+mkdir /bldocuments/support/asapo/data/test/asapo_test
+mkdir /bldocuments/support/asapo/data/test1/asapo_test1
+mkdir /bldocuments/support/asapo/data/test2/asapo_test2
+chown asapo: -R /bldocuments/support/asapo/data/test/
+
+ssh $mongoaddr docker exec $image bash -c "'echo \"db.dropDatabase()\" | mongo --host $mongoaddr asapo_test'"
+ssh $mongoaddr docker exec $image bash -c "'echo \"db.dropDatabase()\" | mongo --host $mongoaddr asapo_test1'"
+ssh $mongoaddr docker exec $image bash -c "'echo \"db.dropDatabase()\" | mongo --host $mongoaddr asapo_test2'"
+
+
+
+nomad stop asapo-brokers
+nomad run asapo-brokers.nmd
\ No newline at end of file
diff --git a/tests/manual/tests_via_nomad/run_test.sh b/tests/manual/tests_via_nomad/run_test.sh
new file mode 100755
index 000000000..0460ca5e6
--- /dev/null
+++ b/tests/manual/tests_via_nomad/run_test.sh
@@ -0,0 +1,7 @@
+. ./clean_after_tests.sh
+nomad stop asapo-filemon-producer
+nomad run asapo-test_filemon_producer_toreceiver.nomad
+sleep 1
+nomad stop asapo-test
+nomad run asapo-test_filegen_filemon.nomad
+
diff --git a/tests/manual/tests_via_nomad/run_test_dummy_producer.sh b/tests/manual/tests_via_nomad/run_test_dummy_producer.sh
new file mode 100755
index 000000000..dda2a1b3b
--- /dev/null
+++ b/tests/manual/tests_via_nomad/run_test_dummy_producer.sh
@@ -0,0 +1,4 @@
+. ./clean_after_tests.sh
+nomad stop asapo-test
+nomad run asapo-test_dummy_producer_linux_noworker.nomad
+
diff --git a/tests/manual/tests_via_nomad/run_test_linuxonly.sh b/tests/manual/tests_via_nomad/run_test_linuxonly.sh
new file mode 100755
index 000000000..039793552
--- /dev/null
+++ b/tests/manual/tests_via_nomad/run_test_linuxonly.sh
@@ -0,0 +1,7 @@
+. ./clean_after_tests.sh
+nomad stop asapo-filemon-producer
+nomad run asapo-test_filemon_producer_toreceiver.nomad
+sleep 1
+nomad stop asapo-test
+nomad run asapo-test_filegen_filemon_linuxonly.nomad
+
--
GitLab