From fa4212ac4ab95c47545e0d639ccc943d6a5fe144 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Tue, 22 Oct 2019 14:00:56 +0200
Subject: [PATCH] add maxwell tests to git
---
deploy/docker/cluster/run.sh | 45 ++++++++++++-------
receiver/src/data_cache.cpp | 4 +-
receiver/unittests/test_datacache.cpp | 24 +++++-----
.../transfer_single_file/check_linux.sh | 2 +
tests/manual/maxwell/asapo_deploy/asapo_host | 1 +
.../asapo_deploy/asapo_overwrite_vars.tfvars | 19 ++++++++
tests/manual/maxwell/asapo_deploy/job.sh | 9 ++++
.../maxwell/asapo_deploy/start_asapo.sh | 13 ++++++
.../manual/maxwell/asapo_deploy/stop_asapo.sh | 8 ++++
tests/manual/maxwell/asapo_test/asapo_host | 1 +
tests/manual/maxwell/asapo_test/clean.sh | 12 +++++
tests/manual/maxwell/asapo_test/consume.py | 13 ++++++
.../manual/maxwell/asapo_test/get_binaries.sh | 5 +++
.../maxwell/asapo_test/job_consume_python.sh | 12 +++++
.../maxwell/asapo_test/job_produce_python.sh | 12 +++++
tests/manual/maxwell/asapo_test/produce.py | 13 ++++++
tests/manual/maxwell/asapo_test/token | 1 +
17 files changed, 163 insertions(+), 31 deletions(-)
create mode 100644 tests/manual/maxwell/asapo_deploy/asapo_host
create mode 100644 tests/manual/maxwell/asapo_deploy/asapo_overwrite_vars.tfvars
create mode 100755 tests/manual/maxwell/asapo_deploy/job.sh
create mode 100755 tests/manual/maxwell/asapo_deploy/start_asapo.sh
create mode 100755 tests/manual/maxwell/asapo_deploy/stop_asapo.sh
create mode 100644 tests/manual/maxwell/asapo_test/asapo_host
create mode 100755 tests/manual/maxwell/asapo_test/clean.sh
create mode 100644 tests/manual/maxwell/asapo_test/consume.py
create mode 100755 tests/manual/maxwell/asapo_test/get_binaries.sh
create mode 100755 tests/manual/maxwell/asapo_test/job_consume_python.sh
create mode 100755 tests/manual/maxwell/asapo_test/job_produce_python.sh
create mode 100644 tests/manual/maxwell/asapo_test/produce.py
create mode 100644 tests/manual/maxwell/asapo_test/token
diff --git a/deploy/docker/cluster/run.sh b/deploy/docker/cluster/run.sh
index c26f7f0e3..433a5deb5 100755
--- a/deploy/docker/cluster/run.sh
+++ b/deploy/docker/cluster/run.sh
@@ -13,6 +13,9 @@ ASAPO_USER=`id -u`:`id -g`
#SERVER_ADRESSES=
#N_SERVERS=
+ASAPO_VAR_FILE=`pwd`/asapo_overwrite_vars.tfvars
+
+
mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED
chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED
@@ -23,25 +26,33 @@ chmod 777 *
mmc=`cat /proc/sys/vm/max_map_count`
if (( mmc < 262144 )); then
- echo increase max_map_count - needed for elasticsearch
+ echo increase max_map_count - needed for elasticsearch
exit 1
fi
+if [ -f $ASAPO_VAR_FILE ]; then
+ chmod 666 $ASAPO_VAR_FILE
+ MOUNT_VAR_FILE="-v $ASAPO_VAR_FILE:/var/run/asapo/user_vars.tfvars"
+fi
+
+
docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \
- -u $ASAPO_USER \
- -v /var/lib/docker:/var/lib/docker \
- -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \
- -v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \
- -v $DATA_GLOBAL_SHARED:$DATA_GLOBAL_SHARED \
- -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \
- -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \
- -e TF_VAR_data_dir=$DATA_GLOBAL_SHARED \
- -e TF_VAR_mongo_dir=$MONGO_DIR \
- -e ADVERTISE_IP=$ADVERTISE_IP \
- -e RECURSORS=$RECURSORS \
- -e TF_VAR_asapo_user=$ASAPO_USER \
- -e IB_ADDRESS=$IB_ADDRESS \
- -e SERVER_ADRESSES=$SERVER_ADRESSES \
- -e N_SERVERS=$N_SERVERS \
- --name asapo --net=host -d yakser/asapo-cluster
+ -u $ASAPO_USER \
+ --group-add `getent group docker | cut -d: -f3` \
+ -v /var/lib/docker:/var/lib/docker \
+ -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \
+ -v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \
+ -v $DATA_GLOBAL_SHARED:$DATA_GLOBAL_SHARED \
+ -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \
+ -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \
+ -e TF_VAR_data_dir=$DATA_GLOBAL_SHARED \
+ -e TF_VAR_mongo_dir=$MONGO_DIR \
+ $MOUNT_VAR_FILE \
+ -e ADVERTISE_IP=$ADVERTISE_IP \
+ -e RECURSORS=$RECURSORS \
+ -e TF_VAR_asapo_user=$ASAPO_USER \
+ -e IB_ADDRESS=$IB_ADDRESS \
+ -e SERVER_ADRESSES=$SERVER_ADRESSES \
+ -e N_SERVERS=$N_SERVERS \
+ --name asapo --net=host -d yakser/asapo-cluster
diff --git a/receiver/src/data_cache.cpp b/receiver/src/data_cache.cpp
index 8355ba2b3..dbcb01fd2 100644
--- a/receiver/src/data_cache.cpp
+++ b/receiver/src/data_cache.cpp
@@ -116,8 +116,8 @@ bool DataCache::CleanOldSlots(uint64_t size) {
if (meta_[i]->lock > 0) return false;
}
- if (last_del >=0) {
- meta_.erase(meta_.begin(), meta_.begin() + last_del + 1);
+ if (last_del >= 0) {
+ meta_.erase(meta_.begin(), meta_.begin() + last_del + 1);
}
return true;
}
diff --git a/receiver/unittests/test_datacache.cpp b/receiver/unittests/test_datacache.cpp
index d305c5005..28bbde72b 100644
--- a/receiver/unittests/test_datacache.cpp
+++ b/receiver/unittests/test_datacache.cpp
@@ -156,27 +156,27 @@ TEST_F(DataCacheTests, GetFreeSlotRemovesOldMetadataRecords) {
TEST_F(DataCacheTests, GetFreeSlotRemovesOldWhenCrossTheBoundary) {
DataCache cache{expected_cache_size, 0};
CacheMeta* meta1, *meta2, *meta3;
- CacheMeta* meta4,*meta5,*meta;
- auto addr1_alloc = cache.GetFreeSlotAndLock(expected_cache_size/3-1, &meta1);
- auto addr2_alloc = cache.GetFreeSlotAndLock(expected_cache_size/3-1, &meta2);
- auto addr3_alloc = cache.GetFreeSlotAndLock(expected_cache_size/3-1, &meta3);
+ CacheMeta* meta4, *meta5, *meta;
+ auto addr1_alloc = cache.GetFreeSlotAndLock(expected_cache_size / 3 - 1, &meta1);
+ auto addr2_alloc = cache.GetFreeSlotAndLock(expected_cache_size / 3 - 1, &meta2);
+ auto addr3_alloc = cache.GetFreeSlotAndLock(expected_cache_size / 3 - 1, &meta3);
auto id1 = meta1->id;
auto id2 = meta2->id;
auto id3 = meta3->id;
cache.UnlockSlot(meta1);
cache.UnlockSlot(meta2);
cache.UnlockSlot(meta3);
- auto addr4_alloc = cache.GetFreeSlotAndLock(expected_cache_size/2+5, &meta4);
+ auto addr4_alloc = cache.GetFreeSlotAndLock(expected_cache_size / 2 + 5, &meta4);
auto id4 = meta4->id;
cache.UnlockSlot(meta4);
- auto addr5_alloc = cache.GetFreeSlotAndLock(expected_cache_size/2+5, &meta5);
+ auto addr5_alloc = cache.GetFreeSlotAndLock(expected_cache_size / 2 + 5, &meta5);
auto id5 = meta5->id;
- uint8_t* addr1 = (uint8_t*) cache.GetSlotToReadAndLock(id1, expected_cache_size/3-1, &meta);
- uint8_t* addr2 = (uint8_t*) cache.GetSlotToReadAndLock(id2, expected_cache_size/3-1, &meta);
- uint8_t* addr3 = (uint8_t*) cache.GetSlotToReadAndLock(id3, expected_cache_size/3-1, &meta);
- uint8_t* addr4 = (uint8_t*) cache.GetSlotToReadAndLock(id4, expected_cache_size/2+5, &meta);
- uint8_t* addr5 = (uint8_t*) cache.GetSlotToReadAndLock(id5, expected_cache_size/2+5, &meta);
+ uint8_t* addr1 = (uint8_t*) cache.GetSlotToReadAndLock(id1, expected_cache_size / 3 - 1, &meta);
+ uint8_t* addr2 = (uint8_t*) cache.GetSlotToReadAndLock(id2, expected_cache_size / 3 - 1, &meta);
+ uint8_t* addr3 = (uint8_t*) cache.GetSlotToReadAndLock(id3, expected_cache_size / 3 - 1, &meta);
+ uint8_t* addr4 = (uint8_t*) cache.GetSlotToReadAndLock(id4, expected_cache_size / 2 + 5, &meta);
+ uint8_t* addr5 = (uint8_t*) cache.GetSlotToReadAndLock(id5, expected_cache_size / 2 + 5, &meta);
ASSERT_THAT(addr1_alloc, Ne(nullptr));
ASSERT_THAT(addr2_alloc, Ne(nullptr));
@@ -191,7 +191,7 @@ TEST_F(DataCacheTests, GetFreeSlotRemovesOldWhenCrossTheBoundary) {
ASSERT_THAT(addr5, Ne(nullptr));
- ASSERT_THAT(meta->size, Eq(expected_cache_size/2+5));
+ ASSERT_THAT(meta->size, Eq(expected_cache_size / 2 + 5));
}
diff --git a/tests/automatic/producer_receiver/transfer_single_file/check_linux.sh b/tests/automatic/producer_receiver/transfer_single_file/check_linux.sh
index 83a0382b7..ae156e6f9 100644
--- a/tests/automatic/producer_receiver/transfer_single_file/check_linux.sh
+++ b/tests/automatic/producer_receiver/transfer_single_file/check_linux.sh
@@ -32,6 +32,8 @@ nomad run discovery.nmd
mkdir -p ${receiver_folder}
+sleep 1
+
$1 localhost:8400 ${beamtime_id} 100 1 1 0 30
ls -ln ${receiver_folder}/1 | awk '{ print $5 }'| grep 100000
diff --git a/tests/manual/maxwell/asapo_deploy/asapo_host b/tests/manual/maxwell/asapo_deploy/asapo_host
new file mode 100644
index 000000000..165dea9fa
--- /dev/null
+++ b/tests/manual/maxwell/asapo_deploy/asapo_host
@@ -0,0 +1 @@
+max-wn001
diff --git a/tests/manual/maxwell/asapo_deploy/asapo_overwrite_vars.tfvars b/tests/manual/maxwell/asapo_deploy/asapo_overwrite_vars.tfvars
new file mode 100644
index 000000000..873f70add
--- /dev/null
+++ b/tests/manual/maxwell/asapo_deploy/asapo_overwrite_vars.tfvars
@@ -0,0 +1,19 @@
+#asapo_image_tag = "develop.19.09-64-g24f0c4c"
+
+elk_logs = true
+
+receiver_total_memory_size = 35000
+receiver_dataserver_cache_size = 30 #gb
+receiver_dataserver_nthreads = 8
+
+grafana_total_memory_size = 2000
+influxdb_total_memory_size = 2000
+fluentd_total_memory_size = 1000
+elasticsearch_total_memory_size = 3000
+kibana_total_memory_size = 1000
+mongo_total_memory_size = 20000
+authorizer_total_memory_size = 512
+discovery_total_memory_size = 512
+
+n_receivers = 4
+n_brokers = 1
\ No newline at end of file
diff --git a/tests/manual/maxwell/asapo_deploy/job.sh b/tests/manual/maxwell/asapo_deploy/job.sh
new file mode 100755
index 000000000..561693a17
--- /dev/null
+++ b/tests/manual/maxwell/asapo_deploy/job.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+#SBATCH --nodes=5
+#SBATCH -t 6-10:40:00
+
+
+echo start on $SLURM_JOB_NUM_NODES
+srun --ntasks=$SLURM_JOB_NUM_NODES --ntasks-per-node=1 ./run_maxwell.sh
+
diff --git a/tests/manual/maxwell/asapo_deploy/start_asapo.sh b/tests/manual/maxwell/asapo_deploy/start_asapo.sh
new file mode 100755
index 000000000..ac98c90e0
--- /dev/null
+++ b/tests/manual/maxwell/asapo_deploy/start_asapo.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+asapo_host=`cat asapo_host`
+
+! ssh $asapo_host DOCKER_CONT_NAME=asapo dockerexec cat /var/nomad/token > token
+
+
+ssh $asapo_host DOCKER_CONT_NAME=asapo dockerexec asapo-start $@
+ssh $asapo_host DOCKER_CONT_NAME=asapo dockerexec cat /var/nomad/token > token
+
+cat token
\ No newline at end of file
diff --git a/tests/manual/maxwell/asapo_deploy/stop_asapo.sh b/tests/manual/maxwell/asapo_deploy/stop_asapo.sh
new file mode 100755
index 000000000..5c23ee746
--- /dev/null
+++ b/tests/manual/maxwell/asapo_deploy/stop_asapo.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+
+asapo_host=`cat asapo_host`
+
+ssh $asapo_host DOCKER_CONT_NAME=asapo dockerexec asapo-stop $@
+
diff --git a/tests/manual/maxwell/asapo_test/asapo_host b/tests/manual/maxwell/asapo_test/asapo_host
new file mode 100644
index 000000000..165dea9fa
--- /dev/null
+++ b/tests/manual/maxwell/asapo_test/asapo_host
@@ -0,0 +1 @@
+max-wn001
diff --git a/tests/manual/maxwell/asapo_test/clean.sh b/tests/manual/maxwell/asapo_test/clean.sh
new file mode 100755
index 000000000..3160c9320
--- /dev/null
+++ b/tests/manual/maxwell/asapo_test/clean.sh
@@ -0,0 +1,12 @@
+export asapo_host=`cat asapo_host`
+
+dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream
+dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream0
+dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream1
+dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream2
+dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream3
+dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream4
+dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream5
+dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream6
+dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream7
+dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream8
diff --git a/tests/manual/maxwell/asapo_test/consume.py b/tests/manual/maxwell/asapo_test/consume.py
new file mode 100644
index 000000000..48a62ac79
--- /dev/null
+++ b/tests/manual/maxwell/asapo_test/consume.py
@@ -0,0 +1,13 @@
+from mpi4py import MPI
+import os
+comm = MPI.COMM_WORLD
+rank = comm.Get_rank()
+
+with open('asapo_host', 'r') as file:
+ host = file.read().replace('\n', '')
+
+token="KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc="
+
+os.system("hostname")
+os.system("./getnext_broker "+host+":8400 /gpfs/petra3/scratch/yakubov/asapo_shared/test/asapo_test asapo_test%stream"+str(rank)+" 8 "+token+" 1000 0")
+
diff --git a/tests/manual/maxwell/asapo_test/get_binaries.sh b/tests/manual/maxwell/asapo_test/get_binaries.sh
new file mode 100755
index 000000000..02735d754
--- /dev/null
+++ b/tests/manual/maxwell/asapo_test/get_binaries.sh
@@ -0,0 +1,5 @@
+scp zitpcx27016:/home/yakubov/projects/asapo/cmake-build-release/examples/producer/dummy-data-producer/dummy-data-producer .
+scp zitpcx27016:/home/yakubov/projects/asapo/cmake-build-release/examples/consumer/getnext_broker/getnext_broker .
+
+
+
diff --git a/tests/manual/maxwell/asapo_test/job_consume_python.sh b/tests/manual/maxwell/asapo_test/job_consume_python.sh
new file mode 100755
index 000000000..bece7e8f6
--- /dev/null
+++ b/tests/manual/maxwell/asapo_test/job_consume_python.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+#SBATCH --nodes=4
+#SBATCH --ntasks=4
+#SBATCH -t 10:10:00
+
+export asapo_host=`cat asapo_host`
+
+module load mpi/openmpi-x86_64
+
+mpirun --map-by node --mca mpi_warn_on_fork 0 python consume.py
+
diff --git a/tests/manual/maxwell/asapo_test/job_produce_python.sh b/tests/manual/maxwell/asapo_test/job_produce_python.sh
new file mode 100755
index 000000000..d03788cf7
--- /dev/null
+++ b/tests/manual/maxwell/asapo_test/job_produce_python.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+#SBATCH --nodes=4
+#SBATCH --ntasks=4
+#SBATCH -t 00:10:00
+
+export asapo_host=`cat asapo_host`
+
+module load mpi/openmpi-x86_64
+
+mpirun --map-by node --mca mpi_warn_on_fork 0 python produce.py
+
diff --git a/tests/manual/maxwell/asapo_test/produce.py b/tests/manual/maxwell/asapo_test/produce.py
new file mode 100644
index 000000000..ee2f00c31
--- /dev/null
+++ b/tests/manual/maxwell/asapo_test/produce.py
@@ -0,0 +1,13 @@
+from mpi4py import MPI
+import os
+comm = MPI.COMM_WORLD
+rank = comm.Get_rank()
+
+with open('asapo_host', 'r') as file:
+ host = file.read().replace('\n', '')
+
+token="KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc="
+os.system("hostname")
+os.system("./dummy-data-producer "+host+":8400 asapo_test%stream"+str(rank)+"%"+token+" 1000000 50 8 0 1000")
+
+
diff --git a/tests/manual/maxwell/asapo_test/token b/tests/manual/maxwell/asapo_test/token
new file mode 100644
index 000000000..0071119ce
--- /dev/null
+++ b/tests/manual/maxwell/asapo_test/token
@@ -0,0 +1 @@
+KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc=
--
GitLab