diff --git a/deploy/docker/cluster/run.sh b/deploy/docker/cluster/run.sh index c26f7f0e36aefc11fd2312e1621f90304faeb7b0..433a5deb5b9d9a522bed64a2b2718dbd5238e723 100755 --- a/deploy/docker/cluster/run.sh +++ b/deploy/docker/cluster/run.sh @@ -13,6 +13,9 @@ ASAPO_USER=`id -u`:`id -g` #SERVER_ADRESSES= #N_SERVERS= +ASAPO_VAR_FILE=`pwd`/asapo_overwrite_vars.tfvars + + mkdir -p $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED chmod 777 $NOMAD_ALLOC_HOST_SHARED $SERVICE_DATA_CLUSTER_SHARED $DATA_GLOBAL_SHARED @@ -23,25 +26,33 @@ chmod 777 * mmc=`cat /proc/sys/vm/max_map_count` if (( mmc < 262144 )); then - echo increase max_map_count - needed for elasticsearch + echo increase max_map_count - needed for elasticsearch exit 1 fi +if [ -f $ASAPO_VAR_FILE ]; then + chmod 666 $ASAPO_VAR_FILE + MOUNT_VAR_FILE="-v $ASAPO_VAR_FILE:/var/run/asapo/user_vars.tfvars" +fi + + docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \ - -u $ASAPO_USER \ - -v /var/lib/docker:/var/lib/docker \ - -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \ - -v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \ - -v $DATA_GLOBAL_SHARED:$DATA_GLOBAL_SHARED \ - -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \ - -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \ - -e TF_VAR_data_dir=$DATA_GLOBAL_SHARED \ - -e TF_VAR_mongo_dir=$MONGO_DIR \ - -e ADVERTISE_IP=$ADVERTISE_IP \ - -e RECURSORS=$RECURSORS \ - -e TF_VAR_asapo_user=$ASAPO_USER \ - -e IB_ADDRESS=$IB_ADDRESS \ - -e SERVER_ADRESSES=$SERVER_ADRESSES \ - -e N_SERVERS=$N_SERVERS \ - --name asapo --net=host -d yakser/asapo-cluster + -u $ASAPO_USER \ + --group-add `getent group docker | cut -d: -f3` \ + -v /var/lib/docker:/var/lib/docker \ + -v $NOMAD_ALLOC_HOST_SHARED:$NOMAD_ALLOC_HOST_SHARED \ + -v $SERVICE_DATA_CLUSTER_SHARED:$SERVICE_DATA_CLUSTER_SHARED \ + -v $DATA_GLOBAL_SHARED:$DATA_GLOBAL_SHARED \ + -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \ + -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \ + -e TF_VAR_data_dir=$DATA_GLOBAL_SHARED \ + -e TF_VAR_mongo_dir=$MONGO_DIR \ + $MOUNT_VAR_FILE \ + -e ADVERTISE_IP=$ADVERTISE_IP \ + -e RECURSORS=$RECURSORS \ + -e TF_VAR_asapo_user=$ASAPO_USER \ + -e IB_ADDRESS=$IB_ADDRESS \ + -e SERVER_ADRESSES=$SERVER_ADRESSES \ + -e N_SERVERS=$N_SERVERS \ + --name asapo --net=host -d yakser/asapo-cluster diff --git a/receiver/src/data_cache.cpp b/receiver/src/data_cache.cpp index 8355ba2b3c90e530bf2d61e6affe59e6c2fe8f89..dbcb01fd2e9ed2f56b87e46feeaaaaf96f83e551 100644 --- a/receiver/src/data_cache.cpp +++ b/receiver/src/data_cache.cpp @@ -116,8 +116,8 @@ bool DataCache::CleanOldSlots(uint64_t size) { if (meta_[i]->lock > 0) return false; } - if (last_del >=0) { - meta_.erase(meta_.begin(), meta_.begin() + last_del + 1); + if (last_del >= 0) { + meta_.erase(meta_.begin(), meta_.begin() + last_del + 1); } return true; } diff --git a/receiver/unittests/test_datacache.cpp b/receiver/unittests/test_datacache.cpp index d305c50051bdd7ba628ef6f69722dd3e842edbca..28bbde72be4debc5ba378710724af92784becd4a 100644 --- a/receiver/unittests/test_datacache.cpp +++ b/receiver/unittests/test_datacache.cpp @@ -156,27 +156,27 @@ TEST_F(DataCacheTests, GetFreeSlotRemovesOldMetadataRecords) { TEST_F(DataCacheTests, GetFreeSlotRemovesOldWhenCrossTheBoundary) { DataCache cache{expected_cache_size, 0}; CacheMeta* meta1, *meta2, *meta3; - CacheMeta* meta4,*meta5,*meta; - auto addr1_alloc = cache.GetFreeSlotAndLock(expected_cache_size/3-1, &meta1); - auto addr2_alloc = cache.GetFreeSlotAndLock(expected_cache_size/3-1, &meta2); - auto addr3_alloc = cache.GetFreeSlotAndLock(expected_cache_size/3-1, &meta3); + CacheMeta* meta4, *meta5, *meta; + auto addr1_alloc = cache.GetFreeSlotAndLock(expected_cache_size / 3 - 1, &meta1); + auto addr2_alloc = cache.GetFreeSlotAndLock(expected_cache_size / 3 - 1, &meta2); + auto addr3_alloc = cache.GetFreeSlotAndLock(expected_cache_size / 3 - 1, &meta3); auto id1 = meta1->id; auto id2 = meta2->id; auto id3 = meta3->id; cache.UnlockSlot(meta1); cache.UnlockSlot(meta2); cache.UnlockSlot(meta3); - auto addr4_alloc = cache.GetFreeSlotAndLock(expected_cache_size/2+5, &meta4); + auto addr4_alloc = cache.GetFreeSlotAndLock(expected_cache_size / 2 + 5, &meta4); auto id4 = meta4->id; cache.UnlockSlot(meta4); - auto addr5_alloc = cache.GetFreeSlotAndLock(expected_cache_size/2+5, &meta5); + auto addr5_alloc = cache.GetFreeSlotAndLock(expected_cache_size / 2 + 5, &meta5); auto id5 = meta5->id; - uint8_t* addr1 = (uint8_t*) cache.GetSlotToReadAndLock(id1, expected_cache_size/3-1, &meta); - uint8_t* addr2 = (uint8_t*) cache.GetSlotToReadAndLock(id2, expected_cache_size/3-1, &meta); - uint8_t* addr3 = (uint8_t*) cache.GetSlotToReadAndLock(id3, expected_cache_size/3-1, &meta); - uint8_t* addr4 = (uint8_t*) cache.GetSlotToReadAndLock(id4, expected_cache_size/2+5, &meta); - uint8_t* addr5 = (uint8_t*) cache.GetSlotToReadAndLock(id5, expected_cache_size/2+5, &meta); + uint8_t* addr1 = (uint8_t*) cache.GetSlotToReadAndLock(id1, expected_cache_size / 3 - 1, &meta); + uint8_t* addr2 = (uint8_t*) cache.GetSlotToReadAndLock(id2, expected_cache_size / 3 - 1, &meta); + uint8_t* addr3 = (uint8_t*) cache.GetSlotToReadAndLock(id3, expected_cache_size / 3 - 1, &meta); + uint8_t* addr4 = (uint8_t*) cache.GetSlotToReadAndLock(id4, expected_cache_size / 2 + 5, &meta); + uint8_t* addr5 = (uint8_t*) cache.GetSlotToReadAndLock(id5, expected_cache_size / 2 + 5, &meta); ASSERT_THAT(addr1_alloc, Ne(nullptr)); ASSERT_THAT(addr2_alloc, Ne(nullptr)); @@ -191,7 +191,7 @@ TEST_F(DataCacheTests, GetFreeSlotRemovesOldWhenCrossTheBoundary) { ASSERT_THAT(addr5, Ne(nullptr)); - ASSERT_THAT(meta->size, Eq(expected_cache_size/2+5)); + ASSERT_THAT(meta->size, Eq(expected_cache_size / 2 + 5)); } diff --git a/tests/automatic/producer_receiver/transfer_single_file/check_linux.sh b/tests/automatic/producer_receiver/transfer_single_file/check_linux.sh index 83a0382b7db821745ee4e73672213c39750d1c9f..ae156e6f9c1fe72284825687b8ae614be6f5e1c3 100644 --- a/tests/automatic/producer_receiver/transfer_single_file/check_linux.sh +++ b/tests/automatic/producer_receiver/transfer_single_file/check_linux.sh @@ -32,6 +32,8 @@ nomad run discovery.nmd mkdir -p ${receiver_folder} +sleep 1 + $1 localhost:8400 ${beamtime_id} 100 1 1 0 30 ls -ln ${receiver_folder}/1 | awk '{ print $5 }'| grep 100000 diff --git a/tests/manual/maxwell/asapo_deploy/asapo_host b/tests/manual/maxwell/asapo_deploy/asapo_host new file mode 100644 index 0000000000000000000000000000000000000000..165dea9fafd18f80586632c5c7a4ef7fcd48cbfc --- /dev/null +++ b/tests/manual/maxwell/asapo_deploy/asapo_host @@ -0,0 +1 @@ +max-wn001 diff --git a/tests/manual/maxwell/asapo_deploy/asapo_overwrite_vars.tfvars b/tests/manual/maxwell/asapo_deploy/asapo_overwrite_vars.tfvars new file mode 100644 index 0000000000000000000000000000000000000000..873f70addc186ce01dfeeb8731a6348bfb9516d3 --- /dev/null +++ b/tests/manual/maxwell/asapo_deploy/asapo_overwrite_vars.tfvars @@ -0,0 +1,19 @@ +#asapo_image_tag = "develop.19.09-64-g24f0c4c" + +elk_logs = true + +receiver_total_memory_size = 35000 +receiver_dataserver_cache_size = 30 #gb +receiver_dataserver_nthreads = 8 + +grafana_total_memory_size = 2000 +influxdb_total_memory_size = 2000 +fluentd_total_memory_size = 1000 +elasticsearch_total_memory_size = 3000 +kibana_total_memory_size = 1000 +mongo_total_memory_size = 20000 +authorizer_total_memory_size = 512 +discovery_total_memory_size = 512 + +n_receivers = 4 +n_brokers = 1 \ No newline at end of file diff --git a/tests/manual/maxwell/asapo_deploy/job.sh b/tests/manual/maxwell/asapo_deploy/job.sh new file mode 100755 index 0000000000000000000000000000000000000000..561693a17b8a9ded8c47c033171b97308c2b1d38 --- /dev/null +++ b/tests/manual/maxwell/asapo_deploy/job.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +#SBATCH --nodes=5 +#SBATCH -t 6-10:40:00 + + +echo start on $SLURM_JOB_NUM_NODES +srun --ntasks=$SLURM_JOB_NUM_NODES --ntasks-per-node=1 ./run_maxwell.sh + diff --git a/tests/manual/maxwell/asapo_deploy/start_asapo.sh b/tests/manual/maxwell/asapo_deploy/start_asapo.sh new file mode 100755 index 0000000000000000000000000000000000000000..ac98c90e0783803d50df879f488ffa9463bc260c --- /dev/null +++ b/tests/manual/maxwell/asapo_deploy/start_asapo.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +asapo_host=`cat asapo_host` + +! ssh $asapo_host DOCKER_CONT_NAME=asapo dockerexec cat /var/nomad/token > token + + +ssh $asapo_host DOCKER_CONT_NAME=asapo dockerexec asapo-start $@ +ssh $asapo_host DOCKER_CONT_NAME=asapo dockerexec cat /var/nomad/token > token + +cat token \ No newline at end of file diff --git a/tests/manual/maxwell/asapo_deploy/stop_asapo.sh b/tests/manual/maxwell/asapo_deploy/stop_asapo.sh new file mode 100755 index 0000000000000000000000000000000000000000..5c23ee746482255be76dd0eac26f167e4515e904 --- /dev/null +++ b/tests/manual/maxwell/asapo_deploy/stop_asapo.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + +asapo_host=`cat asapo_host` + +ssh $asapo_host DOCKER_CONT_NAME=asapo dockerexec asapo-stop $@ + diff --git a/tests/manual/maxwell/asapo_test/asapo_host b/tests/manual/maxwell/asapo_test/asapo_host new file mode 100644 index 0000000000000000000000000000000000000000..165dea9fafd18f80586632c5c7a4ef7fcd48cbfc --- /dev/null +++ b/tests/manual/maxwell/asapo_test/asapo_host @@ -0,0 +1 @@ +max-wn001 diff --git a/tests/manual/maxwell/asapo_test/clean.sh b/tests/manual/maxwell/asapo_test/clean.sh new file mode 100755 index 0000000000000000000000000000000000000000..3160c9320cd1d85a49365bd683f5ccf2d8fde3ae --- /dev/null +++ b/tests/manual/maxwell/asapo_test/clean.sh @@ -0,0 +1,12 @@ +export asapo_host=`cat asapo_host` + +dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream +dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream0 +dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream1 +dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream2 +dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream3 +dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream4 +dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream5 +dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream6 +dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream7 +dockerrun -v `pwd`:/tmp/yakubov mongo mongo --host `curl -s $asapo_host:8400/discovery/mongo` --eval "db.dropDatabase()" asapo_test_stream8 diff --git a/tests/manual/maxwell/asapo_test/consume.py b/tests/manual/maxwell/asapo_test/consume.py new file mode 100644 index 0000000000000000000000000000000000000000..48a62ac791e8149b7f066afd83500203764e3403 --- /dev/null +++ b/tests/manual/maxwell/asapo_test/consume.py @@ -0,0 +1,13 @@ +from mpi4py import MPI +import os +comm = MPI.COMM_WORLD +rank = comm.Get_rank() + +with open('asapo_host', 'r') as file: + host = file.read().replace('\n', '') + +token="KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc=" + +os.system("hostname") +os.system("./getnext_broker "+host+":8400 /gpfs/petra3/scratch/yakubov/asapo_shared/test/asapo_test asapo_test%stream"+str(rank)+" 8 "+token+" 1000 0") + diff --git a/tests/manual/maxwell/asapo_test/get_binaries.sh b/tests/manual/maxwell/asapo_test/get_binaries.sh new file mode 100755 index 0000000000000000000000000000000000000000..02735d75463d37121f2c3e4962363c938b43e3fe --- /dev/null +++ b/tests/manual/maxwell/asapo_test/get_binaries.sh @@ -0,0 +1,5 @@ +scp zitpcx27016:/home/yakubov/projects/asapo/cmake-build-release/examples/producer/dummy-data-producer/dummy-data-producer . +scp zitpcx27016:/home/yakubov/projects/asapo/cmake-build-release/examples/consumer/getnext_broker/getnext_broker . + + + diff --git a/tests/manual/maxwell/asapo_test/job_consume_python.sh b/tests/manual/maxwell/asapo_test/job_consume_python.sh new file mode 100755 index 0000000000000000000000000000000000000000..bece7e8f60c5d354b5c16ea0ee4882afecc5e953 --- /dev/null +++ b/tests/manual/maxwell/asapo_test/job_consume_python.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH -t 10:10:00 + +export asapo_host=`cat asapo_host` + +module load mpi/openmpi-x86_64 + +mpirun --map-by node --mca mpi_warn_on_fork 0 python consume.py + diff --git a/tests/manual/maxwell/asapo_test/job_produce_python.sh b/tests/manual/maxwell/asapo_test/job_produce_python.sh new file mode 100755 index 0000000000000000000000000000000000000000..d03788cf7e09e3b02ee4b16ad7ed07b843b0a4bb --- /dev/null +++ b/tests/manual/maxwell/asapo_test/job_produce_python.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +#SBATCH --nodes=4 +#SBATCH --ntasks=4 +#SBATCH -t 00:10:00 + +export asapo_host=`cat asapo_host` + +module load mpi/openmpi-x86_64 + +mpirun --map-by node --mca mpi_warn_on_fork 0 python produce.py + diff --git a/tests/manual/maxwell/asapo_test/produce.py b/tests/manual/maxwell/asapo_test/produce.py new file mode 100644 index 0000000000000000000000000000000000000000..ee2f00c315d7403cea7c9a339ce02192071098e3 --- /dev/null +++ b/tests/manual/maxwell/asapo_test/produce.py @@ -0,0 +1,13 @@ +from mpi4py import MPI +import os +comm = MPI.COMM_WORLD +rank = comm.Get_rank() + +with open('asapo_host', 'r') as file: + host = file.read().replace('\n', '') + +token="KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc=" +os.system("hostname") +os.system("./dummy-data-producer "+host+":8400 asapo_test%stream"+str(rank)+"%"+token+" 1000000 50 8 0 1000") + + diff --git a/tests/manual/maxwell/asapo_test/token b/tests/manual/maxwell/asapo_test/token new file mode 100644 index 0000000000000000000000000000000000000000..0071119ced72f72869422e2c286a5393a30e69dc --- /dev/null +++ b/tests/manual/maxwell/asapo_test/token @@ -0,0 +1 @@ +KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc=