diff --git a/common/cpp/src/system_io/system_io_linux.cpp b/common/cpp/src/system_io/system_io_linux.cpp index 5b8c2e7d9b8aafc6ccc8cb33899412a254e63dd7..2c97e377fa03feb64a112bcf4abba10754f30c16 100644 --- a/common/cpp/src/system_io/system_io_linux.cpp +++ b/common/cpp/src/system_io/system_io_linux.cpp @@ -110,6 +110,11 @@ SystemIO::~SystemIO() { } } +void SystemIO::SetThreadName(std::thread* threadHandle, const std::string& name) const { + // If the length of name is greater than 15 characters, the excess characters are ignored. + pthread_setname_np(threadHandle->native_handle(), name.c_str()); +} + void asapo::SystemIO::CloseSocket(SocketDescriptor fd, Error* err) const { if (err) { *err = nullptr; diff --git a/common/cpp/src/system_io/system_io_linux_mac.cpp b/common/cpp/src/system_io/system_io_linux_mac.cpp index 56fb401e7dc54427cb3c2d6ecbb45d80674f25e4..aa8d193ae60102b55081cd37c278e94bf12b0063 100644 --- a/common/cpp/src/system_io/system_io_linux_mac.cpp +++ b/common/cpp/src/system_io/system_io_linux_mac.cpp @@ -201,11 +201,6 @@ void SystemIO::CollectFileInformationRecursively(const std::string& path, closedir(dir); } -void SystemIO::SetThreadName(std::thread* threadHandle, const std::string& name) const { - // If the length of name is greater than 15 characters, the excess characters are ignored. - pthread_setname_np(threadHandle->native_handle(), name.c_str()); -} - void SystemIO::ApplyNetworkOptions(SocketDescriptor socket_fd, Error* err) const { //TODO: Need to change network layer code, so everything can be NonBlocking int flag = 1; diff --git a/common/cpp/src/system_io/system_io_mac.cpp b/common/cpp/src/system_io/system_io_mac.cpp index 6d70c051ae31b046a06f9d3ac781dc70e269a213..2e240568e18a6663edc66a07b434e2b6a6bdff3b 100644 --- a/common/cpp/src/system_io/system_io_mac.cpp +++ b/common/cpp/src/system_io/system_io_mac.cpp @@ -76,6 +76,10 @@ ListSocketDescriptors SystemIO::WaitSocketsActivity(SocketDescriptor master_sock return active_sockets; } +void SystemIO::SetThreadName(std::thread* threadHandle, const std::string& name) const { + // does not work on macos (could only set name for current thread, which is not what we want) +} + void asapo::SystemIO::CloseSocket(SocketDescriptor fd, Error* err) const { if (err) { *err = nullptr; diff --git a/deploy/asapo_orchestration_docker/scripts/vars.tf b/deploy/asapo_orchestration_docker/scripts/vars.tf index b7d43250493347cf961bcedaab57c90f1f2a5326..7e5bf1e0a8de718ba5f801e1c163a858650036f3 100644 --- a/deploy/asapo_orchestration_docker/scripts/vars.tf +++ b/deploy/asapo_orchestration_docker/scripts/vars.tf @@ -9,8 +9,6 @@ variable "nginx_version" {} variable "job_scripts_dir" {} variable "service_dir" {} -variable "data_dir" {} - variable "grafana_total_memory_size" {} variable "telegraf_total_memory_size" {} diff --git a/deploy/asapo_services/asap3.tfvars b/deploy/asapo_services/asap3.tfvars index 0b32c995fab90837b7d357e9223a1c351b4a403c..392572994f0961d135ef942b19c57ca869865b4b 100644 --- a/deploy/asapo_services/asap3.tfvars +++ b/deploy/asapo_services/asap3.tfvars @@ -1,7 +1,8 @@ elk_logs = true service_dir="/gpfs/asapo/shared/service_dir" -data_dir="/beamline" +online_dir="/beamline" +offline_dir="/asap3" mongo_dir="/gpfs/asapo/shared/service_dir/mongodb" asapo_user="35841:1000" job_scripts_dir="/gpfs/asapo/shared/terraform" @@ -22,3 +23,6 @@ discovery_total_memory_size = 512 n_receivers = 1 n_brokers = 1 +n_fts = 1 + + diff --git a/deploy/asapo_services/run.sh b/deploy/asapo_services/run.sh index e33bed710657515f2d4696dca0f7b07df83b84c2..12e23fbd336e194172de41d6debb5dedaaebc742 100755 --- a/deploy/asapo_services/run.sh +++ b/deploy/asapo_services/run.sh @@ -3,6 +3,8 @@ NOMAD_ALLOC_HOST_SHARED=/var/tmp/asapo/container_host_shared/nomad_alloc SERVICE_DATA_CLUSTER_SHARED=/var/tmp/asapo/asapo_cluster_shared/service_data DATA_GLOBAL_SHARED=/var/tmp/asapo/global_shared/data +DATA_GLOBAL_SHARED_ONLINE=/var/tmp/asapo/global_shared/online_data + MONGO_DIR=$SERVICE_DATA_CLUSTER_SHARED/mongodb ASAPO_USER=`id -u`:`id -g` @@ -27,7 +29,7 @@ mmc=`cat /proc/sys/vm/max_map_count` if (( mmc < 262144 )); then echo increase max_map_count - needed for elasticsearch - exit 1 + exit 1 fi if [ -f $ASAPO_VAR_FILE ]; then @@ -45,7 +47,8 @@ docker run --privileged --rm -v /var/run/docker.sock:/var/run/docker.sock \ -v $DATA_GLOBAL_SHARED:$DATA_GLOBAL_SHARED \ -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \ -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \ - -e TF_VAR_data_dir=$DATA_GLOBAL_SHARED \ + -e TF_VAR_online_dir=$DATA_GLOBAL_SHARED_ONLINE \ + -e TF_VAR_offline_dir=$DATA_GLOBAL_SHARED \ -e TF_VAR_mongo_dir=$MONGO_DIR \ $MOUNT_VAR_FILE \ -e ADVERTISE_IP=$ADVERTISE_IP \ diff --git a/deploy/asapo_services/run_maxwell.sh b/deploy/asapo_services/run_maxwell.sh index 47d92d548b1c114f5c650651b8090dd01ba79eab..cdba4bda248f91270bf7b6f713936a65daad660b 100755 --- a/deploy/asapo_services/run_maxwell.sh +++ b/deploy/asapo_services/run_maxwell.sh @@ -4,6 +4,7 @@ NOMAD_ALLOC_HOST_SHARED=/var/tmp/asapo/container_host_shared/nomad_alloc SERVICE_DATA_CLUSTER_SHARED=/home/yakubov/asapo/asapo_cluster_shared/service_data DATA_GLOBAL_SHARED=/gpfs/petra3/scratch/yakubov/asapo_shared +DATA_GLOBAL_SHARED_ONLINE=/tmp MONGO_DIR=/scratch/mongodb # due to performance reasons mongodb can benefit from writing to local filesystem (HA to be worked on) #service distribution MAX_NOMAD_SERVERS=3 # rest are clients @@ -72,7 +73,8 @@ dockerrun --rm \ $MOUNT_VAR_FILE \ -e NOMAD_ALLOC_DIR=$NOMAD_ALLOC_HOST_SHARED \ -e TF_VAR_service_dir=$SERVICE_DATA_CLUSTER_SHARED \ - -e TF_VAR_data_dir=$DATA_GLOBAL_SHARED \ + -e TF_VAR_online_dir=$DATA_GLOBAL_SHARED_ONLINE \ + -e TF_VAR_offline_dir=$DATA_GLOBAL_SHARED \ -e TF_VAR_mongo_dir=$MONGO_DIR \ -e ADVERTISE_IP=$ADVERTISE_IP \ -e RECURSORS=$RECURSORS \ diff --git a/deploy/asapo_services/scripts/asapo-fts.nmd.tpl b/deploy/asapo_services/scripts/asapo-fts.nmd.tpl new file mode 100644 index 0000000000000000000000000000000000000000..b3bd2c3063eb4f92c3651c99ded205bf4a2571f5 --- /dev/null +++ b/deploy/asapo_services/scripts/asapo-fts.nmd.tpl @@ -0,0 +1,86 @@ +job "asapo-fts" { + datacenters = ["dc1"] + affinity { + attribute = "$${meta.node_group}" + value = "utl" + weight = 100 + } + + update { + max_parallel = 1 + min_healthy_time = "10s" + healthy_deadline = "3m" + auto_revert = false + } + + group "fts" { + count = ${n_fts} + restart { + attempts = 2 + interval = "3m" + delay = "15s" + mode = "fail" + } + + task "fts" { + driver = "docker" + user = "${asapo_user}" + config { + network_mode = "host" + security_opt = ["no-new-privileges"] + userns_mode = "host" + image = "yakser/asapo-file-transfer${image_suffix}" + force_pull = true + volumes = ["local/config.json:/var/lib/file_transfer/config.json", + "${offline_dir}:${offline_dir}", + "${online_dir}:${online_dir}" + ] + %{ if ! nomad_logs } + logging { + type = "fluentd" + config { + fluentd-address = "localhost:9881" + fluentd-async-connect = true + tag = "asapo.docker" + } + } + %{endif} + } + + resources { + network { + port "fts" {} + } + } + + service { + port = "fts" + name = "asapo-fts" + check { + name = "asapo-fts-alive" + type = "http" + path = "/health-check" + interval = "10s" + timeout = "2s" + } + check_restart { + limit = 2 + grace = "90s" + ignore_warnings = false + } + } + + template { + source = "${scripts_dir}/fts.json.tpl" + destination = "local/config.json" + change_mode = "restart" + } + + template { + source = "${scripts_dir}/auth_secret.key" + destination = "local/secret.key" + change_mode = "restart" + } + } #task brokers + } +} diff --git a/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl b/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl index 969ef465274b6b4ec43f96d60e520645278ef62c..bbbf4dfd86e25e1297a69fdac15fa76456923d31 100644 --- a/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl +++ b/deploy/asapo_services/scripts/asapo-receivers.nmd.tpl @@ -33,7 +33,8 @@ job "asapo-receivers" { image = "yakser/asapo-receiver${image_suffix}" force_pull = true volumes = ["local/config.json:/var/lib/receiver/config.json", - "${data_dir}:/var/lib/receiver/data"] + "${offline_dir}:${offline_dir}", + "${online_dir}:${online_dir}"] %{ if ! nomad_logs } logging { type = "fluentd" diff --git a/deploy/asapo_services/scripts/asapo-services.nmd.tpl b/deploy/asapo_services/scripts/asapo-services.nmd.tpl index 4dc5ceafd6f2ff6ab95441840c506002ba5aad64..818819dc111c028f7476e39ed0f0e5324c20924c 100644 --- a/deploy/asapo_services/scripts/asapo-services.nmd.tpl +++ b/deploy/asapo_services/scripts/asapo-services.nmd.tpl @@ -20,7 +20,10 @@ job "asapo-services" { userns_mode = "host" image = "yakser/asapo-authorizer${image_suffix}" force_pull = true - volumes = ["local/config.json:/var/lib/authorizer/config.json"] + volumes = ["local/config.json:/var/lib/authorizer/config.json", + "${offline_dir}:${offline_dir}", + "${online_dir}:${online_dir}"] + %{ if ! nomad_logs } logging { type = "fluentd" @@ -42,6 +45,11 @@ job "asapo-services" { } } + meta { + offline_dir = "${offline_dir}" + online_dir = "${online_dir}" + } + service { name = "asapo-authorizer" port = "authorizer" diff --git a/deploy/asapo_services/scripts/asapo.auto.tfvars.in b/deploy/asapo_services/scripts/asapo.auto.tfvars.in index e25ef9f32c017628d762bab568ca5e2986207d67..7c7c153422e7a5154dd812716d13409f75041fa9 100644 --- a/deploy/asapo_services/scripts/asapo.auto.tfvars.in +++ b/deploy/asapo_services/scripts/asapo.auto.tfvars.in @@ -43,4 +43,5 @@ authorizer_port = 5007 consul_dns_port = 8600 n_receivers = 1 -n_brokers = 1 \ No newline at end of file +n_brokers = 1 +n_fts = 1 \ No newline at end of file diff --git a/deploy/asapo_services/scripts/authorizer.json.tpl b/deploy/asapo_services/scripts/authorizer.json.tpl index 75f151a0dfce23875857e59e0a1d898dba272a0c..e4f4e62bc86d2b15bd1bfc1fec751073cbc7ce1a 100644 --- a/deploy/asapo_services/scripts/authorizer.json.tpl +++ b/deploy/asapo_services/scripts/authorizer.json.tpl @@ -1,8 +1,11 @@ { "Port": {{ env "NOMAD_PORT_authorizer" }}, "LogLevel":"debug", - "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","core-path":"/var/lib/receiver/data/test_facility/gpfs/test/2019/data/asapo_test"}, - {"beamtimeId":"asapo_test1","beamline":"test1","core-path":"/var/lib/receiver/data/test_facility/gpfs/test1/2019/data/asapo_test1"}, - {"beamtimeId":"asapo_test2","beamline":"test2","core-path":"/var/lib/receiver/data/test_facility/gpfs/test2/2019/data/asapo_test2"}], - "SecretFile":"/local/secret.key" + "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","core-path":"{{ env "NOMAD_META_offline_dir" }}/test_facility/gpfs/test/2019/data/asapo_test"}, + {"beamtimeId":"asapo_test1","beamline":"test1","core-path":"{{ env "NOMAD_META_offline_dir" }}/test_facility/gpfs/test1/2019/data/asapo_test1"}, + {"beamtimeId":"asapo_test2","beamline":"test2","core-path":"{{ env "NOMAD_META_offline_dir" }}/test_facility/gpfs/test2/2019/data/asapo_test2"}], + "RootBeamtimesFolder":"{{ env "NOMAD_META_offline_dir" }}", + "CurrentBeamlinesFolder":"{{ env "NOMAD_META_online_dir" }}", + "SecretFile":"/local/secret.key", + "TokenDurationMin":600 } diff --git a/deploy/asapo_services/scripts/fts.json.tpl b/deploy/asapo_services/scripts/fts.json.tpl new file mode 100644 index 0000000000000000000000000000000000000000..5f0f028a0fccdc606a728c6a50ec6f9af8ee1408 --- /dev/null +++ b/deploy/asapo_services/scripts/fts.json.tpl @@ -0,0 +1,5 @@ +{ + "Port": {{ env "NOMAD_PORT_fts" }}, + "LogLevel":"debug", + "SecretFile":"/local/secret.key" +} diff --git a/deploy/asapo_services/scripts/resources.tf b/deploy/asapo_services/scripts/resources.tf index 0ad2631b9c4abf9bda6109f7214b3dfb11289212..1539150f6436c3920ebe58520fb05637882b205a 100644 --- a/deploy/asapo_services/scripts/resources.tf +++ b/deploy/asapo_services/scripts/resources.tf @@ -29,3 +29,9 @@ resource "nomad_job" "asapo-brokers" { jobspec = "${data.template_file.asapo_brokers.rendered}" depends_on = [nomad_job.asapo-services,null_resource.asapo-authorizer,null_resource.asapo-discovery] } + +resource "nomad_job" "asapo-fts" { + jobspec = "${data.template_file.asapo_fts.rendered}" + depends_on = [nomad_job.asapo-services,null_resource.asapo-authorizer,null_resource.asapo-discovery] +} + diff --git a/deploy/asapo_services/scripts/resources_services.tf b/deploy/asapo_services/scripts/resources_services.tf index dc18b817b76c141fd3142108d133c47ce51ee8a1..0c2ddf51c8c14fd0b342bca62d73d359ea311015 100644 --- a/deploy/asapo_services/scripts/resources_services.tf +++ b/deploy/asapo_services/scripts/resources_services.tf @@ -51,6 +51,13 @@ resource "null_resource" "asapo-broker" { depends_on = [nomad_job.asapo-brokers] } +resource "null_resource" "asapo-fts" { + provisioner "local-exec" { + command = "asapo-wait-service asapo-fts" + } + depends_on = [nomad_job.asapo-fts] +} + resource "null_resource" "asapo-receiver" { provisioner "local-exec" { command = "asapo-wait-service asapo-receiver" diff --git a/deploy/asapo_services/scripts/templates.tf b/deploy/asapo_services/scripts/templates.tf index 709e46451ae6fa0ea322306cc685e92b91c60f9b..50b860111241aa7d6d587e74d9b62b16575adb92 100644 --- a/deploy/asapo_services/scripts/templates.tf +++ b/deploy/asapo_services/scripts/templates.tf @@ -20,6 +20,8 @@ data "template_file" "asapo_services" { template = "${file("${var.job_scripts_dir}/asapo-services.nmd.tpl")}" vars = { scripts_dir = "${var.job_scripts_dir}" + online_dir = "${var.online_dir}" + offline_dir = "${var.offline_dir}" image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}" nomad_logs = "${var.nomad_logs}" authorizer_total_memory_size = "${var.authorizer_total_memory_size}" @@ -34,7 +36,8 @@ data "template_file" "asapo_receivers" { template = "${file("${var.job_scripts_dir}/asapo-receivers.nmd.tpl")}" vars = { scripts_dir = "${var.job_scripts_dir}" - data_dir = "${var.data_dir}" + online_dir = "${var.online_dir}" + offline_dir = "${var.offline_dir}" image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}" nomad_logs = "${var.nomad_logs}" receiver_total_memory_size = "${var.receiver_total_memory_size}" @@ -58,6 +61,19 @@ data "template_file" "asapo_brokers" { } +data "template_file" "asapo_fts" { + template = "${file("${var.job_scripts_dir}/asapo-fts.nmd.tpl")}" + vars = { + scripts_dir = "${var.job_scripts_dir}" + online_dir = "${var.online_dir}" + offline_dir = "${var.offline_dir}" + image_suffix = "${var.asapo_imagename_suffix}:${var.asapo_image_tag}" + nomad_logs = "${var.nomad_logs}" + asapo_user = "${var.asapo_user}" + n_fts = "${var.n_fts}" + } +} + data "template_file" "asapo_perfmetrics" { template = "${file("${var.job_scripts_dir}/asapo-perfmetrics.nmd.tpl")}" vars = { diff --git a/deploy/asapo_services/scripts/vars.tf b/deploy/asapo_services/scripts/vars.tf index 0de5e3678a78d41f98a21be6c5fead134738c793..77412ea3f8b16c2e300b6b5266c2c9224a77d5e8 100644 --- a/deploy/asapo_services/scripts/vars.tf +++ b/deploy/asapo_services/scripts/vars.tf @@ -24,7 +24,8 @@ variable "job_scripts_dir" {} variable "service_dir" {} -variable "data_dir" {} +variable "online_dir" {} +variable "offline_dir" {} variable "mongo_dir" {} @@ -74,4 +75,6 @@ variable "consul_dns_port" {} variable "n_receivers" {} -variable "n_brokers" {} \ No newline at end of file +variable "n_brokers" {} + +variable "n_fts" {} \ No newline at end of file diff --git a/tests/manual/python_tests/consumer/consumer_api.py b/tests/manual/python_tests/consumer/consumer_api.py new file mode 100644 index 0000000000000000000000000000000000000000..c59d022e451abc660e711e36904a6164da7aee43 --- /dev/null +++ b/tests/manual/python_tests/consumer/consumer_api.py @@ -0,0 +1,17 @@ +from __future__ import print_function + +import asapo_consumer +import sys + +source, beamtime,path, token = sys.argv[1:] +broker = asapo_consumer.create_server_broker(source,path,False, beamtime,"",token,1000) +group_id = broker.generate_group_id() + + +data, meta = broker.get_by_id(1, group_id, meta_only=False) + +print (meta) +print (len(data)) + + +sys.exit(0) \ No newline at end of file diff --git a/tests/manual/python_tests/consumer/test.sh b/tests/manual/python_tests/consumer/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..6b5254e86fc977cf3afda84823a48a00f5ae6386 --- /dev/null +++ b/tests/manual/python_tests/consumer/test.sh @@ -0,0 +1 @@ +python3 consumer_api.py asapo-services.desy.de:8400 asapo_test /shared_data/test_facility/gpfs/test/2019/data/asapo_test KmUDdacgBzaOD3NIJvN1NmKGqWKtx0DK-NyPjdpeWkc= \ No newline at end of file