From 38522f8fb1685cf9fb61812e34d63a506119ee16 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Wed, 8 Apr 2020 14:20:27 +0200
Subject: [PATCH] fix curl client
---
.../cpp/src/http_client/curl_http_client.cpp | 2 +-
.../curl_http_client_command/check_linux.sh | 7 +++--
.../check_windows.bat | 2 +-
.../curl_httpclient_command.cpp | 30 +++++++++++++++----
.../rest_api/check_linux.sh | 5 ++--
.../python_tests/consumer/consumer_api.py | 11 ++++---
.../manual/python_tests/consumer/test_k8s.sh | 4 +++
7 files changed, 46 insertions(+), 15 deletions(-)
create mode 100755 tests/manual/python_tests/consumer/test_k8s.sh
diff --git a/common/cpp/src/http_client/curl_http_client.cpp b/common/cpp/src/http_client/curl_http_client.cpp
index ca543c0a6..829c6a7f1 100644
--- a/common/cpp/src/http_client/curl_http_client.cpp
+++ b/common/cpp/src/http_client/curl_http_client.cpp
@@ -33,7 +33,7 @@ size_t curl_write( void* ptr, size_t size, size_t nmemb, void* data_container) {
if (container->bytes_received + nbytes > container->array_size) {
return -1;
}
- memcpy(container->p_array->get(), ptr, nbytes);
+ memcpy(container->p_array->get()+container->bytes_received, ptr, nbytes);
container->bytes_received += nbytes;
break;
case CurlDataMode::file:
diff --git a/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh b/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh
index 989a4562e..406b111aa 100644
--- a/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh
+++ b/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh
@@ -11,7 +11,7 @@ Cleanup() {
echo cleanup
nomad stop authorizer
nomad stop file_transfer
- rm -rf $file_transfer_folder bbb
+ rm -rf $file_transfer_folder bbb random
}
nomad run authorizer.nmd
@@ -21,7 +21,10 @@ sleep 1
mkdir -p $file_transfer_folder
echo -n hello > $file_transfer_folder/aaa
-$1 127.0.0.1:5007 127.0.0.1:5008 $file_transfer_folder aaa
+dd if=/dev/urandom of=$file_transfer_folder/random bs=1 count=100000
+
+$1 127.0.0.1:5007 127.0.0.1:5008 $file_transfer_folder
cat bbb | tee /dev/stderr | grep hello
+diff -q random $file_transfer_folder/random
diff --git a/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat b/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat
index 1748f6e8c..25d7e5c10 100644
--- a/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat
+++ b/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat
@@ -13,7 +13,7 @@ ping 1.0.0.0 -n 1 -w 100 > nul
mkdir %file_transfer_folder%
echo | set /p dummyName="hello" > %file_transfer_folder%\aaa
-"%1" 127.0.0.1:5007 127.0.0.1:5008 %file_transfer_folder% aaa || goto :error
+"%1" 127.0.0.1:5007 127.0.0.1:5008 %file_transfer_folder% || goto :error
type bbb | findstr /c:"hello" || goto :error
diff --git a/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp b/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp
index 63d202b67..75bc3d65b 100644
--- a/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp
+++ b/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp
@@ -4,24 +4,24 @@
#include "testing.h"
#include "../../../consumer/api/cpp/src/server_data_broker.h"
#include "preprocessor/definitions.h"
+#include "io/io_factory.h"
+#include "io/io.h"
struct Args {
std::string uri_authorizer;
std::string uri_fts;
std::string folder;
- std::string fname;
};
Args GetArgs(int argc, char* argv[]) {
- if (argc != 5) {
+ if (argc != 4) {
std::cout << "Wrong number of arguments" << std::endl;
exit(EXIT_FAILURE);
}
std::string uri_authorizer{argv[1]};
std::string uri_fts{argv[2]};
std::string folder{argv[3]};
- std::string fname{argv[4]};
- return Args{uri_authorizer, uri_fts, folder, fname};
+ return Args{uri_authorizer, uri_fts, folder};
}
@@ -58,14 +58,34 @@ int main(int argc, char* argv[]) {
auto content = server_broker->httpclient__->Post(args.uri_fts + "/transfer", cookie, transfer, &code, &err);
M_AssertEq("hello", content);
M_AssertTrue(code == asapo::HttpCode::OK);
-
+// with array
asapo::FileData data;
err = server_broker->httpclient__->Post(args.uri_fts + "/transfer", cookie, transfer, &data, 5, &code);
M_AssertEq( "hello", reinterpret_cast<char const*>(data.get()));
M_AssertTrue(code == asapo::HttpCode::OK);
+ transfer = "{\"Folder\":\"" + args.folder + "\",\"FileName\":\"random\"}";
+ auto io = asapo::GenerateDefaultIO();
+ auto fname = args.folder+asapo::kPathSeparator+"random";
+ uint64_t size=0;
+ auto expected_data = io->GetDataFromFile(fname,&size,&err);
+ M_AssertEq(nullptr, err);
+ err = server_broker->httpclient__->Post(args.uri_fts + "/transfer", cookie, transfer, &data, size, &code);
+ M_AssertTrue(code == asapo::HttpCode::OK);
+ for (uint64_t i=0;i<size;i++) {
+ if (expected_data[i] != data[i]) {
+ M_AssertTrue(false,"recieve array equal to sent array");
+ }
+ }
+
+// with file
+ transfer = "{\"Folder\":\"" + args.folder + "\",\"FileName\":\"aaa\"}";
err = server_broker->httpclient__->Post(args.uri_fts + "/transfer", cookie, transfer, "bbb", &code);
M_AssertTrue(code == asapo::HttpCode::OK);
+ transfer = "{\"Folder\":\"" + args.folder + "\",\"FileName\":\"random\"}";
+ err = server_broker->httpclient__->Post(args.uri_fts + "/transfer", cookie, transfer, "random", &code);
+ M_AssertTrue(code == asapo::HttpCode::OK);
+
return 0;
}
diff --git a/tests/automatic/file_transfer_service/rest_api/check_linux.sh b/tests/automatic/file_transfer_service/rest_api/check_linux.sh
index 89f45c4c8..b1b61e262 100644
--- a/tests/automatic/file_transfer_service/rest_api/check_linux.sh
+++ b/tests/automatic/file_transfer_service/rest_api/check_linux.sh
@@ -25,11 +25,12 @@ token=bnCXpOdBV90wU1zybEw1duQNSORuwaKz6oDHqmL35p0= #token for aaa
folder_token=`curl --silent --data "{\"Folder\":\"$file_transfer_folder\",\"BeamtimeId\":\"aaa\",\"Token\":\"$token\"}" 127.0.0.1:5007/folder`
echo $folder_token
-echo hello > $file_transfer_folder/aaa
+
+dd if=/dev/urandom of=$file_transfer_folder/aaa bs=1 count=100000
curl -o aaa --silent -H "Authorization: Bearer ${folder_token}" --data "{\"Folder\":\"$file_transfer_folder\",\"FileName\":\"aaa\",\"Token\":\"$folder_token\"}" 127.0.0.1:5008/transfer --stderr - | tee /dev/stderr
-cat aaa | grep hello
+diff -q aaa $file_transfer_folder/aaa
dd if=/dev/zero of=$file_transfer_folder/big_file bs=1 count=0 seek=5368709120
diff --git a/tests/manual/python_tests/consumer/consumer_api.py b/tests/manual/python_tests/consumer/consumer_api.py
index c59d022e4..d67e7577d 100644
--- a/tests/manual/python_tests/consumer/consumer_api.py
+++ b/tests/manual/python_tests/consumer/consumer_api.py
@@ -3,15 +3,18 @@ from __future__ import print_function
import asapo_consumer
import sys
-source, beamtime,path, token = sys.argv[1:]
+source, path,beamtime, token = sys.argv[1:]
broker = asapo_consumer.create_server_broker(source,path,False, beamtime,"",token,1000)
group_id = broker.generate_group_id()
-data, meta = broker.get_by_id(1, group_id, meta_only=False)
+_, meta = broker.get_by_id(1,group_id, meta_only=True)
-print (meta)
-print (len(data))
+meta["buf_id"]=0
+data = broker.retrieve_data(meta)
+print (meta)
+print (len(data),data[0:100])
+data.tofile("out")
sys.exit(0)
\ No newline at end of file
diff --git a/tests/manual/python_tests/consumer/test_k8s.sh b/tests/manual/python_tests/consumer/test_k8s.sh
new file mode 100755
index 000000000..3cd69a940
--- /dev/null
+++ b/tests/manual/python_tests/consumer/test_k8s.sh
@@ -0,0 +1,4 @@
+export PYTHONPATH=/Users/yakubov/projects/asapo/cmake-build-debug/consumer/api/python
+export token=IEfwsWa0GXky2S3MkxJSUHJT1sI8DD5teRdjBUXVRxk=
+python3 consumer_api.py gest-k8s-test2.desy.de/yakser /test_offline/test_facility/gpfs/test/2019/data/asapo_test asapo_test $token
+#python3 getnext.py gest-k8s-test2.desy.de/yakser /test_offline/test_facility/gpfs/test/2019/data/asapo_test asapo_test $token new
\ No newline at end of file
--
GitLab