From 8df4f09209b48097cff439b6940b592eef832484 Mon Sep 17 00:00:00 2001
From: Sergey Yakubov <sergey.yakubov@desy.de>
Date: Tue, 17 Mar 2020 10:30:08 +0100
Subject: [PATCH] check folder in folder token request, refactor authorizer and
tests
---
CMakeModules/prepare_asapo.cmake | 11 ++++++
.../src/asapo_authorizer/server/authorize.go | 2 +-
.../asapo_authorizer/server/authorize_test.go | 4 +-
.../asapo_authorizer/server/folder_token.go | 26 +++++++++++++
.../server/folder_token_test.go | 39 +++++++++++++++----
.../consumer_api_python/CMakeLists.txt | 3 ++
.../authorizer_settings.json.tpl.in | 8 ++++
.../consumer_api_python/check_linux.sh | 2 +-
.../consumer_api_python/check_windows.bat | 6 ++-
.../curl_http_client_command/CMakeLists.txt | 2 +
.../authorizer_settings.json.tpl.in | 8 ++++
.../curl_http_client_command/check_linux.sh | 3 +-
.../check_windows.bat | 7 +++-
.../rest_api/CMakeLists.txt | 2 +-
.../rest_api/authorizer_settings.json.tpl.in | 8 ++++
.../rest_api/check_linux.sh | 6 ++-
.../rest_api/check_windows.bat | 10 +++--
17 files changed, 125 insertions(+), 22 deletions(-)
create mode 100644 tests/automatic/consumer/consumer_api_python/authorizer_settings.json.tpl.in
create mode 100644 tests/automatic/curl_http_client/curl_http_client_command/authorizer_settings.json.tpl.in
create mode 100644 tests/automatic/file_transfer_service/rest_api/authorizer_settings.json.tpl.in
diff --git a/CMakeModules/prepare_asapo.cmake b/CMakeModules/prepare_asapo.cmake
index f3d264437..6249467e5 100644
--- a/CMakeModules/prepare_asapo.cmake
+++ b/CMakeModules/prepare_asapo.cmake
@@ -7,6 +7,17 @@ function(prepare_asapo)
get_target_property(BROKER_FULLPATH asapo-broker EXENAME)
set(WORK_DIR ${CMAKE_CURRENT_BINARY_DIR})
+ file(TO_NATIVE_PATH ${CMAKE_CURRENT_BINARY_DIR}/asap3 ASAP3_FOLDER )
+ file(TO_NATIVE_PATH ${CMAKE_CURRENT_BINARY_DIR}/beamline CURRENT_BEAMLINES_FOLDER )
+
+ if (WIN32)
+ string(REPLACE "\\" "\\\\" ASAP3_FOLDER "${ASAP3_FOLDER}")
+ string(REPLACE "\\" "\\\\" CURRENT_BEAMLINES_FOLDER "${CURRENT_BEAMLINES_FOLDER}")
+ endif()
+
+ set (ASAP3_FOLDER "${ASAP3_FOLDER}" PARENT_SCOPE)
+ set (CURRENT_BEAMLINES_FOLDER "${CURRENT_BEAMLINES_FOLDER}" PARENT_SCOPE)
+
if(NOT DEFINED RECEIVER_USE_CACHE)
set(RECEIVER_USE_CACHE true)
endif()
diff --git a/authorizer/src/asapo_authorizer/server/authorize.go b/authorizer/src/asapo_authorizer/server/authorize.go
index 023dc3878..42ee3cd02 100644
--- a/authorizer/src/asapo_authorizer/server/authorize.go
+++ b/authorizer/src/asapo_authorizer/server/authorize.go
@@ -89,7 +89,7 @@ func beamtimeMetaFromMatch(match string) (beamtimeMeta, error) {
return beamtimeMeta{}, errors.New("skipped fodler")
}
- bt.OfflinePath = match
+ bt.OfflinePath = settings.RootBeamtimesFolder+string(filepath.Separator)+match
bt.Beamline, bt.BeamtimeId = vars[2], vars[5]
return bt, nil
diff --git a/authorizer/src/asapo_authorizer/server/authorize_test.go b/authorizer/src/asapo_authorizer/server/authorize_test.go
index e4f2d89cf..ca2c21f76 100644
--- a/authorizer/src/asapo_authorizer/server/authorize_test.go
+++ b/authorizer/src/asapo_authorizer/server/authorize_test.go
@@ -148,7 +148,7 @@ func TestAuthorizeWithToken(t *testing.T) {
assert.Contains(t, body_str, "tf/gpfs/bl1/2019/data/test", "")
if (test.beamtime_id == "test_online") {
assert.Contains(t, body_str, "tf/gpfs/bl1/2019/data/test_online", "")
- assert.Contains(t, body_str, "./bl1/current", "")
+ assert.Contains(t, body_str, "bl1/current", "")
} else {
assert.NotContains(t, body_str, "current", "")
}
@@ -339,7 +339,7 @@ func TestGetBeamtimeInfo(t *testing.T) {
settings.RootBeamtimesFolder=test.root
bt,err:= beamtimeMetaFromMatch(test.root+string(filepath.Separator)+test.fname)
if test.ok {
- assert.Equal(t,bt.OfflinePath,test.fname)
+ assert.Equal(t,bt.OfflinePath,test.root+string(filepath.Separator)+test.fname)
assert.Equal(t,bt.Beamline,test.beamline)
assert.Equal(t,bt.BeamtimeId,test.id)
assert.Nil(t,err,"should not be error")
diff --git a/authorizer/src/asapo_authorizer/server/folder_token.go b/authorizer/src/asapo_authorizer/server/folder_token.go
index b72fc1a86..8c50ae062 100644
--- a/authorizer/src/asapo_authorizer/server/folder_token.go
+++ b/authorizer/src/asapo_authorizer/server/folder_token.go
@@ -6,6 +6,7 @@ import (
"time"
log "asapo_common/logger"
"errors"
+ "path/filepath"
)
type folderTokenRequest struct {
@@ -64,6 +65,23 @@ func extractFolderTokenrequest(r *http.Request) (folderTokenRequest,error) {
}
+func checkBeamtimeFolder(request folderTokenRequest) error {
+ beamtimeMeta, err := findMeta(SourceCredentials{request.BeamtimeId,"auto","",""})
+ if err != nil {
+ log.Error("cannot get beamtime meta"+err.Error())
+ return err
+ }
+
+ folder := filepath.Clean(request.Folder)
+ if (folder != filepath.Clean(beamtimeMeta.OnlinePath) && folder != filepath.Clean(beamtimeMeta.OfflinePath)) {
+ err_string := folder + " does not match beamtime folders "+beamtimeMeta.OnlinePath+" or " +beamtimeMeta.OfflinePath
+ log.Error(err_string)
+ return errors.New(err_string)
+ }
+
+ return nil
+}
+
func routeFolderToken(w http.ResponseWriter, r *http.Request) {
request, err := extractFolderTokenrequest(r)
if err != nil {
@@ -77,12 +95,20 @@ func routeFolderToken(w http.ResponseWriter, r *http.Request) {
return
}
+ err = checkBeamtimeFolder(request)
+ if err != nil {
+ utils.WriteServerError(w,err,http.StatusUnauthorized)
+ return
+ }
+
token, err := prepareJWTToken(request)
if err != nil {
utils.WriteServerError(w,err,http.StatusInternalServerError)
return
}
+ log.Debug("generated folder token for beamtime " + request.BeamtimeId + ", folder " + request.Folder)
+
answer := folderTokenResponce(token)
w.WriteHeader(http.StatusOK)
w.Write(answer)
diff --git a/authorizer/src/asapo_authorizer/server/folder_token_test.go b/authorizer/src/asapo_authorizer/server/folder_token_test.go
index d8fc02db9..dbd48e2c4 100644
--- a/authorizer/src/asapo_authorizer/server/folder_token_test.go
+++ b/authorizer/src/asapo_authorizer/server/folder_token_test.go
@@ -6,36 +6,59 @@ import (
"io/ioutil"
"net/http"
"testing"
+ "os"
+ "path/filepath"
+ "fmt"
)
var fodlerTokenTests = [] struct {
beamtime_id string
+ root_folder string
token string
status int
message string
}{
- {"11111111", prepareToken("11111111"),http.StatusOK,"beamtime found"},
- {"11111111", prepareToken("11111112"),http.StatusUnauthorized,"wrong token"},
- {"11111111", prepareToken("11111111"),http.StatusBadRequest,"bad request"},
-
+ {"test", "tf/gpfs/bl1/2019/data/test",prepareToken("test"),http.StatusOK,"beamtime found"},
+ {"test_online", "bl1/current",prepareToken("test_online"),http.StatusOK,"online beamtime found"},
+ {"test", "bl1/current",prepareToken("test"),http.StatusUnauthorized,"no online beamtime found"},
+ {"test_online", "bl2/current",prepareToken("test_online"),http.StatusUnauthorized,"wrong online folder"},
+ {"test", "tf/gpfs/bl1/2019/data/test1",prepareToken("test"),http.StatusUnauthorized,"wrong folder"},
+ {"test", "tf/gpfs/bl1/2019/data/test",prepareToken("test1"),http.StatusUnauthorized,"wrong token"},
+ {"11111111", "tf/gpfs/bl1/2019/data/test",prepareToken("11111111"),http.StatusBadRequest,"bad request"},
}
func TestFolderToken(t *testing.T) {
+ allowBeamlines([]beamtimeMeta{})
+ settings.RootBeamtimesFolder ="."
+ settings.CurrentBeamlinesFolder="."
+ os.MkdirAll(filepath.Clean("tf/gpfs/bl1/2019/data/test"), os.ModePerm)
+ os.MkdirAll(filepath.Clean("tf/gpfs/bl1/2019/data/test_online"), os.ModePerm)
+
+ os.MkdirAll(filepath.Clean("bl1/current"), os.ModePerm)
+ ioutil.WriteFile(filepath.Clean("bl1/current/beamtime-metadata-test_online.json"), []byte(beamtime_meta_online), 0644)
+
+ defer os.RemoveAll("tf")
+ defer os.RemoveAll("bl1")
+
for _, test := range fodlerTokenTests {
- root_folder := "/abc/def"
authJWT = utils.NewJWTAuth("secret")
- request := makeRequest(folderTokenRequest{root_folder,test.beamtime_id,test.token})
+ abs_path:=settings.RootBeamtimesFolder + string(filepath.Separator)+test.root_folder
+ request := makeRequest(folderTokenRequest{abs_path,test.beamtime_id,test.token})
if test.status == http.StatusBadRequest {
request =makeRequest(authorizationRequest{})
}
w := doPostRequest("/folder",request)
- if test.status == http.StatusOK {
+ if w.Code == http.StatusOK {
body, _ := ioutil.ReadAll(w.Body)
claims,_ := utils.CheckJWTToken(string(body),"secret")
var extra_claim utils.FolderTokenTokenExtraClaim
utils.MapToStruct(claims.(*utils.CustomClaims).ExtraClaims.(map[string]interface{}), &extra_claim)
- assert.Equal(t, root_folder, extra_claim.RootFolder, test.message)
+ assert.Equal(t, abs_path, extra_claim.RootFolder, test.message)
+ } else {
+ body, _ := ioutil.ReadAll(w.Body)
+ fmt.Println(string(body))
}
+
assert.Equal(t, test.status, w.Code, test.message)
}
}
diff --git a/tests/automatic/consumer/consumer_api_python/CMakeLists.txt b/tests/automatic/consumer/consumer_api_python/CMakeLists.txt
index 7d598dab6..f63e4734a 100644
--- a/tests/automatic/consumer/consumer_api_python/CMakeLists.txt
+++ b/tests/automatic/consumer/consumer_api_python/CMakeLists.txt
@@ -9,5 +9,8 @@ else()
get_target_property(PYTHON_LIBS asapo_consumer BINARY_DIR)
endif()
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/authorizer_settings.json.tpl.in authorizer.json.tpl @ONLY)
+
+
add_script_test("${TARGET_NAME}" "${PYTHON_LIBS} ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}" nomem)
diff --git a/tests/automatic/consumer/consumer_api_python/authorizer_settings.json.tpl.in b/tests/automatic/consumer/consumer_api_python/authorizer_settings.json.tpl.in
new file mode 100644
index 000000000..7b88592e4
--- /dev/null
+++ b/tests/automatic/consumer/consumer_api_python/authorizer_settings.json.tpl.in
@@ -0,0 +1,8 @@
+{
+ "Port": {{ env "NOMAD_PORT_authorizer" }},
+ "LogLevel":"debug",
+ "RootBeamtimesFolder":"@ASAP3_FOLDER@",
+ "CurrentBeamlinesFolder":"@CURRENT_BEAMLINES_FOLDER@",
+ "SecretFile":"auth_secret.key",
+ "TokenDurationMin":600
+}
diff --git a/tests/automatic/consumer/consumer_api_python/check_linux.sh b/tests/automatic/consumer/consumer_api_python/check_linux.sh
index 3084c8cb2..04f3c058b 100644
--- a/tests/automatic/consumer/consumer_api_python/check_linux.sh
+++ b/tests/automatic/consumer/consumer_api_python/check_linux.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
-source_path=/tmp/asapo/consumer_test/files
beamtime_id=test_run
+source_path=`pwd`/asap3/petra3/gpfs/p01/2019/data/$beamtime_id
stream=detector
database_name=${beamtime_id}_${stream}
token_test_run=K38Mqc90iRv8fC7prcFHd994mF_wfUiJnWBfIjIzieo=
diff --git a/tests/automatic/consumer/consumer_api_python/check_windows.bat b/tests/automatic/consumer/consumer_api_python/check_windows.bat
index e707efb12..86d08b03e 100644
--- a/tests/automatic/consumer/consumer_api_python/check_windows.bat
+++ b/tests/automatic/consumer/consumer_api_python/check_windows.bat
@@ -1,6 +1,8 @@
-SET source_path=c:\\tmp\\asapo\\consumer_test\\files
-
+setlocal
SET beamtime_id=test_run
+SET source_path=%cd%\asap3\petra3\gpfs\p01\2019\data\%beamtime_id%
+set source_path=%source_path:\=\\%
+
SET stream=detector
SET database_name=%beamtime_id%_%stream%
diff --git a/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt b/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt
index 5b93903a5..506902ea3 100644
--- a/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt
+++ b/tests/automatic/curl_http_client/curl_http_client_command/CMakeLists.txt
@@ -20,4 +20,6 @@ target_link_libraries(${TARGET_NAME} test_common asapo-consumer)
prepare_asapo()
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/authorizer_settings.json.tpl.in authorizer.json.tpl @ONLY)
+
add_script_test("${TARGET_NAME}" "$<TARGET_FILE:${TARGET_NAME}>" nomem)
diff --git a/tests/automatic/curl_http_client/curl_http_client_command/authorizer_settings.json.tpl.in b/tests/automatic/curl_http_client/curl_http_client_command/authorizer_settings.json.tpl.in
new file mode 100644
index 000000000..7b88592e4
--- /dev/null
+++ b/tests/automatic/curl_http_client/curl_http_client_command/authorizer_settings.json.tpl.in
@@ -0,0 +1,8 @@
+{
+ "Port": {{ env "NOMAD_PORT_authorizer" }},
+ "LogLevel":"debug",
+ "RootBeamtimesFolder":"@ASAP3_FOLDER@",
+ "CurrentBeamlinesFolder":"@CURRENT_BEAMLINES_FOLDER@",
+ "SecretFile":"auth_secret.key",
+ "TokenDurationMin":600
+}
diff --git a/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh b/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh
index 032a91d37..989a4562e 100644
--- a/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh
+++ b/tests/automatic/curl_http_client/curl_http_client_command/check_linux.sh
@@ -4,7 +4,8 @@ set -e
trap Cleanup EXIT
-file_transfer_folder=/tmp/asapo/file_transfer/files
+beamtime_id=aaa
+file_transfer_folder=`pwd`/asap3/petra3/gpfs/p01/2019/data/$beamtime_id
Cleanup() {
echo cleanup
diff --git a/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat b/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat
index 6e9245472..1748f6e8c 100644
--- a/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat
+++ b/tests/automatic/curl_http_client/curl_http_client_command/check_windows.bat
@@ -1,4 +1,9 @@
-SET file_transfer_folder=c:\\tmp\\asapo\\file_transfer\\files
+setlocal
+SET beamtime_id=aaa
+SET file_transfer_folder=%cd%\asap3\petra3\gpfs\p01\2019\data\%beamtime_id%
+set file_transfer_folder=%file_transfer_folder:\=\\%
+
+
c:\opt\consul\nomad run authorizer.nmd
c:\opt\consul\nomad run file_transfer.nmd
diff --git a/tests/automatic/file_transfer_service/rest_api/CMakeLists.txt b/tests/automatic/file_transfer_service/rest_api/CMakeLists.txt
index 1f550cc4e..7ead397a7 100644
--- a/tests/automatic/file_transfer_service/rest_api/CMakeLists.txt
+++ b/tests/automatic/file_transfer_service/rest_api/CMakeLists.txt
@@ -3,7 +3,7 @@ set(TARGET_NAME file_transfer_rest_api)
################################
# Testing
################################
-
prepare_asapo()
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/authorizer_settings.json.tpl.in authorizer.json.tpl @ONLY)
add_script_test("${TARGET_NAME}" "" nomem)
diff --git a/tests/automatic/file_transfer_service/rest_api/authorizer_settings.json.tpl.in b/tests/automatic/file_transfer_service/rest_api/authorizer_settings.json.tpl.in
new file mode 100644
index 000000000..7b88592e4
--- /dev/null
+++ b/tests/automatic/file_transfer_service/rest_api/authorizer_settings.json.tpl.in
@@ -0,0 +1,8 @@
+{
+ "Port": {{ env "NOMAD_PORT_authorizer" }},
+ "LogLevel":"debug",
+ "RootBeamtimesFolder":"@ASAP3_FOLDER@",
+ "CurrentBeamlinesFolder":"@CURRENT_BEAMLINES_FOLDER@",
+ "SecretFile":"auth_secret.key",
+ "TokenDurationMin":600
+}
diff --git a/tests/automatic/file_transfer_service/rest_api/check_linux.sh b/tests/automatic/file_transfer_service/rest_api/check_linux.sh
index b0bb0f327..89f45c4c8 100644
--- a/tests/automatic/file_transfer_service/rest_api/check_linux.sh
+++ b/tests/automatic/file_transfer_service/rest_api/check_linux.sh
@@ -4,7 +4,7 @@ set -e
trap Cleanup EXIT
-file_transfer_folder=/tmp/asapo/file_transfer/files
+file_transfer_folder=`pwd`/asap3/petra3/gpfs/p01/2019/data/aaa
Cleanup() {
@@ -19,10 +19,12 @@ nomad run file_transfer.nmd
sleep 1
+mkdir -p $file_transfer_folder
+
token=bnCXpOdBV90wU1zybEw1duQNSORuwaKz6oDHqmL35p0= #token for aaa
folder_token=`curl --silent --data "{\"Folder\":\"$file_transfer_folder\",\"BeamtimeId\":\"aaa\",\"Token\":\"$token\"}" 127.0.0.1:5007/folder`
+echo $folder_token
-mkdir -p $file_transfer_folder
echo hello > $file_transfer_folder/aaa
curl -o aaa --silent -H "Authorization: Bearer ${folder_token}" --data "{\"Folder\":\"$file_transfer_folder\",\"FileName\":\"aaa\",\"Token\":\"$folder_token\"}" 127.0.0.1:5008/transfer --stderr - | tee /dev/stderr
diff --git a/tests/automatic/file_transfer_service/rest_api/check_windows.bat b/tests/automatic/file_transfer_service/rest_api/check_windows.bat
index cd940974f..a193eaefc 100644
--- a/tests/automatic/file_transfer_service/rest_api/check_windows.bat
+++ b/tests/automatic/file_transfer_service/rest_api/check_windows.bat
@@ -1,4 +1,8 @@
-SET file_transfer_folder=c:\\tmp\\asapo\\file_transfer\\files
+setlocal
+SET beamtime_id=aaa
+SET file_transfer_folder=%cd%\asap3\petra3\gpfs\p01\2019\data\%beamtime_id%
+set file_transfer_folder=%file_transfer_folder:\=\\%
+
c:\opt\consul\nomad run authorizer.nmd
c:\opt\consul\nomad run file_transfer.nmd
@@ -7,11 +11,11 @@ ping 1.0.0.0 -n 1 -w 100 > nul
set token=bnCXpOdBV90wU1zybEw1duQNSORuwaKz6oDHqmL35p0=
+mkdir %file_transfer_folder%
+
C:\Curl\curl.exe --silent --data "{\"Folder\":\"%file_transfer_folder%\",\"BeamtimeId\":\"aaa\",\"Token\":\"%token%\"}" 127.0.0.1:5007/folder > token
set /P folder_token=< token
-mkdir %file_transfer_folder%
-
echo hello > %file_transfer_folder%\aaa
C:\Curl\curl.exe --silent -H "Authorization: Bearer %folder_token%" --data "{\"Folder\":\"%file_transfer_folder%\",\"FileName\":\"aaa\",\"Token\":\"%folder_token%\"}" 127.0.0.1:5008/transfer --stderr - | findstr hello || goto :error
--
GitLab