diff --git a/CHANGELOG.md b/CHANGELOG.md index f7df3b7236360ac97cbd7d1fd2ede6e8f3add914..258270253dcdf7ba8295289316473cc28211551c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ ## 21.06.0 (in progress) IMPROVEMENTS -* Consumer/Producer API - allow any characters in source/stream/group names +* Consumer/Producer API - allow any characters in source/stream/group names +* Consumer API - an option to auto discovery of data folder when consumer client uses file transfer service (has_filesystem=False) BUG FIXES * Consumer API: multiple consumers from same group receive stream finished error diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ac928819283903b0d364efd8e614df3c37a434b..fc1c853904b53c2b72f3d5be1a63a5c64f40b7d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,9 +6,9 @@ set (ASAPO_CONSUMER_PROTOCOL "v0.4") set (ASAPO_PRODUCER_PROTOCOL "v0.3") set (ASAPO_DISCOVERY_API_VER "v0.1") -set (ASAPO_AUTHORIZER_API_VER "v0.1") +set (ASAPO_AUTHORIZER_API_VER "v0.2") set (ASAPO_BROKER_API_VER "v0.4") -set (ASAPO_FILE_TRANSFER_SERVICE_API_VER "v0.1") +set (ASAPO_FILE_TRANSFER_SERVICE_API_VER "v0.2") set (ASAPO_RECEIVER_API_VER "v0.3") set (ASAPO_RDS_API_VER "v0.1") diff --git a/authorizer/src/asapo_authorizer/server/folder_token.go b/authorizer/src/asapo_authorizer/server/folder_token.go index 6b80193478187250f1d14df00f754bcd6bde5181..dcdfbe38ba74631c5c31ad96dacef18a474dd6ca 100644 --- a/authorizer/src/asapo_authorizer/server/folder_token.go +++ b/authorizer/src/asapo_authorizer/server/folder_token.go @@ -17,6 +17,11 @@ type folderTokenRequest struct { Token string } +type tokenFolders struct { + RootFolder string + SecondFolder string +} + type folderToken struct { Token string } @@ -25,11 +30,13 @@ type folderToken struct { utils.ProcessJWTAuth(processFolderTokenRequest,settings.secret)(w,r) }*/ -func prepareJWTToken(request folderTokenRequest) (string, error) { +func prepareJWTToken(folders tokenFolders) (string, error) { var claims utils.CustomClaims var extraClaim structs.FolderTokenTokenExtraClaim - extraClaim.RootFolder = request.Folder + extraClaim.RootFolder = folders.RootFolder + extraClaim.SecondFolder = folders.SecondFolder + claims.ExtraClaims = &extraClaim claims.SetExpiration(time.Duration(settings.FolderTokenDurationMin) * time.Minute) return Auth.JWTAuth().GenerateToken(&claims) @@ -59,30 +66,37 @@ func extractFolderTokenrequest(r *http.Request) (folderTokenRequest, error) { } -func checkBeamtimeFolder(request folderTokenRequest) error { +func checkBeamtimeFolder(request folderTokenRequest, ver utils.VersionNum) (folders tokenFolders, err error) { beamtimeMeta, err := findMeta(SourceCredentials{request.BeamtimeId, "auto", "", "", ""}) if err != nil { log.Error("cannot get beamtime meta" + err.Error()) - return err + return folders,err + } + + if request.Folder=="auto" && ver.Id > 1 { + folders.RootFolder = beamtimeMeta.OfflinePath + folders.SecondFolder = beamtimeMeta.OnlinePath + return folders,nil } folder := filepath.Clean(request.Folder) if folder != filepath.Clean(beamtimeMeta.OnlinePath) && folder != filepath.Clean(beamtimeMeta.OfflinePath) { err_string := folder + " does not match beamtime folders " + beamtimeMeta.OnlinePath + " or " + beamtimeMeta.OfflinePath log.Error(err_string) - return errors.New(err_string) + return folders,errors.New(err_string) } - return nil + folders.RootFolder = request.Folder + return folders,nil } -func checkAuthorizerApiVersion(w http.ResponseWriter, r *http.Request) bool { - _, ok := utils.PrecheckApiVersion(w, r, version.GetAuthorizerApiVersion()) - return ok +func checkAuthorizerApiVersion(w http.ResponseWriter, r *http.Request) (utils.VersionNum, bool) { + return utils.PrecheckApiVersion(w, r, version.GetAuthorizerApiVersion()) } func routeFolderToken(w http.ResponseWriter, r *http.Request) { - if ok := checkAuthorizerApiVersion(w, r); !ok { + ver,ok := checkAuthorizerApiVersion(w, r); + if !ok { return } @@ -98,13 +112,13 @@ func routeFolderToken(w http.ResponseWriter, r *http.Request) { return } - err = checkBeamtimeFolder(request) + folders,err := checkBeamtimeFolder(request,ver) if err != nil { utils.WriteServerError(w, err, http.StatusUnauthorized) return } - token, err := prepareJWTToken(request) + token, err := prepareJWTToken(folders) if err != nil { utils.WriteServerError(w, err, http.StatusInternalServerError) return diff --git a/authorizer/src/asapo_authorizer/server/folder_token_test.go b/authorizer/src/asapo_authorizer/server/folder_token_test.go index ee9ab2f5088474e753e811c6440196bfa6f2d0ba..d5cf1f6c195904dbac3608cd1064d4ad5fc80a0d 100644 --- a/authorizer/src/asapo_authorizer/server/folder_token_test.go +++ b/authorizer/src/asapo_authorizer/server/folder_token_test.go @@ -4,6 +4,7 @@ import ( "asapo_authorizer/authorization" "asapo_common/structs" "asapo_common/utils" + "asapo_common/version" "github.com/stretchr/testify/assert" "io/ioutil" "net/http" @@ -14,19 +15,25 @@ import ( ) var fodlerTokenTests = [] struct { - beamtime_id string - root_folder string - token string - status int - message string + beamtime_id string + auto bool + root_folder string + second_folder string + token string + status int + message string }{ - {"test", "tf/gpfs/bl1/2019/data/test", prepareUserToken("bt_test",[]string{"read"}),http.StatusOK,"beamtime found"}, - {"test_online", "bl1/current", prepareUserToken("bt_test_online",[]string{"read"}),http.StatusOK,"online beamtime found"}, - {"test", "bl1/current", prepareUserToken("bt_test",[]string{"read"}),http.StatusUnauthorized,"no online beamtime found"}, - {"test_online", "bl2/current", prepareUserToken("bt_test_online",[]string{"read"}),http.StatusUnauthorized,"wrong online folder"}, - {"test", "tf/gpfs/bl1/2019/data/test1", prepareUserToken("bt_test",[]string{"read"}),http.StatusUnauthorized,"wrong folder"}, - {"test", "tf/gpfs/bl1/2019/data/test", prepareUserToken("bt_test1",[]string{"read"}),http.StatusUnauthorized,"wrong token"}, - {"11111111", "tf/gpfs/bl1/2019/data/test", prepareUserToken("bt_11111111",[]string{"read"}),http.StatusBadRequest,"bad request"}, + {"test", false,"tf/gpfs/bl1/2019/data/test", "",prepareUserToken("bt_test",[]string{"read"}),http.StatusOK,"beamtime found"}, + {"test_online",false, "bl1/current", "",prepareUserToken("bt_test_online",[]string{"read"}),http.StatusOK,"online beamtime found"}, + {"test", false,"bl1/current", "",prepareUserToken("bt_test",[]string{"read"}),http.StatusUnauthorized,"no online beamtime found"}, + {"test_online",false, "bl2/current", "",prepareUserToken("bt_test_online",[]string{"read"}),http.StatusUnauthorized,"wrong online folder"}, + {"test", false,"tf/gpfs/bl1/2019/data/test1", "",prepareUserToken("bt_test",[]string{"read"}),http.StatusUnauthorized,"wrong folder"}, + {"test", false,"tf/gpfs/bl1/2019/data/test", "",prepareUserToken("bt_test1",[]string{"read"}),http.StatusUnauthorized,"wrong token"}, + {"11111111", false,"tf/gpfs/bl1/2019/data/test", "",prepareUserToken("bt_11111111",[]string{"read"}),http.StatusBadRequest,"bad request"}, + + {"test", true,"tf/gpfs/bl1/2019/data/test", "",prepareUserToken("bt_test",[]string{"read"}),http.StatusOK,"auto without onilne"}, + {"test_online",true, "tf/gpfs/bl1/2019/data/test_online", "bl1/current",prepareUserToken("bt_test_online",[]string{"read"}),http.StatusOK,"auto with online"}, + } func TestFolderToken(t *testing.T) { @@ -46,17 +53,26 @@ func TestFolderToken(t *testing.T) { for _, test := range fodlerTokenTests { abs_path:=settings.RootBeamtimesFolder + string(filepath.Separator)+test.root_folder - request := makeRequest(folderTokenRequest{abs_path,test.beamtime_id,test.token}) + abs_path_second :="" + if test.second_folder!="" { + abs_path_second =settings.RootBeamtimesFolder + string(filepath.Separator)+test.second_folder + } + path_in_token:=abs_path + if test.auto { + path_in_token = "auto" + } + request := makeRequest(folderTokenRequest{path_in_token,test.beamtime_id,test.token}) if test.status == http.StatusBadRequest { request =makeRequest(authorizationRequest{}) } - w := doPostRequest("/v0.1/folder",request,"") + w := doPostRequest("/"+version.GetAuthorizerApiVersion()+"/folder",request,"") if w.Code == http.StatusOK { body, _ := ioutil.ReadAll(w.Body) claims,_ := utils.CheckJWTToken(string(body),"secret_folder") var extra_claim structs.FolderTokenTokenExtraClaim utils.MapToStruct(claims.(*utils.CustomClaims).ExtraClaims.(map[string]interface{}), &extra_claim) - assert.Equal(t, abs_path, extra_claim.RootFolder, test.message) + assert.Equal(t, filepath.Clean(abs_path), filepath.Clean(extra_claim.RootFolder), test.message) + assert.Equal(t, filepath.Clean(abs_path_second), filepath.Clean(extra_claim.SecondFolder), test.message) } else { body, _ := ioutil.ReadAll(w.Body) fmt.Println(string(body)) @@ -68,7 +84,7 @@ func TestFolderToken(t *testing.T) { func TestFolderTokenWrongProtocol(t *testing.T) { request := makeRequest(folderTokenRequest{"abs_path","beamtime_id","token"}) - w := doPostRequest("/v0.2/folder",request,"") + w := doPostRequest("/v10000.2/folder",request,"") assert.Equal(t, http.StatusUnsupportedMediaType, w.Code, "wrong protocol") } diff --git a/common/go/src/asapo_common/structs/structs.go b/common/go/src/asapo_common/structs/structs.go index e6517c12d67cae9799a098500798a5a663dba3af..e62928a867ea083c7a9258294a7109649bfcee97 100644 --- a/common/go/src/asapo_common/structs/structs.go +++ b/common/go/src/asapo_common/structs/structs.go @@ -2,6 +2,7 @@ package structs type FolderTokenTokenExtraClaim struct { RootFolder string + SecondFolder string } type AccessTokenExtraClaim struct { diff --git a/consumer/api/cpp/unittests/test_consumer_impl.cpp b/consumer/api/cpp/unittests/test_consumer_impl.cpp index 862f2f64e3c77921ca8680bf24c55838f9e4c773..517809314ec2da0d45d9e34fcc967042174d2eeb 100644 --- a/consumer/api/cpp/unittests/test_consumer_impl.cpp +++ b/consumer/api/cpp/unittests/test_consumer_impl.cpp @@ -1159,7 +1159,7 @@ void ConsumerImplTests::ExpectFolderToken() { expected_beamtime_id + "\",\"Token\":\"" + expected_token + "\"}"; - EXPECT_CALL(mock_http_client, Post_t(HasSubstr(expected_server_uri + "/asapo-authorizer/v0.1/folder"), _, + EXPECT_CALL(mock_http_client, Post_t(HasSubstr(expected_server_uri + "/asapo-authorizer/v0.2/folder"), _, expected_folder_query_string, _, _)).WillOnce(DoAll( SetArgPointee<3>(HttpCode::OK), SetArgPointee<4>(nullptr), @@ -1177,7 +1177,7 @@ ACTION_P(AssignArg3, assign) { } void ConsumerImplTests::ExpectFileTransfer(const asapo::ConsumerErrorTemplate* p_err_template) { - EXPECT_CALL(mock_http_client, PostReturnArray_t(HasSubstr(expected_fts_uri + "/v0.1/transfer"), + EXPECT_CALL(mock_http_client, PostReturnArray_t(HasSubstr(expected_fts_uri + "/v0.2/transfer"), expected_cookie, expected_fts_query_string, _, @@ -1190,7 +1190,7 @@ void ConsumerImplTests::ExpectFileTransfer(const asapo::ConsumerErrorTemplate* p } void ConsumerImplTests::ExpectRepeatedFileTransfer() { - EXPECT_CALL(mock_http_client, PostReturnArray_t(HasSubstr(expected_fts_uri + "/v0.1/transfer"), + EXPECT_CALL(mock_http_client, PostReturnArray_t(HasSubstr(expected_fts_uri + "/v0.2/transfer"), expected_cookie, expected_fts_query_string, _, @@ -1235,7 +1235,7 @@ TEST_F(ConsumerImplTests, FileTransferReadsFileSize) { Return("{\"file_size\":5}") )); - EXPECT_CALL(mock_http_client, PostReturnArray_t(HasSubstr(expected_fts_uri + "/v0.1/transfer"), + EXPECT_CALL(mock_http_client, PostReturnArray_t(HasSubstr(expected_fts_uri + "/v0.2/transfer"), expected_cookie, expected_fts_query_string, _, diff --git a/discovery/src/asapo_discovery/protocols/hard_coded_consumer.go b/discovery/src/asapo_discovery/protocols/hard_coded_consumer.go index 1410b30a9c767aef2c441de9cb8e7bb76ef76e26..8c91240a3c47dfa0c0c492e08e15645632f10150 100644 --- a/discovery/src/asapo_discovery/protocols/hard_coded_consumer.go +++ b/discovery/src/asapo_discovery/protocols/hard_coded_consumer.go @@ -15,9 +15,9 @@ func GetSupportedConsumerProtocols() []Protocol { Protocol{"v0.4", map[string]string{ "Discovery": "v0.1", - "Authorizer": "v0.1", + "Authorizer": "v0.2", "Broker": "v0.4", - "File Transfer": "v0.1", + "File Transfer": "v0.2", "Data cache service": "v0.1", }, &protocolValidatorCurrent{}}, Protocol{"v0.3", diff --git a/file_transfer/src/asapo_file_transfer/server/transfer.go b/file_transfer/src/asapo_file_transfer/server/transfer.go index cb50067dca71a6fcb13bb08ee4b0a9c0c03e22d2..2e0cfb92755eae706c617ecb038d11664ad45c6d 100644 --- a/file_transfer/src/asapo_file_transfer/server/transfer.go +++ b/file_transfer/src/asapo_file_transfer/server/transfer.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "path" + "path/filepath" "strconv" ) @@ -25,11 +26,16 @@ func Exists(name string) bool { return err==nil } -func checkClaim(r *http.Request,request* fileTransferRequest) (int,error) { +func checkClaim(r *http.Request,ver utils.VersionNum,request* fileTransferRequest) (int,error) { var extraClaim structs.FolderTokenTokenExtraClaim if err := utils.JobClaimFromContext(r, nil, &extraClaim); err != nil { return http.StatusInternalServerError,err } + if ver.Id > 1 { + request.Folder = extraClaim.RootFolder + return http.StatusOK,nil + } + if extraClaim.RootFolder!=request.Folder { err_txt := "access forbidden for folder "+request.Folder log.Error("cannot transfer file: "+err_txt) @@ -48,17 +54,23 @@ func checkFileExists(r *http.Request,name string) (int,error) { } -func checkRequest(r *http.Request) (string,int,error) { +func checkRequest(r *http.Request, ver utils.VersionNum) (string,int,error) { var request fileTransferRequest err := utils.ExtractRequest(r,&request) if err != nil { return "",http.StatusBadRequest,err } - if status,err := checkClaim(r,&request); err != nil { + if status,err := checkClaim(r,ver, &request); err != nil { return "",status,err } - fullName := request.Folder+string(os.PathSeparator)+request.FileName + var fullName string + if ver.Id == 1 { // protocol v0.1 + fullName = filepath.Clean(request.Folder+string(os.PathSeparator)+request.FileName) + } else { + fullName = filepath.Clean(request.Folder+string(os.PathSeparator)+request.FileName) + } + if status,err := checkFileExists(r,fullName); err != nil { return "",status,err } @@ -90,17 +102,17 @@ func serveFileSize(w http.ResponseWriter, r *http.Request, fullName string) { } -func checkFtsApiVersion(w http.ResponseWriter, r *http.Request) bool { - _, ok := utils.PrecheckApiVersion(w, r, version.GetFtsApiVersion()) - return ok +func checkFtsApiVersion(w http.ResponseWriter, r *http.Request) (utils.VersionNum,bool) { + return utils.PrecheckApiVersion(w, r, version.GetFtsApiVersion()) } func routeFileTransfer(w http.ResponseWriter, r *http.Request) { - if ok := checkFtsApiVersion(w, r); !ok { + ver, ok := checkFtsApiVersion(w, r); + if !ok { return } - fullName, status,err := checkRequest(r); + fullName, status,err := checkRequest(r,ver); if err != nil { utils.WriteServerError(w,err,status) return diff --git a/file_transfer/src/asapo_file_transfer/server/transfer_test.go b/file_transfer/src/asapo_file_transfer/server/transfer_test.go index d4117dcfd9a71939914f3f7227c8d18b148320ab..1b381ca4382293e69a5dfbe82756b84ea56c8acd 100644 --- a/file_transfer/src/asapo_file_transfer/server/transfer_test.go +++ b/file_transfer/src/asapo_file_transfer/server/transfer_test.go @@ -106,6 +106,6 @@ func TestTransferFileSize(t *testing.T) { func TestTransferWrongApiVersion(t *testing.T) { request := makeRequest(fileTransferRequest{"folder","fname"}) token := prepareToken("folder") - w := doPostRequest("/v0.2/transfer?sizeonly=true",request,token) + w := doPostRequest("/v100000.2/transfer?sizeonly=true",request,token) assert.Equal(t, http.StatusUnsupportedMediaType, w.Code, "wrong api version") } diff --git a/tests/automatic/file_transfer_service/rest_api/check_linux.sh b/tests/automatic/file_transfer_service/rest_api/check_linux.sh index fa404b575a2034e88c4dfa0f3498091433831c35..edd06164338e04d7d994ca0d12c23ed1d2a21ee1 100644 --- a/tests/automatic/file_transfer_service/rest_api/check_linux.sh +++ b/tests/automatic/file_transfer_service/rest_api/check_linux.sh @@ -9,7 +9,7 @@ file_transfer_folder=/tmp/asapo/asap3/petra3/gpfs/p01/2019/data/aaa Cleanup() { echo cleanup - rm -rf $file_transfer_folder aaa big_file + rm -rf $file_transfer_folder aaa aaa1 big_file } mkdir -p $file_transfer_folder @@ -28,6 +28,20 @@ curl -H "Authorization: Bearer ${folder_token}" --data "{\"Folder\":\"$file_tran diff -q aaa $file_transfer_folder/aaa +# auto folder +folder_token_auto=`curl --silent --data "{\"Folder\":\"auto\",\"BeamtimeId\":\"aaa\",\"Token\":\"$token\"}" 127.0.0.1:5007/v0.2/folder` +echo $folder_token_auto +curl -o aaa1 --silent -H "Authorization: Bearer ${folder_token_auto}" --data "{\"FileName\":\"aaa\",\"Token\":\"$folder_token_auto\"}" 127.0.0.1:5008/v0.2/transfer --stderr - | tee /dev/stderr +diff -q aaa1 $file_transfer_folder/aaa + +#auto folder, old protocol +curl --silent --data "{\"Folder\":\"auto\",\"BeamtimeId\":\"aaa\",\"Token\":\"$token\"}" 127.0.0.1:5007/v0.1/folder | grep "auto does not match" +curl --silent -H "Authorization: Bearer ${folder_token_auto}" --data "{\"FileName\":\"aaa\",\"Token\":\"$folder_token_auto\"}" 127.0.0.1:5008/v0.1/transfer --stderr - | tee /dev/stderr | grep forbidden + +exit 0 + + + chmod -r $file_transfer_folder/aaa curl --silent -H "Authorization: Bearer ${folder_token}" --data "{\"Folder\":\"$file_transfer_folder\",\"FileName\":\"aaa\",\"Token\":\"$folder_token\"}" 127.0.0.1:5008/v0.1/transfer?sizeonly=true --stderr - | tee /dev/stderr | grep "does not exist" @@ -38,5 +52,3 @@ curl -vvv -o big_file -H "Authorization: Bearer ${folder_token}" --data "{\"Fold ls -ln big_file | awk '{ print $5 }' | tee /dev/stderr | grep 5368709120 - - diff --git a/tests/automatic/file_transfer_service/rest_api/check_windows.bat b/tests/automatic/file_transfer_service/rest_api/check_windows.bat index 047b29362cc612dfcf2e0644bd46d098d34994cd..a7712bb9706d8591a66037f90b13776afb681fa2 100644 --- a/tests/automatic/file_transfer_service/rest_api/check_windows.bat +++ b/tests/automatic/file_transfer_service/rest_api/check_windows.bat @@ -7,14 +7,15 @@ set token=%BT_AAA_TOKEN% mkdir %file_transfer_folder% -C:\Curl\curl.exe --silent --data "{\"Folder\":\"%file_transfer_folder%\",\"BeamtimeId\":\"aaa\",\"Token\":\"%token%\"}" 127.0.0.1:5007/v0.1/folder > token +C:\Curl\curl.exe --silent --data "{\"Folder\":\"auto\",\"BeamtimeId\":\"aaa\",\"Token\":\"%token%\"}" 127.0.0.1:5007/v0.2/folder > token set /P folder_token=< token echo hello > %file_transfer_folder%\aaa ping 192.0.2.1 -n 1 -w 1000 > nul -C:\Curl\curl.exe -v --silent -H "Authorization: Bearer %folder_token%" --data "{\"Folder\":\"%file_transfer_folder%\",\"FileName\":\"aaa\",\"Token\":\"%folder_token%\"}" 127.0.0.1:5008/v0.1/transfer -C:\Curl\curl.exe --silent -H "Authorization: Bearer %folder_token%" --data "{\"Folder\":\"%file_transfer_folder%\",\"FileName\":\"aaa\",\"Token\":\"%folder_token%\"}" 127.0.0.1:5008/v0.1/transfer --stderr - | findstr hello || goto :error +C:\Curl\curl.exe -v --silent -H "Authorization: Bearer %folder_token%" --data "{\"FileName\":\"aaa\",\"Token\":\"%folder_token%\"}" 127.0.0.1:5008/v0.2/transfer +C:\Curl\curl.exe --silent -H "Authorization: Bearer %folder_token%" --data "{\"Folder\":\"%file_transfer_folder%\",\"FileName\":\"aaa\",\"Token\":\"%folder_token%\"}" 127.0.0.1:5008/v0.2/transfer --stderr - | findstr hello || goto :error + goto :clean