diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4ad0fbbd1894dee6468cb605e076da4c8df56fc3..0a0fcc607fecc90f7c134e56fcfb32def7cea47c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,14 +2,16 @@
FEATURES
* introduced substreams for producer/consumer
* introduced timeout for producer requests
+* producer accepts "auto" for beamtime, will automatically select a current one for a given beamline
IMPROVEMENTS
* switch to MongoDB 4.2
-* receiver use ASAP3 directory structure to save files to
-* API documentation is available at
+* receiver use file paths provided during connection authorization structure
+* API documentation is available for C++ and Python
* switch to using cmake 3.7+
* error messages in Python as Python strings, not byte objects
+
BUG FIXES
* consumer operation timout - take duration of the operation into account
* giving warning/error on attempt to send data/metadata with same id
\ No newline at end of file
diff --git a/CMakeModules/prepare_asapo.cmake b/CMakeModules/prepare_asapo.cmake
index 9523b6ba3e5f0714b312998e5192a2e882fa18d3..c5163021646a4307e61dc521f47da6ab80e89db3 100644
--- a/CMakeModules/prepare_asapo.cmake
+++ b/CMakeModules/prepare_asapo.cmake
@@ -17,10 +17,11 @@ function(prepare_asapo)
if (WIN32)
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/receiver.json.tpl.win.in receiver.json.tpl @ONLY)
configure_file(${CMAKE_SOURCE_DIR}/config/nomad/nginx_kill_win.nmd nginx_kill.nmd @ONLY)
-
+ configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/authorizer_settings.json.tpl.win authorizer.json.tpl COPYONLY)
else()
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/receiver.json.tpl.lin.in receiver.json.tpl @ONLY)
configure_file(${CMAKE_SOURCE_DIR}/config/nomad/nginx_kill_lin.nmd nginx_kill.nmd @ONLY)
+ configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/authorizer_settings.json.tpl.lin authorizer.json.tpl COPYONLY)
endif()
configure_file(${CMAKE_SOURCE_DIR}/config/nomad/receiver.nmd.in receiver.nmd @ONLY)
@@ -28,7 +29,6 @@ function(prepare_asapo)
configure_file(${CMAKE_SOURCE_DIR}/config/nomad/authorizer.nmd.in authorizer.nmd @ONLY)
configure_file(${CMAKE_SOURCE_DIR}/config/nomad/broker.nmd.in broker.nmd @ONLY)
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/discovery_settings.json.tpl discovery.json.tpl COPYONLY)
- configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/authorizer_settings.json.tpl authorizer.json.tpl COPYONLY)
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/broker_settings.json.tpl broker.json.tpl COPYONLY)
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/auth_secret.key auth_secret.key COPYONLY)
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/nginx.conf.tpl nginx.conf.tpl COPYONLY)
diff --git a/asapo_tools/src/asapo_tools/cli/token.go b/asapo_tools/src/asapo_tools/cli/token.go
index 9282a48e756263c3c6d987e18891d89979f29230..cc68a114f10690966116d9ce74c7b73386bd9e69 100644
--- a/asapo_tools/src/asapo_tools/cli/token.go
+++ b/asapo_tools/src/asapo_tools/cli/token.go
@@ -8,7 +8,7 @@ import (
)
type tokenFlags struct {
- BeamtimeID string
+ Name string
SecretFile string
}
@@ -42,7 +42,7 @@ func (cmd *command) CommandToken() error {
return err
}
- fmt.Fprintf(outBuf, "%s\n", generateToken(flags.BeamtimeID,secret))
+ fmt.Fprintf(outBuf, "%s\n", generateToken(flags.Name,secret))
return nil
}
@@ -51,7 +51,7 @@ func (cmd *command) CommandToken() error {
func (cmd *command) parseTokenFlags(message_string string) (tokenFlags, error) {
var flags tokenFlags
- flagset := cmd.createDefaultFlagset(message_string, "<beamtime id>")
+ flagset := cmd.createDefaultFlagset(message_string, "<token_body>")
flagset.StringVar(&flags.SecretFile, "secret", "", "path to file with secret")
flagset.Parse(cmd.args)
@@ -60,10 +60,10 @@ func (cmd *command) parseTokenFlags(message_string string) (tokenFlags, error) {
os.Exit(0)
}
- flags.BeamtimeID = flagset.Arg(0)
+ flags.Name = flagset.Arg(0)
- if flags.BeamtimeID == "" {
- return flags, errors.New("beamtime id missed ")
+ if flags.Name == "" {
+ return flags, errors.New("beamtime id or beamline missed ")
}
if flags.SecretFile == "" {
diff --git a/authorizer/src/asapo_authorizer/server/authorize.go b/authorizer/src/asapo_authorizer/server/authorize.go
index 624a1405ea6927c60df2c14e953a3713a2880680..9574aa9e3d9566fbe4538fb53e1f0dc297d4a510 100644
--- a/authorizer/src/asapo_authorizer/server/authorize.go
+++ b/authorizer/src/asapo_authorizer/server/authorize.go
@@ -12,26 +12,40 @@ import (
type SourceCredentials struct {
BeamtimeId string
- Stream string
- Token string
+ Beamline string
+ Stream string
+ Token string
}
type authorizationRequest struct {
SourceCredentials string
- OriginHost string
+ OriginHost string
}
-func getSourceCredentials(request authorizationRequest ) (SourceCredentials,error){
- vals := strings.Split(request.SourceCredentials,"%")
+func getSourceCredentials(request authorizationRequest) (SourceCredentials, error) {
+ vals := strings.Split(request.SourceCredentials, "%")
- if len(vals)!=3 {
- return SourceCredentials{}, errors.New("cannot get source credentials from "+request.SourceCredentials)
+ if len(vals) != 4 {
+ return SourceCredentials{}, errors.New("cannot get source credentials from " + request.SourceCredentials)
}
- creds := SourceCredentials{vals[0],vals[1],vals[2]}
- if creds.Stream=="" {
- creds.Stream="detector"
+ creds := SourceCredentials{vals[0], vals[1], vals[2], vals[3]}
+ if creds.Stream == "" {
+ creds.Stream = "detector"
}
- return creds,nil
+
+ if creds.Beamline == "" {
+ creds.Beamline = "auto"
+ }
+
+ if creds.BeamtimeId == "" {
+ creds.BeamtimeId = "auto"
+ }
+
+ if creds.BeamtimeId == "auto" && creds.Beamline == "auto" {
+ return SourceCredentials{}, errors.New("cannot automaticaly detect both beamline and beamtime_id ")
+ }
+
+ return creds, nil
}
func extractRequest(r *http.Request) (request authorizationRequest, err error) {
@@ -59,116 +73,185 @@ func getBeamlineFromIP(ip string) (string, error) {
return lines[0], nil
}
-func beamtimeInfoFromMatch(match string) (beamtimeInfo,error) {
+func beamtimeMetaFromJson(fname string) (beamtimeMeta, error) {
+ var meta beamtimeMeta
+ err := utils.ReadJsonFromFile(fname, &meta)
+ if err != nil {
+ return beamtimeMeta{}, err
+ }
+ return meta, nil
+}
+
+func beamtimeMetaFromMatch(match string) (beamtimeMeta, error) {
match = strings.TrimPrefix(match, settings.RootBeamtimesFolder)
match = strings.TrimPrefix(match, string(filepath.Separator))
- vars := strings.Split(match,string(filepath.Separator))
- if len(vars)!=6 {
- return beamtimeInfo{},errors.New("bad pattern")
+ vars := strings.Split(match, string(filepath.Separator))
+ if len(vars) != 6 {
+ return beamtimeMeta{}, errors.New("bad pattern")
}
- var bt beamtimeInfo
- ignoredFoldersAfterGpfs:=[]string{"common","BeamtimeUsers","state","support"}
- if utils.StringInSlice(vars[2],ignoredFoldersAfterGpfs) {
- return beamtimeInfo{},errors.New("skipped fodler")
+ var bt beamtimeMeta
+ ignoredFoldersAfterGpfs := []string{"common", "BeamtimeUsers", "state", "support"}
+ if utils.StringInSlice(vars[2], ignoredFoldersAfterGpfs) {
+ return beamtimeMeta{}, errors.New("skipped fodler")
}
- bt.Facility,bt.Beamline,bt.Year,bt.BeamtimeId = vars[0],vars[2],vars[3],vars[5]
+ bt.OfflinePath = match
+ bt.Beamline, bt.BeamtimeId = vars[2], vars[5]
- return bt,nil
+ return bt, nil
}
-func findBeamtime(beamtime_id string) (beamtimeInfo,bool) {
+func findBeamtimeInfoFromId(beamtime_id string) (beamtimeMeta, error) {
sep := string(filepath.Separator)
- pattern := sep+"*"+sep+"gpfs"+sep+"*"+sep+"*"+sep+"*"+sep
- matches, err := filepath.Glob(settings.RootBeamtimesFolder+pattern+beamtime_id)
+ pattern := sep + "*" + sep + "gpfs" + sep + "*" + sep + "*" + sep + "*" + sep
+ matches, err := filepath.Glob(settings.RootBeamtimesFolder + pattern + beamtime_id)
- if err!=nil || len(matches)==0 {
- return beamtimeInfo{},false
+ if err != nil || len(matches) == 0 {
+ return beamtimeMeta{}, errors.New("Cannot find beamline for "+beamtime_id)
}
- for _,match := range (matches) {
- btInfo,err := beamtimeInfoFromMatch(match)
- if err!= nil {
+ for _, match := range (matches) {
+ btInfo, err := beamtimeMetaFromMatch(match)
+ if err != nil {
continue
}
if btInfo.BeamtimeId == beamtime_id {
- return btInfo,true
+ return btInfo, nil
}
}
- return beamtimeInfo{},false
+ return beamtimeMeta{}, errors.New("Cannot find beamline for "+beamtime_id)
}
-func alwaysAllowed(creds SourceCredentials)(beamtimeInfo,bool) {
+func findBeamtimeMetaFromBeamline(beamline string) (beamtimeMeta, error) {
+ sep := string(filepath.Separator)
+ pattern := "beamtime-metadata-*.json"
+ online_path := settings.CurrentBeamlinesFolder + sep + beamline + sep + "current"
+
+ matches, err := filepath.Glob(online_path + sep + pattern)
+ if err != nil || len(matches) != 1 {
+ return beamtimeMeta{}, err
+ }
+
+ meta, err := beamtimeMetaFromJson(matches[0])
+ if (err != nil) {
+ return beamtimeMeta{}, err
+ }
+ meta.OnlinePath = online_path
+ return meta, nil
+}
+
+func alwaysAllowed(creds SourceCredentials) (beamtimeMeta, bool) {
for _, pair := range settings.AlwaysAllowedBeamtimes {
if pair.BeamtimeId == creds.BeamtimeId {
pair.Stream = creds.Stream
- return pair,true
+ return pair, true
}
}
- return beamtimeInfo{},false
+ return beamtimeMeta{}, false
}
-func authorizeByHost(host,beamline string) (bool) {
+func authorizeByHost(host, beamline string) (error) {
active_beamline, err := getBeamlineFromIP(host)
if err != nil {
log.Error("cannot find active beamline for " + host + " - " + err.Error())
- return false
+ return err
}
if (active_beamline != beamline) {
- log.Error("beamine for host " + host +" - "+ active_beamline+ " does not match " + beamline)
- return false
+ err_string := "beamine for host " + host + " - " + active_beamline + " does not match " + beamline
+ log.Error(err_string)
+ return errors.New(err_string)
}
- return true
+ return nil
}
func needHostAuthorization(creds SourceCredentials) bool {
- return strings.HasPrefix(creds.Stream,"detector") || len(creds.Token)==0
+ return strings.HasPrefix(creds.Stream, "detector") || len(creds.Token) == 0
}
-func authorizeByToken(creds SourceCredentials) bool {
- token_expect, _ := auth.GenerateToken(&creds.BeamtimeId)
+func authorizeByToken(creds SourceCredentials) error {
+ var token_expect string
+ if (creds.BeamtimeId != "auto") {
+ token_expect, _ = auth.GenerateToken(&creds.BeamtimeId)
+ } else {
+ key := "bl_" + creds.Beamline
+ token_expect, _ = auth.GenerateToken(&key)
+ }
+ var err_string string
if creds.Token != token_expect {
- log.Error("wrong token for beamtime " + creds.BeamtimeId)
- return false
+ if creds.BeamtimeId != "auto" {
+ err_string = "wrong token for beamtime " + creds.BeamtimeId
+ } else {
+ err_string = "wrong token for beamline " + creds.Beamline
+ }
+ log.Error(err_string)
+ return errors.New(err_string)
}
- return true
+ return nil
}
-
-func authorize(request authorizationRequest,creds SourceCredentials) (beamtimeInfo,bool) {
- if answer,ok := alwaysAllowed(creds);ok {
- return answer,ok
+func findMeta(creds SourceCredentials) (beamtimeMeta, error) {
+ var err error
+ var meta beamtimeMeta
+ if (creds.BeamtimeId != "auto") {
+ meta, err = findBeamtimeInfoFromId(creds.BeamtimeId)
+ if (err == nil ) {
+ meta_onilne, err_online := findBeamtimeMetaFromBeamline(meta.Beamline)
+ if err_online == nil && meta.BeamtimeId == meta_onilne.BeamtimeId {
+ meta.OnlinePath = meta_onilne.OnlinePath
+ }
+ }
+ } else {
+ meta, err = findBeamtimeMetaFromBeamline(creds.Beamline)
}
- beamlineInfo,ok := findBeamtime(creds.BeamtimeId)
- if (!ok) {
- log.Error("cannot find beamline for " + creds.BeamtimeId)
- return beamtimeInfo{},false
+ if (err != nil) {
+ log.Error(err.Error())
+ return beamtimeMeta{}, err
}
- if needHostAuthorization(creds) {
- if !authorizeByHost(request.OriginHost,beamlineInfo.Beamline) {
- return beamtimeInfo{}, false
+ return meta, nil
+}
+
+func authorizeMeta(meta beamtimeMeta, request authorizationRequest, creds SourceCredentials) error {
+
+ if needHostAuthorization(creds) {
+ if err := authorizeByHost(request.OriginHost, meta.Beamline); err != nil {
+ return err
}
} else {
- if !authorizeByToken(creds) {
- return beamtimeInfo{}, false
+ if err := authorizeByToken(creds); err != nil {
+ return err
}
}
- var answer beamtimeInfo
- answer.Beamline = beamlineInfo.Beamline
- answer.Facility = beamlineInfo.Facility
- answer.Year = beamlineInfo.Year
- answer.BeamtimeId = creds.BeamtimeId
- answer.Stream = creds.Stream
+ if creds.Beamline != "auto" && meta.Beamline != creds.Beamline {
+ err_string := "given beamline (" + creds.Beamline + ") does not match the found one (" + meta.Beamline + ")"
+ log.Debug(err_string)
+ return errors.New(err_string)
+ }
+ return nil
+}
- log.Debug("authorized beamtime " + answer.BeamtimeId + " for " + request.OriginHost + " in " + answer.Beamline)
+func authorize(request authorizationRequest, creds SourceCredentials) (beamtimeMeta, error) {
+ if meta, ok := alwaysAllowed(creds); ok {
+ return meta, nil
+ }
- return answer,true
+ meta, err := findMeta(creds)
+ if err != nil {
+ return beamtimeMeta{}, err
+ }
+
+ if err := authorizeMeta(meta, request, creds); err != nil {
+ return beamtimeMeta{}, err
+ }
+ meta.Stream = creds.Stream
+
+ log.Debug("authorized beamtime " + meta.BeamtimeId + " for " + request.OriginHost + " in " + meta.Beamline)
+ return meta, nil
}
func routeAuthorize(w http.ResponseWriter, r *http.Request) {
@@ -179,17 +262,17 @@ func routeAuthorize(w http.ResponseWriter, r *http.Request) {
return
}
-
- creds,err := getSourceCredentials(request)
- if err!=nil {
+ creds, err := getSourceCredentials(request)
+ if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))
return
}
- beamtimeInfo,ok := authorize(request,creds)
- if (!ok) {
+ beamtimeInfo, err := authorize(request, creds)
+ if (err != nil) {
w.WriteHeader(http.StatusUnauthorized)
+ w.Write([]byte(err.Error()))
return
}
diff --git a/authorizer/src/asapo_authorizer/server/authorize_test.go b/authorizer/src/asapo_authorizer/server/authorize_test.go
index 23427d6435c06990cd0ab33c6a6a3394da3267c1..3867622f1b4aca3fd437fc24d8b52e2e795b2630 100644
--- a/authorizer/src/asapo_authorizer/server/authorize_test.go
+++ b/authorizer/src/asapo_authorizer/server/authorize_test.go
@@ -12,9 +12,9 @@ import (
"testing"
)
-func prepareToken(beamtime_id string) string{
+func prepareToken(beamtime_or_beamline string) string{
auth = utils.NewHMACAuth("secret")
- token, _ := auth.GenerateToken(&beamtime_id)
+ token, _ := auth.GenerateToken(&beamtime_or_beamline)
return token
}
@@ -26,7 +26,7 @@ type request struct {
message string
}
-func allowBeamlines(beamlines []beamtimeInfo) {
+func allowBeamlines(beamlines []beamtimeMeta) {
settings.AlwaysAllowedBeamtimes=beamlines
}
@@ -49,15 +49,23 @@ func doAuthorizeRequest(path string,buf string) *httptest.ResponseRecorder {
}
+
var credTests = [] struct {
request string
cred SourceCredentials
+ ok bool
message string
} {
- {"asapo_test%%", SourceCredentials{"asapo_test","detector",""},"default stream and no token"},
- {"asapo_test%%token", SourceCredentials{"asapo_test","detector","token"},"default stream"},
- {"asapo_test%stream%", SourceCredentials{"asapo_test","stream",""},"no token"},
- {"asapo_test%stream%token", SourceCredentials{"asapo_test","stream","token"},"all set"},
+ {"asapo_test%auto%%", SourceCredentials{"asapo_test","auto","detector",""},true,"auto beamline, stream and no token"},
+ {"asapo_test%auto%%token", SourceCredentials{"asapo_test","auto","detector","token"},true,"auto beamline, stream"},
+ {"asapo_test%auto%stream%", SourceCredentials{"asapo_test","auto","stream",""},true,"auto beamline, no token"},
+ {"asapo_test%auto%stream%token", SourceCredentials{"asapo_test","auto","stream","token"},true,"auto beamline,stream, token"},
+ {"asapo_test%beamline%stream%token", SourceCredentials{"asapo_test","beamline","stream","token"},true,"all set"},
+ {"auto%beamline%stream%token", SourceCredentials{"auto","beamline","stream","token"},true,"auto beamtime"},
+ {"auto%auto%stream%token", SourceCredentials{},false,"auto beamtime and beamline"},
+ {"%beamline%stream%token", SourceCredentials{"auto","beamline","stream","token"},true,"empty beamtime"},
+ {"asapo_test%%stream%token", SourceCredentials{"asapo_test","auto","stream","token"},true,"empty bealine"},
+ {"%%stream%token", SourceCredentials{},false,"both empty"},
}
func TestSplitCreds(t *testing.T) {
@@ -65,14 +73,19 @@ func TestSplitCreds(t *testing.T) {
for _, test := range credTests {
request := authorizationRequest{test.request,"host"}
creds,err := getSourceCredentials(request)
- assert.Nil(t,err)
- assert.Equal(t,creds,test.cred,test.message)
+ if test.ok {
+ assert.Nil(t,err)
+ assert.Equal(t,creds,test.cred,test.message)
+ } else {
+ assert.NotNil(t,err,test.message)
+ }
+
}
}
func TestAuthorizeDefaultOK(t *testing.T) {
- allowBeamlines([]beamtimeInfo{{"asapo_test","beamline","","2019","tf"}})
- request := makeRequest(authorizationRequest{"asapo_test%%","host"})
+ allowBeamlines([]beamtimeMeta{{"asapo_test","beamline","","2019","tf"}})
+ request := makeRequest(authorizationRequest{"asapo_test%%%","host"})
w := doAuthorizeRequest("/authorize",request)
body, _ := ioutil.ReadAll(w.Body)
@@ -84,36 +97,62 @@ func TestAuthorizeDefaultOK(t *testing.T) {
assert.Equal(t, http.StatusOK, w.Code, "")
}
+var beamtime_meta_online =`
+{
+"beamline": "bl1",
+"beamtimeId": "test_online"
+}
+`
+
var authTests = [] struct {
beamtime_id string
+ beamline string
stream string
token string
status int
message string
}{
- {"test","stream", prepareToken("test"),http.StatusOK,"user stream with correct token"},
- {"test1","stream", prepareToken("test1"),http.StatusUnauthorized,"correct token, beamtime not found"},
- {"test","stream", prepareToken("wrong"),http.StatusUnauthorized,"user stream with wrong token"},
- {"test","detector_aaa", prepareToken("test"),http.StatusUnauthorized,"detector stream with correct token and wroung source"},
+ {"test","auto","stream", prepareToken("test"),http.StatusOK,"user stream with correct token"},
+ {"test_online","auto","stream", prepareToken("test_online"),http.StatusOK,"with online path"},
+ {"test1","auto","stream", prepareToken("test1"),http.StatusUnauthorized,"correct token, beamtime not found"},
+ {"test","auto","stream", prepareToken("wrong"),http.StatusUnauthorized,"user stream with wrong token"},
+ {"test","auto","detector_aaa", prepareToken("test"),http.StatusUnauthorized,"detector stream with correct token and wroung source"},
+ {"test","bl1","stream", prepareToken("test"),http.StatusOK,"correct beamline given"},
+ {"test","bl2","stream", prepareToken("test"),http.StatusUnauthorized,"incorrect beamline given"},
}
func TestAuthorizeWithToken(t *testing.T) {
- allowBeamlines([]beamtimeInfo{})
+ allowBeamlines([]beamtimeMeta{})
settings.RootBeamtimesFolder ="."
+ settings.CurrentBeamlinesFolder="."
os.MkdirAll(filepath.Clean("tf/gpfs/bl1/2019/data/test"), os.ModePerm)
+ os.MkdirAll(filepath.Clean("tf/gpfs/bl1/2019/data/test_online"), os.ModePerm)
+
+ os.MkdirAll(filepath.Clean("bl1/current"), os.ModePerm)
+ ioutil.WriteFile(filepath.Clean("bl1/current/beamtime-metadata-test_online.json"), []byte(beamtime_meta_online), 0644)
+
defer os.RemoveAll("tf")
+ defer os.RemoveAll("bl1")
for _, test := range authTests {
- request := makeRequest(authorizationRequest{test.beamtime_id+"%"+test.stream+"%"+test.token,"host"})
+ request := makeRequest(authorizationRequest{test.beamtime_id+"%"+test.beamline+"%"+test.stream+"%"+test.token,"host"})
w := doAuthorizeRequest("/authorize",request)
body, _ := ioutil.ReadAll(w.Body)
if test.status==http.StatusOK {
- assert.Contains(t, string(body), test.beamtime_id, "")
- assert.Contains(t, string(body), "bl1", "")
- assert.Contains(t, string(body), "stream", "")
- assert.Contains(t, string(body), "2019", "")
- assert.Contains(t, string(body), "tf", "")
- assert.Contains(t, string(body), test.stream, "")
+ body_str:=string(body)
+ body_str = strings.Replace(body_str,string(os.PathSeparator),"/",-1)
+ body_str = strings.Replace(body_str,"//","/",-1)
+ assert.Contains(t, body_str, test.beamtime_id, "")
+ assert.Contains(t, body_str, "bl1", "")
+ assert.Contains(t, body_str, "stream", "")
+ assert.Contains(t, body_str, "tf/gpfs/bl1/2019/data/test", "")
+ if (test.beamtime_id == "test_online") {
+ assert.Contains(t, body_str, "tf/gpfs/bl1/2019/data/test_online", "")
+ assert.Contains(t, body_str, "./bl1/current", "")
+ } else {
+ assert.NotContains(t, body_str, "current", "")
+ }
+ assert.Contains(t, body_str, test.stream, "")
}
assert.Equal(t, test.status, w.Code, test.message)
@@ -123,9 +162,87 @@ func TestAuthorizeWithToken(t *testing.T) {
}
+var beamtime_meta =`
+{
+"applicant": {
+"email": "test",
+"institute": "test",
+"lastname": "test",
+"userId": "1234",
+"username": "test"
+},
+"beamline": "p07",
+"beamline_alias": "P07",
+"beamtimeId": "11111111",
+"contact": "None",
+"core-path": "asap3/petra3/gpfs/p07/2020/data/11111111",
+"event-end": "2020-03-03 09:00:00",
+"event-start": "2020-03-02 09:00:00",
+"facility": "PETRA III",
+"generated": "2020-02-22 22:37:16",
+"pi": {
+"email": "test",
+"institute": "test",
+"lastname": "test",
+"userId": "14",
+"username": "test"
+},
+"proposalId": "12345678",
+"proposalType": "H",
+"title": "In-House Research (P07)",
+"unixId": "None",
+"users": {
+"door-db": [
+"test"
+],
+"special": [],
+"unknown": []
+}
+}
+`
+
+var authBeamlineTests = [] struct {
+ beamtime_id string
+ beamline string
+ token string
+ status int
+ message string
+}{
+ {"11111111","p07", prepareToken("bl_p07"),http.StatusOK,"beamtime found"},
+ {"11111111","p07", prepareToken("bl_p06"),http.StatusUnauthorized,"wrong token"},
+ {"11111111","p08", prepareToken("bl_p08"),http.StatusUnauthorized,"beamtime not found"},
+}
+
+func TestAuthorizeBeamline(t *testing.T) {
+ allowBeamlines([]beamtimeMeta{})
+ settings.CurrentBeamlinesFolder="."
+ os.MkdirAll(filepath.Clean("p07/current"), os.ModePerm)
+ ioutil.WriteFile(filepath.Clean("p07/current/beamtime-metadata-11111111.json"), []byte(beamtime_meta), 0644)
+ defer os.RemoveAll("p07")
+
+ for _, test := range authBeamlineTests {
+ request := makeRequest(authorizationRequest{"auto%"+test.beamline+"%stream%"+test.token,"host"})
+ w := doAuthorizeRequest("/authorize",request)
+
+ body, _ := ioutil.ReadAll(w.Body)
+ body_str:=string(body)
+ body_str = strings.Replace(body_str,string(os.PathSeparator),"/",-1)
+ body_str = strings.Replace(body_str,"//","/",-1)
+ if test.status==http.StatusOK {
+ assert.Contains(t, body_str, test.beamtime_id, "")
+ assert.Contains(t, body_str, test.beamline, "")
+ assert.Contains(t, body_str, "asap3/petra3/gpfs/p07/2020/data/11111111", "")
+ assert.Contains(t, body_str, "p07/current", "")
+ assert.Contains(t, body_str, "stream", "")
+ }
+
+ assert.Equal(t, test.status, w.Code, test.message)
+ }
+}
+
func TestNotAuthorized(t *testing.T) {
- request := makeRequest(authorizationRequest{"any_id%%","host"})
+ request := makeRequest(authorizationRequest{"any_id%%%","host"})
w := doAuthorizeRequest("/authorize",request)
assert.Equal(t, http.StatusUnauthorized, w.Code, "")
}
@@ -143,12 +260,12 @@ func TestAuthorizeWrongPath(t *testing.T) {
}
func TestDoNotAuthorizeIfNotInAllowed(t *testing.T) {
- allowBeamlines([]beamtimeInfo{{"test","beamline","","2019","tf"}})
+ allowBeamlines([]beamtimeMeta{{"test","beamline","","2019","tf"}})
request := authorizationRequest{"asapo_test%%","host"}
creds,_ := getSourceCredentials(request)
- _,ok := authorize(request,creds)
- assert.Equal(t,false, ok, "")
+ _,err := authorize(request,creds)
+ assert.Error(t,err, "")
}
func TestSplitHost(t *testing.T) {
@@ -178,18 +295,20 @@ func TestAuthorizeWithFile(t *testing.T) {
ioutil.WriteFile("127.0.0.1", []byte("bl1"), 0644)
- request := authorizationRequest{"11003924%%","127.0.0.1"}
+ request := authorizationRequest{"11003924%%%","127.0.0.1"}
w := doAuthorizeRequest("/authorize",makeRequest(request))
body, _ := ioutil.ReadAll(w.Body)
- assert.Contains(t, string(body), "11003924", "")
- assert.Contains(t, string(body), "bl1", "")
- assert.Contains(t, string(body), "detector", "")
- assert.Contains(t, string(body), "2019", "")
- assert.Contains(t, string(body), "tf", "")
+ body_str:=string(body)
+ body_str = strings.Replace(body_str,string(os.PathSeparator),"/",-1)
+ body_str = strings.Replace(body_str,"//","/",-1)
+ assert.Contains(t,body_str,"tf/gpfs/bl1/2019/data/11003924")
+ assert.Contains(t, body_str, "11003924", "")
+ assert.Contains(t, body_str, "bl1", "")
+ assert.Contains(t, body_str, "detector", "")
assert.Equal(t, http.StatusOK, w.Code, "")
- request = authorizationRequest{"wrong%%","127.0.0.1"}
+ request = authorizationRequest{"wrong%%%","127.0.0.1"}
w = doAuthorizeRequest("/authorize",makeRequest(request))
assert.Equal(t, http.StatusUnauthorized, w.Code, "")
@@ -202,31 +321,27 @@ func TestAuthorizeWithFile(t *testing.T) {
var extractBtinfoTests = [] struct {
root string
fname string
- facility string
beamline string
- year string
id string
ok bool
}{
- {".",filepath.Clean("tf/gpfs/bl1.01/2019/data/123"),"tf", "bl1.01","2019","123",true},
- {filepath.Clean("/blabla/tratartra"),filepath.Clean("tf/gpfs/bl1.01/2019/data/123"),"tf", "bl1.01","2019","123",true},
- {".",filepath.Clean("tf/gpfs/common/2019/data/123"),"tf", "bl1.01","2019","123",false},
- {".",filepath.Clean("tf/gpfs/BeamtimeUsers/2019/data/123"),"tf", "bl1.01","2019","123",false},
- {".",filepath.Clean("tf/gpfs/state/2019/data/123"),"tf", "bl1.01","2019","123",false},
- {".",filepath.Clean("tf/gpfs/support/2019/data/123"),"tf", "bl1.01","2019","123",false},
- {".",filepath.Clean("petra3/gpfs/p01/2019/comissioning/c20180508-000-COM20181"),"petra3", "p01","2019","c20180508-000-COM20181",true},
+ {".",filepath.Clean("tf/gpfs/bl1.01/2019/data/123"),"bl1.01","123",true},
+ {filepath.Clean("/blabla/tratartra"),filepath.Clean("tf/gpfs/bl1.01/2019/data/123"), "bl1.01","123",true},
+ {".",filepath.Clean("tf/gpfs/common/2019/data/123"), "bl1.01","123",false},
+ {".",filepath.Clean("tf/gpfs/BeamtimeUsers/2019/data/123"), "bl1.01","123",false},
+ {".",filepath.Clean("tf/gpfs/state/2019/data/123"), "bl1.01","123",false},
+ {".",filepath.Clean("tf/gpfs/support/2019/data/123"), "bl1.01","123",false},
+ {".",filepath.Clean("petra3/gpfs/p01/2019/comissioning/c20180508-000-COM20181"), "p01","c20180508-000-COM20181",true},
}
func TestGetBeamtimeInfo(t *testing.T) {
for _, test := range extractBtinfoTests {
settings.RootBeamtimesFolder=test.root
- bt,err:=beamtimeInfoFromMatch(test.root+string(filepath.Separator)+test.fname)
+ bt,err:= beamtimeMetaFromMatch(test.root+string(filepath.Separator)+test.fname)
if test.ok {
- assert.Equal(t,bt.Facility,test.facility)
+ assert.Equal(t,bt.OfflinePath,test.fname)
assert.Equal(t,bt.Beamline,test.beamline)
- assert.Equal(t,bt.Year,test.year)
assert.Equal(t,bt.BeamtimeId,test.id)
- assert.Equal(t,bt.Facility,test.facility)
assert.Nil(t,err,"should not be error")
} else {
assert.NotNil(t,err,"should be error")
diff --git a/authorizer/src/asapo_authorizer/server/server.go b/authorizer/src/asapo_authorizer/server/server.go
index e0785c8f304e93bbc182f44af710432896c0301c..e6b6f632a30cc32a3e8a910fde86a5d87f6318c8 100644
--- a/authorizer/src/asapo_authorizer/server/server.go
+++ b/authorizer/src/asapo_authorizer/server/server.go
@@ -2,12 +2,12 @@ package server
import "asapo_common/utils"
-type beamtimeInfo struct {
- BeamtimeId string
- Beamline string
- Stream string
- Year string
- Facility string
+type beamtimeMeta struct {
+ BeamtimeId string `json:"beamtimeId"`
+ Beamline string `json:"beamline"`
+ Stream string `json:"stream"`
+ OfflinePath string `json:"core-path"`
+ OnlinePath string `json:"beamline-path"`
}
type serverSettings struct {
@@ -15,7 +15,8 @@ type serverSettings struct {
LogLevel string
IpBeamlineMappingFolder string
RootBeamtimesFolder string
- AlwaysAllowedBeamtimes []beamtimeInfo
+ CurrentBeamlinesFolder string
+ AlwaysAllowedBeamtimes []beamtimeMeta
SecretFile string
}
diff --git a/common/cpp/include/common/data_structs.h b/common/cpp/include/common/data_structs.h
index 24b9649fd88064741372fa41422f5b79ac0b0c8c..e46b6ec7ae0ff3ec09ba260841c58c45624210ad 100644
--- a/common/cpp/include/common/data_structs.h
+++ b/common/cpp/include/common/data_structs.h
@@ -49,12 +49,22 @@ using SubDirList = std::vector<std::string>;
struct SourceCredentials {
+ SourceCredentials(std::string beamtime, std::string beamline, std::string stream, std::string token):
+ beamtime_id{std::move(beamtime)},
+ beamline{std::move(beamline)},
+ stream{std::move(stream)},
+ user_token{std::move(token)} {
+ };
+ SourceCredentials() {};
static const std::string kDefaultStream;
+ static const std::string kDefaultBeamline;
+ static const std::string kDefaultBeamtimeId;
std::string beamtime_id;
+ std::string beamline;
std::string stream;
std::string user_token;
std::string GetString() {
- return beamtime_id + "%" + stream + "%" + user_token;
+ return beamtime_id + "%" + beamline + "%" + stream + "%" + user_token;
};
};
diff --git a/common/cpp/include/common/networking.h b/common/cpp/include/common/networking.h
index 8e6dd304606621514ff029e93a0682f3dd7a7f7f..5348d85a9414cd3642edf9400f897496dc4c4e2a 100644
--- a/common/cpp/include/common/networking.h
+++ b/common/cpp/include/common/networking.h
@@ -25,6 +25,7 @@ enum Opcode : uint8_t {
enum NetworkErrorCode : uint16_t {
kNetErrorNoError,
+ kNetErrorReauthorize,
kNetErrorWarning,
kNetErrorWrongRequest,
kNetErrorNoData,
diff --git a/common/cpp/include/request/request_handler.h b/common/cpp/include/request/request_handler.h
index 60fbd767baf396e7036bd2c430d188164324773a..06cd2a59bafe61a6cef1b4d7fe2903deaedba434 100644
--- a/common/cpp/include/request/request_handler.h
+++ b/common/cpp/include/request/request_handler.h
@@ -12,8 +12,8 @@ namespace asapo {
class RequestHandler {
public:
virtual void PrepareProcessingRequestLocked() = 0;
- virtual void TearDownProcessingRequestLocked(bool processing_succeeded) = 0;
- virtual bool ProcessRequestUnlocked(GenericRequest* request) = 0;
+ virtual void TearDownProcessingRequestLocked(bool success) = 0;
+ virtual bool ProcessRequestUnlocked(GenericRequest* request, bool* retry) = 0;
virtual void ProcessRequestTimeout(GenericRequest* request) = 0;
virtual bool ReadyProcessRequest() = 0;
virtual ~RequestHandler() = default;
diff --git a/common/cpp/src/data_structs/data_structs.cpp b/common/cpp/src/data_structs/data_structs.cpp
index bdda7fa5f6f45dc79c59c4deea0eb9e676da3f76..b3b0a863c16407f5a55c423099dadad1e547f2f3 100644
--- a/common/cpp/src/data_structs/data_structs.cpp
+++ b/common/cpp/src/data_structs/data_structs.cpp
@@ -20,6 +20,8 @@ using std::chrono::system_clock;
namespace asapo {
const std::string SourceCredentials::kDefaultStream = "detector";
+const std::string SourceCredentials::kDefaultBeamline = "auto";
+const std::string SourceCredentials::kDefaultBeamtimeId = "auto";
std::string FileInfo::Json() const {
diff --git a/common/cpp/src/request/request_pool.cpp b/common/cpp/src/request/request_pool.cpp
index de91b3f1501b6b42507df7318c71e7187a3e629b..f49f28c75265b0e869a0d90f953b05c16a72770c 100644
--- a/common/cpp/src/request/request_pool.cpp
+++ b/common/cpp/src/request/request_pool.cpp
@@ -50,11 +50,12 @@ void RequestPool::ProcessRequest(const std::unique_ptr<RequestHandler>& request_
request_handler->PrepareProcessingRequestLocked();
requests_in_progress_++;
thread_info->lock.unlock();
- auto success = request_handler->ProcessRequestUnlocked(request.get());
+ bool retry;
+ auto success = request_handler->ProcessRequestUnlocked(request.get(), &retry);
thread_info->lock.lock();
requests_in_progress_--;
request_handler->TearDownProcessingRequestLocked(success);
- if (!success) {
+ if (retry) {
PutRequestBackToQueue(std::move(request));
thread_info->lock.unlock();
condition_.notify_all();
diff --git a/common/cpp/unittests/request/mocking.h b/common/cpp/unittests/request/mocking.h
index d37fdd79d7ff50b58597f814b35307cc56acba48..fdf7a3c86f2f9326d95c06c230a7a489048593c6 100644
--- a/common/cpp/unittests/request/mocking.h
+++ b/common/cpp/unittests/request/mocking.h
@@ -15,13 +15,13 @@ class MockRequestHandler : public RequestHandler {
MOCK_METHOD0(PrepareProcessingRequestLocked, void());
MOCK_METHOD0(ReadyProcessRequest, bool());
MOCK_METHOD1(TearDownProcessingRequestLocked, void(bool processing_succeeded));
- MOCK_METHOD1(ProcessRequestUnlocked_t, bool (const GenericRequest* request));
+ MOCK_METHOD2(ProcessRequestUnlocked_t, bool (const GenericRequest* request, bool* retry));
MOCK_METHOD1(ProcessRequestTimeout, void(GenericRequest* request));
uint64_t retry_counter = 0;
- bool ProcessRequestUnlocked(GenericRequest* request) override {
+ bool ProcessRequestUnlocked(GenericRequest* request, bool* retry) override {
retry_counter = request->GetRetryCounter();
std::this_thread::sleep_for(std::chrono::milliseconds(50));
- return ProcessRequestUnlocked_t(request);
+ return ProcessRequestUnlocked_t(request, retry);
}
diff --git a/common/cpp/unittests/request/test_request_pool.cpp b/common/cpp/unittests/request/test_request_pool.cpp
index 447d4c0e9c78206d631621d016b63e8990de1f9f..80894f504c95d616b61181ddc07b44f387839cfd 100644
--- a/common/cpp/unittests/request/test_request_pool.cpp
+++ b/common/cpp/unittests/request/test_request_pool.cpp
@@ -104,7 +104,7 @@ TEST_F(RequestPoolTests, TimeOut) {
EXPECT_CALL(*mock_request_handler, ReadyProcessRequest()).Times(1).WillRepeatedly(Return(true));
EXPECT_CALL(*mock_request_handler, PrepareProcessingRequestLocked()).Times(0);
- EXPECT_CALL(*mock_request_handler, ProcessRequestUnlocked_t(_)).Times(0);
+ EXPECT_CALL(*mock_request_handler, ProcessRequestUnlocked_t(_, _)).Times(0);
EXPECT_CALL(*mock_request_handler, ProcessRequestTimeout(_)).Times(1);
auto err = pool.AddRequest(std::move(request));
@@ -116,14 +116,20 @@ TEST_F(RequestPoolTests, TimeOut) {
void ExpectSend(MockRequestHandler* mock_handler, int ntimes = 1) {
EXPECT_CALL(*mock_handler, ReadyProcessRequest()).Times(ntimes).WillRepeatedly(Return(true));
EXPECT_CALL(*mock_handler, PrepareProcessingRequestLocked()).Times(ntimes);
- EXPECT_CALL(*mock_handler, ProcessRequestUnlocked_t(_)).Times(ntimes).WillRepeatedly(Return(true));
+ EXPECT_CALL(*mock_handler, ProcessRequestUnlocked_t(_, _)).Times(ntimes).WillRepeatedly(
+ DoAll( testing::SetArgPointee<1>(false),
+ Return(true)
+ ));
EXPECT_CALL(*mock_handler, TearDownProcessingRequestLocked(true)).Times(ntimes);
}
void ExpectFailProcessRequest(MockRequestHandler* mock_handler) {
EXPECT_CALL(*mock_handler, ReadyProcessRequest()).Times(AtLeast(1)).WillRepeatedly(Return(true));
EXPECT_CALL(*mock_handler, PrepareProcessingRequestLocked()).Times(AtLeast(1));
- EXPECT_CALL(*mock_handler, ProcessRequestUnlocked_t(_)).Times(AtLeast(1)).WillRepeatedly(Return(false));
+ EXPECT_CALL(*mock_handler, ProcessRequestUnlocked_t(_, _)).Times(AtLeast(1)).WillRepeatedly(
+ DoAll( testing::SetArgPointee<1>(true),
+ Return(false)
+ ));
EXPECT_CALL(*mock_handler, TearDownProcessingRequestLocked(false)).Times(AtLeast(1));
}
diff --git a/consumer/api/cpp/unittests/test_consumer_api.cpp b/consumer/api/cpp/unittests/test_consumer_api.cpp
index 9b3037f7baf7669c9f635b1d19b9af8dfedb2a5a..d683a4233d2d5eb9b0260ca48ec8d9a5da4da060 100644
--- a/consumer/api/cpp/unittests/test_consumer_api.cpp
+++ b/consumer/api/cpp/unittests/test_consumer_api.cpp
@@ -27,7 +27,7 @@ class DataBrokerFactoryTests : public Test {
TEST_F(DataBrokerFactoryTests, CreateServerDataSource) {
- auto data_broker = DataBrokerFactory::CreateServerBroker("server", "path", asapo::SourceCredentials{"beamtime_id", "", "token"}, &error);
+ auto data_broker = DataBrokerFactory::CreateServerBroker("server", "path", asapo::SourceCredentials{"beamtime_id", "", "", "token"}, &error);
ASSERT_THAT(error, Eq(nullptr));
ASSERT_THAT(dynamic_cast<ServerDataBroker*>(data_broker.get()), Ne(nullptr));
diff --git a/consumer/api/cpp/unittests/test_server_broker.cpp b/consumer/api/cpp/unittests/test_server_broker.cpp
index c5c23986860c69b1b449aba34b8fc588f21400f4..26b9e61ebd8db83c175b4d3eac1ffd8952c12bb6 100644
--- a/consumer/api/cpp/unittests/test_server_broker.cpp
+++ b/consumer/api/cpp/unittests/test_server_broker.cpp
@@ -44,7 +44,7 @@ namespace {
TEST(FolderDataBroker, Constructor) {
auto data_broker =
- std::unique_ptr<ServerDataBroker> {new ServerDataBroker("test", "path", asapo::SourceCredentials{"beamtime_id", "", "token"})};
+ std::unique_ptr<ServerDataBroker> {new ServerDataBroker("test", "path", asapo::SourceCredentials{"beamtime_id", "", "", "token"})};
ASSERT_THAT(dynamic_cast<asapo::SystemIO*>(data_broker->io__.get()), Ne(nullptr));
ASSERT_THAT(dynamic_cast<asapo::CurlHttpClient*>(data_broker->httpclient__.get()), Ne(nullptr));
ASSERT_THAT(dynamic_cast<asapo::TcpClient*>(data_broker->net_client__.get()), Ne(nullptr));
@@ -74,7 +74,7 @@ class ServerDataBrokerTests : public Test {
std::string expected_next_substream = "nextsubstream";
void SetUp() override {
data_broker = std::unique_ptr<ServerDataBroker> {
- new ServerDataBroker(expected_server_uri, expected_path, asapo::SourceCredentials{"beamtime_id", expected_stream, expected_token})
+ new ServerDataBroker(expected_server_uri, expected_path, asapo::SourceCredentials{"beamtime_id", "", expected_stream, expected_token})
};
data_broker->io__ = std::unique_ptr<IO> {&mock_io};
data_broker->httpclient__ = std::unique_ptr<asapo::HttpClient> {&mock_http_client};
@@ -137,7 +137,7 @@ TEST_F(ServerDataBrokerTests, DefaultStreamIsDetector) {
data_broker->httpclient__.release();
data_broker->net_client__.release();
data_broker = std::unique_ptr<ServerDataBroker> {
- new ServerDataBroker(expected_server_uri, expected_path, asapo::SourceCredentials{"beamtime_id", "", expected_token})
+ new ServerDataBroker(expected_server_uri, expected_path, asapo::SourceCredentials{"beamtime_id", "", "", expected_token})
};
data_broker->io__ = std::unique_ptr<IO> {&mock_io};
data_broker->httpclient__ = std::unique_ptr<asapo::HttpClient> {&mock_http_client};
diff --git a/deploy/asapo_services/scripts/authorizer.json.tpl b/deploy/asapo_services/scripts/authorizer.json.tpl
index 91fd43b3d150cfb3fe79ee264267a424e61adff2..75f151a0dfce23875857e59e0a1d898dba272a0c 100644
--- a/deploy/asapo_services/scripts/authorizer.json.tpl
+++ b/deploy/asapo_services/scripts/authorizer.json.tpl
@@ -1,10 +1,8 @@
{
"Port": {{ env "NOMAD_PORT_authorizer" }},
"LogLevel":"debug",
- "AlwaysAllowedBeamtimes":[{"BeamtimeId":"asapo_test","Beamline":"test","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test1","Beamline":"test1","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test2","Beamline":"test2","Year":"2019","Facility":"test_facility"}],
+ "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","core-path":"/var/lib/receiver/data/test_facility/gpfs/test/2019/data/asapo_test"},
+ {"beamtimeId":"asapo_test1","beamline":"test1","core-path":"/var/lib/receiver/data/test_facility/gpfs/test1/2019/data/asapo_test1"},
+ {"beamtimeId":"asapo_test2","beamline":"test2","core-path":"/var/lib/receiver/data/test_facility/gpfs/test2/2019/data/asapo_test2"}],
"SecretFile":"/local/secret.key"
}
-
-
diff --git a/deploy/asapo_services/scripts/broker.json.tpl b/deploy/asapo_services/scripts/broker.json.tpl
index f924faa3aa3bed06176b0ce879dcfded9f21ab6f..531fdaa7787aba5d6cf02755c4f9822a70b1c340 100644
--- a/deploy/asapo_services/scripts/broker.json.tpl
+++ b/deploy/asapo_services/scripts/broker.json.tpl
@@ -3,7 +3,7 @@
"DiscoveryServer": "localhost:8400/discovery",
"PerformanceDbServer":"localhost:8400/influxdb",
"PerformanceDbName": "asapo_brokers",
- "port":{{ env "NOMAD_PORT_broker" }},
+ "Port":{{ env "NOMAD_PORT_broker" }},
"LogLevel":"info",
"SecretFile":"/local/secret.key"
}
diff --git a/deploy/asapo_services/scripts/receiver.json.tpl b/deploy/asapo_services/scripts/receiver.json.tpl
index 0000e509aa33432a7d7b40713d69b8f6313232d6..0120daebf1437598e9b2c95fa76afd3a097db603 100644
--- a/deploy/asapo_services/scripts/receiver.json.tpl
+++ b/deploy/asapo_services/scripts/receiver.json.tpl
@@ -20,6 +20,5 @@
"WriteToDisk":true,
"ReceiveToDiskThresholdMB": {{ env "NOMAD_META_receiver_receive_to_disk_threshold" }},
"WriteToDb":true,
- "LogLevel": "{{ keyOrDefault "receiver_log_level" "info" }}",
- "RootFolder" : "/var/lib/receiver/data"
+ "LogLevel": "{{ keyOrDefault "receiver_log_level" "info" }}"
}
diff --git a/examples/consumer/getnext_broker/getnext_broker.cpp b/examples/consumer/getnext_broker/getnext_broker.cpp
index 65719c3d8df2afcf1cbacdc06a7c80122e75cc17..242c82a898d51e1c2e6cbdebdc4a0a04688502bf 100644
--- a/examples/consumer/getnext_broker/getnext_broker.cpp
+++ b/examples/consumer/getnext_broker/getnext_broker.cpp
@@ -55,7 +55,7 @@ std::vector<std::thread> StartThreads(const Args& params,
asapo::FileInfo fi;
Error err;
auto broker = asapo::DataBrokerFactory::CreateServerBroker(params.server, params.file_path,
- asapo::SourceCredentials{params.beamtime_id, params.stream, params.token}, &err);
+ asapo::SourceCredentials{params.beamtime_id, "", params.stream, params.token}, &err);
broker->SetTimeout((uint64_t) params.timeout_ms);
asapo::FileData data;
diff --git a/examples/pipeline/in_to_out/in_to_out.cpp b/examples/pipeline/in_to_out/in_to_out.cpp
index 714e704c2f9e2406ab67166b02c7614f9a52d68d..d9bd76f51c1e6f5775b0012b3a9f480908123f98 100644
--- a/examples/pipeline/in_to_out/in_to_out.cpp
+++ b/examples/pipeline/in_to_out/in_to_out.cpp
@@ -39,7 +39,7 @@ struct Args {
};
void ProcessAfterSend(asapo::GenericRequestHeader header, asapo::Error err) {
- if (err && err!=asapo::ProducerErrorTemplates::kServerWarning) {
+ if (err && err != asapo::ProducerErrorTemplates::kServerWarning) {
std::cerr << "Data was not successfully send: " << err << std::endl;
return;
}
@@ -65,7 +65,7 @@ int ProcessError(const Error& err) {
BrokerPtr CreateBrokerAndGroup(const Args& args, Error* err) {
auto broker = asapo::DataBrokerFactory::CreateServerBroker(args.server, args.file_path,
- asapo::SourceCredentials{args.beamtime_id, args.stream_in, args.token}, err);
+ asapo::SourceCredentials{args.beamtime_id, "", args.stream_in, args.token}, err);
if (*err) {
return nullptr;
}
@@ -188,7 +188,7 @@ std::unique_ptr<asapo::Producer> CreateProducer(const Args& args) {
asapo::Error err;
auto producer = asapo::Producer::Create(args.server, args.nthreads,
asapo::RequestHandlerType::kTcp,
- asapo::SourceCredentials{args.beamtime_id, args.stream_out, args.token }, 60, &err);
+ asapo::SourceCredentials{args.beamtime_id, "", args.stream_out, args.token }, 60, &err);
if(err) {
std::cerr << "Cannot start producer. ProducerError: " << err << std::endl;
exit(EXIT_FAILURE);
diff --git a/examples/pipeline/in_to_out_python/in_to_out.py b/examples/pipeline/in_to_out_python/in_to_out.py
index 4c67b594795c3969babb0e937842088a5f1a9303..9bc02800e7944e485a7ed1e32b8d9486ca24a695 100644
--- a/examples/pipeline/in_to_out_python/in_to_out.py
+++ b/examples/pipeline/in_to_out_python/in_to_out.py
@@ -30,7 +30,7 @@ transfer_data=int(transfer_data)>0
broker = asapo_consumer.create_server_broker(source,path, beamtime,stream_in,token,timeout_s*1000)
-producer = asapo_producer.create_producer(source,beamtime, stream_out, token, nthreads, 600)
+producer = asapo_producer.create_producer(source,beamtime,'auto', stream_out, token, nthreads, 600)
group_id = broker.generate_group_id()
diff --git a/examples/producer/dummy-data-producer/dummy_data_producer.cpp b/examples/producer/dummy-data-producer/dummy_data_producer.cpp
index 9a7be1a9c0fed9041ecbd20e4b675433e916f8ac..59aaf83a5798d4e8b40a1f125a8c1013f9a38a14 100644
--- a/examples/producer/dummy-data-producer/dummy_data_producer.cpp
+++ b/examples/producer/dummy-data-producer/dummy_data_producer.cpp
@@ -180,7 +180,7 @@ std::unique_ptr<asapo::Producer> CreateProducer(const Args& args) {
asapo::Error err;
auto producer = asapo::Producer::Create(args.receiver_address, args.nthreads,
args.mode % 10 == 0 ? asapo::RequestHandlerType::kTcp : asapo::RequestHandlerType::kFilesystem,
- asapo::SourceCredentials{args.beamtime_id, args.stream, args.token }, 3600, &err);
+ asapo::SourceCredentials{args.beamtime_id, "", args.stream, args.token }, 3600, &err);
if(err) {
std::cerr << "Cannot start producer. ProducerError: " << err << std::endl;
exit(EXIT_FAILURE);
diff --git a/producer/api/cpp/include/producer/producer_error.h b/producer/api/cpp/include/producer/producer_error.h
index 077c4108be588e6dd8c9de903a2d7c2210e0fc8d..4e7b00c5f0b4e5b88ba8bf9ae54ae2ffa61c31e4 100644
--- a/producer/api/cpp/include/producer/producer_error.h
+++ b/producer/api/cpp/include/producer/producer_error.h
@@ -11,6 +11,7 @@ enum class ProducerErrorType {
kLocalIOError,
kWrongInput,
kServerWarning,
+ kReAuthorizationNeeded,
kTimeout
};
@@ -42,6 +43,11 @@ auto const kTimeout = ProducerErrorTemplate {
"Timeout", ProducerErrorType::kTimeout
};
+auto const kReAuthorizationNeeded = ProducerErrorTemplate {
+ "reauthorization needed", ProducerErrorType::kReAuthorizationNeeded
+};
+
+
};
}
diff --git a/producer/api/cpp/src/producer_impl.cpp b/producer/api/cpp/src/producer_impl.cpp
index a9a1e40e86e159db8764f101a001054904458eb8..ff32dbb7f649410ccfc83ca2b709063c8838a3be 100644
--- a/producer/api/cpp/src/producer_impl.cpp
+++ b/producer/api/cpp/src/producer_impl.cpp
@@ -172,6 +172,21 @@ Error ProducerImpl::SetCredentials(SourceCredentials source_cred) {
source_cred.stream = SourceCredentials::kDefaultStream;
}
+ if (source_cred.beamline.empty()) {
+ source_cred.beamline = SourceCredentials::kDefaultBeamline;
+ }
+
+ if (source_cred.beamtime_id.empty()) {
+ source_cred.beamtime_id = SourceCredentials::kDefaultBeamtimeId;
+ }
+
+ if (source_cred.beamtime_id == SourceCredentials::kDefaultBeamtimeId
+ && source_cred.beamline == SourceCredentials::kDefaultBeamline) {
+ log__->Error("beamtime or beamline should be set");
+ source_cred_string_ = "";
+ return ProducerErrorTemplates::kWrongInput.Generate("beamtime or beamline should be set");
+ }
+
source_cred_string_ = source_cred.GetString();
if (source_cred_string_.size() + source_cred.user_token.size() > kMaxMessageSize) {
log__->Error("credentials string is too long - " + source_cred_string_);
diff --git a/producer/api/cpp/src/request_handler_filesystem.cpp b/producer/api/cpp/src/request_handler_filesystem.cpp
index fd75e003f20a45be41603a7b5d1c1caa1c55a95f..5e1437d25082d95613ff5251ca17f05e93fec78d 100644
--- a/producer/api/cpp/src/request_handler_filesystem.cpp
+++ b/producer/api/cpp/src/request_handler_filesystem.cpp
@@ -16,13 +16,14 @@ RequestHandlerFilesystem::RequestHandlerFilesystem(std::string destination_folde
thread_id_{thread_id} {
}
-bool RequestHandlerFilesystem::ProcessRequestUnlocked(GenericRequest* request) {
+bool RequestHandlerFilesystem::ProcessRequestUnlocked(GenericRequest* request, bool* retry) {
auto producer_request = static_cast<ProducerRequest*>(request);
Error err;
if (producer_request->DataFromFile()) {
producer_request->data = io__->GetDataFromFile(producer_request->original_filepath, &producer_request->header.data_size,
&err);
if (err) {
+ *retry = true;
return false;
}
}
@@ -32,6 +33,7 @@ bool RequestHandlerFilesystem::ProcessRequestUnlocked(GenericRequest* request) {
if (producer_request->callback) {
producer_request->callback(request->header, std::move(err));
}
+ *retry = false;
return true;
}
diff --git a/producer/api/cpp/src/request_handler_filesystem.h b/producer/api/cpp/src/request_handler_filesystem.h
index 72fb21061d0d20631aa457f5cf24937c14e0ff1a..66b1ca980683ead71925b3956a295b14fb65af7f 100644
--- a/producer/api/cpp/src/request_handler_filesystem.h
+++ b/producer/api/cpp/src/request_handler_filesystem.h
@@ -17,12 +17,12 @@ namespace asapo {
class RequestHandlerFilesystem: public RequestHandler {
public:
explicit RequestHandlerFilesystem(std::string destination_folder, uint64_t thread_id);
- bool ProcessRequestUnlocked(GenericRequest* request) override;
+ bool ProcessRequestUnlocked(GenericRequest* request, bool* retry) override;
bool ReadyProcessRequest() override {
return true;
};
void PrepareProcessingRequestLocked() override {};
- void TearDownProcessingRequestLocked(bool processing_succeeded) override {};
+ void TearDownProcessingRequestLocked(bool request_processed_successfully) override {};
void ProcessRequestTimeout(GenericRequest* request) override;
virtual ~RequestHandlerFilesystem() = default;
diff --git a/producer/api/cpp/src/request_handler_tcp.cpp b/producer/api/cpp/src/request_handler_tcp.cpp
index a1f25ffb56eef143c1ab0d8003855c827282ea51..5a898116c888ce78a912095021c3e396e4eba0ae 100644
--- a/producer/api/cpp/src/request_handler_tcp.cpp
+++ b/producer/api/cpp/src/request_handler_tcp.cpp
@@ -12,8 +12,8 @@ RequestHandlerTcp::RequestHandlerTcp(ReceiverDiscoveryService* discovery_service
ncurrent_connections_{shared_counter} {
}
-Error RequestHandlerTcp::Authorize(const std::string& beamtime_id) {
- GenericRequestHeader header{kOpcodeAuthorize, 0, 0, 0, beamtime_id.c_str()};
+Error RequestHandlerTcp::Authorize(const std::string& source_credentials) {
+ GenericRequestHeader header{kOpcodeAuthorize, 0, 0, 0, source_credentials.c_str()};
Error err;
io__->Send(sd_, &header, sizeof(header), &err);
if(err) {
@@ -23,7 +23,7 @@ Error RequestHandlerTcp::Authorize(const std::string& beamtime_id) {
}
-Error RequestHandlerTcp::ConnectToReceiver(const std::string& beamtime_id, const std::string& receiver_address) {
+Error RequestHandlerTcp::ConnectToReceiver(const std::string& source_credentials, const std::string& receiver_address) {
Error err;
sd_ = io__->CreateAndConnectIPTCPSocket(receiver_address, &err);
@@ -34,7 +34,7 @@ Error RequestHandlerTcp::ConnectToReceiver(const std::string& beamtime_id, const
log__->Debug("connected to receiver at " + receiver_address);
connected_receiver_uri_ = receiver_address;
- err = Authorize(beamtime_id);
+ err = Authorize(source_credentials);
if (err != nullptr) {
log__->Error("authorization failed at " + receiver_address + " - " + err->Explain());
Disconnect();
@@ -82,6 +82,7 @@ Error RequestHandlerTcp::ReceiveResponse(const GenericRequestHeader& request_hea
if(err != nullptr) {
return err;
}
+
switch (sendDataResponse.error_code) {
case kNetAuthorizationError : {
auto res_err = ProducerErrorTemplates::kWrongInput.Generate();
@@ -98,6 +99,10 @@ Error RequestHandlerTcp::ReceiveResponse(const GenericRequestHeader& request_hea
res_err->Append(sendDataResponse.message);
return res_err;
}
+ case kNetErrorReauthorize: {
+ auto res_err = ProducerErrorTemplates::kReAuthorizationNeeded.Generate();
+ return res_err;
+ }
case kNetErrorNoError :
return nullptr;
default:
@@ -206,31 +211,44 @@ bool RequestHandlerTcp::ProcessErrorFromReceiver(const Error& error,
}
-bool RequestHandlerTcp::SendDataToOneOfTheReceivers(ProducerRequest* request) {
+void RequestHandlerTcp::ProcessRequestCallback(Error err, ProducerRequest* request, bool* retry) {
+ if (request->callback) {
+ request->callback(request->header, std::move(err));
+ }
+ *retry = false;
+}
+
+
+bool RequestHandlerTcp::SendDataToOneOfTheReceivers(ProducerRequest* request, bool* retry) {
for (auto receiver_uri : receivers_list_) {
if (Disconnected()) {
auto err = ConnectToReceiver(request->source_credentials, receiver_uri);
- if (err != nullptr ) continue;
+ if (err == ProducerErrorTemplates::kWrongInput) {
+ ProcessRequestCallback(std::move(err), request, retry);
+ return false;
+ } else {
+ if (err != nullptr ) continue;
+ }
}
auto err = TrySendToReceiver(request);
- auto retry = ProcessErrorFromReceiver(err, request, receiver_uri);
- if (retry) {
+ bool server_error_can_retry = ProcessErrorFromReceiver(err, request, receiver_uri);
+ if (server_error_can_retry) {
continue;
}
- if (request->callback) {
- request->callback(request->header, std::move(err));
- }
- return true;
+ bool success = err && err != ProducerErrorTemplates::kServerWarning ? false : true;
+ ProcessRequestCallback(std::move(err), request, retry);
+ return success;
}
log__->Warning("put back to the queue, request opcode: " + std::to_string(request->header.op_code) +
", id: " + std::to_string(request->header.data_id));
+ *retry = true;
return false;
}
-bool RequestHandlerTcp::ProcessRequestUnlocked(GenericRequest* request) {
+bool RequestHandlerTcp::ProcessRequestUnlocked(GenericRequest* request, bool* retry) {
auto producer_request = static_cast<ProducerRequest*>(request);
auto err = producer_request->UpdateDataSizeFromFileIfNeeded(io__.get());
@@ -238,14 +256,15 @@ bool RequestHandlerTcp::ProcessRequestUnlocked(GenericRequest* request) {
if (producer_request->callback) {
producer_request->callback(producer_request->header, std::move(err));
}
- return true;
+ *retry = false;
+ return false;
}
if (NeedRebalance()) {
CloseConnectionToPeformRebalance();
}
- return SendDataToOneOfTheReceivers(producer_request);
+ return SendDataToOneOfTheReceivers(producer_request, retry);
}
bool RequestHandlerTcp::Connected() {
@@ -264,8 +283,8 @@ void RequestHandlerTcp::PrepareProcessingRequestLocked() {
UpdateIfNewConnection();
}
-void RequestHandlerTcp::TearDownProcessingRequestLocked(bool processing_succeeded) {
- if (!processing_succeeded) {
+void RequestHandlerTcp::TearDownProcessingRequestLocked(bool request_processed_successfully) {
+ if (!request_processed_successfully) {
(*ncurrent_connections_)--;
}
}
diff --git a/producer/api/cpp/src/request_handler_tcp.h b/producer/api/cpp/src/request_handler_tcp.h
index 0cba33e5dd747a3564a4bf29c453155ac5e1efaa..fa2d39b7110c989b66899b5b6c7b4f9889065ff0 100644
--- a/producer/api/cpp/src/request_handler_tcp.h
+++ b/producer/api/cpp/src/request_handler_tcp.h
@@ -19,10 +19,10 @@ namespace asapo {
class RequestHandlerTcp: public RequestHandler {
public:
explicit RequestHandlerTcp(ReceiverDiscoveryService* discovery_service, uint64_t thread_id, uint64_t* shared_counter);
- bool ProcessRequestUnlocked(GenericRequest* request) override;
+ bool ProcessRequestUnlocked(GenericRequest* request, bool* retry) override;
bool ReadyProcessRequest() override;
void PrepareProcessingRequestLocked() override;
- void TearDownProcessingRequestLocked(bool processing_succeeded) override;
+ void TearDownProcessingRequestLocked(bool request_processed_successfully) override;
void ProcessRequestTimeout(GenericRequest* request) override;
virtual ~RequestHandlerTcp() = default;
@@ -30,9 +30,9 @@ class RequestHandlerTcp: public RequestHandler {
const AbstractLogger* log__;
ReceiverDiscoveryService* discovery_service__;
private:
- Error Authorize(const std::string& beamtime_id);
- Error ConnectToReceiver(const std::string& beamtime_id, const std::string& receiver_address);
- bool SendDataToOneOfTheReceivers(ProducerRequest* request);
+ Error Authorize(const std::string& source_credentials);
+ Error ConnectToReceiver(const std::string& source_credentials, const std::string& receiver_address);
+ bool SendDataToOneOfTheReceivers(ProducerRequest* request, bool* retry);
Error SendRequestContent(const ProducerRequest* request);
Error ReceiveResponse(const GenericRequestHeader& request_header);
Error TrySendToReceiver(const ProducerRequest* request);
@@ -50,7 +50,7 @@ class RequestHandlerTcp: public RequestHandler {
bool ProcessErrorFromReceiver(const Error& error, const ProducerRequest* request, const std::string& receiver_uri);
ReceiversList receivers_list_;
system_clock::time_point last_receivers_uri_update_;
-
+ void ProcessRequestCallback(Error err, ProducerRequest* request, bool* retry);
uint64_t thread_id_;
uint64_t* ncurrent_connections_;
std::string connected_receiver_uri_;
diff --git a/producer/api/cpp/unittests/test_producer.cpp b/producer/api/cpp/unittests/test_producer.cpp
index 807bf22bed02f7b6db2a0ad11d60d83fb264e09a..9e82be03ed1e4dbc8ee3ea52da6303bebf53e904 100644
--- a/producer/api/cpp/unittests/test_producer.cpp
+++ b/producer/api/cpp/unittests/test_producer.cpp
@@ -15,7 +15,7 @@ namespace {
TEST(CreateProducer, TcpProducer) {
asapo::Error err;
std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("endpoint", 4, asapo::RequestHandlerType::kTcp,
- SourceCredentials{"bt", "", ""}, 3600, &err);
+ SourceCredentials{"bt", "", "", ""}, 3600, &err);
ASSERT_THAT(dynamic_cast<asapo::ProducerImpl*>(producer.get()), Ne(nullptr));
ASSERT_THAT(err, Eq(nullptr));
}
@@ -24,27 +24,24 @@ TEST(CreateProducer, ErrorBeamtime) {
asapo::Error err;
std::string expected_beamtimeid(asapo::kMaxMessageSize * 10, 'a');
std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("endpoint", 4, asapo::RequestHandlerType::kTcp,
- SourceCredentials{expected_beamtimeid, "", ""}, 3600, &err);
+ SourceCredentials{expected_beamtimeid, "", "", ""}, 3600, &err);
ASSERT_THAT(producer, Eq(nullptr));
ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput));
}
-
-//todo: memtest fails on old linux machine. Add valgrind suppressions?
-/*
-TEST(CreateProducer, FileSystemProducer) {
+TEST(CreateProducer, ErrorOnBothAutoBeamlineBeamtime) {
+ asapo::SourceCredentials creds{"auto", "auto", "subname", "token"};
asapo::Error err;
- std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("endpoint", 4,
- asapo::RequestHandlerType::kFilesystem, "bt", &err);
- ASSERT_THAT(dynamic_cast<asapo::ProducerImpl*>(producer.get()), Ne(nullptr));
- ASSERT_THAT(err, Eq(nullptr));
+ std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("endpoint", 4, asapo::RequestHandlerType::kTcp,
+ creds, 3600, &err);
+ ASSERT_THAT(producer, Eq(nullptr));
+ ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput));
}
-*/
TEST(CreateProducer, TooManyThreads) {
asapo::Error err;
std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("", asapo::kMaxProcessingThreads + 1,
- asapo::RequestHandlerType::kTcp, SourceCredentials{"bt", "", ""}, 3600, &err);
+ asapo::RequestHandlerType::kTcp, SourceCredentials{"bt", "", "", ""}, 3600, &err);
ASSERT_THAT(producer, Eq(nullptr));
ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput));
}
@@ -53,7 +50,7 @@ TEST(CreateProducer, TooManyThreads) {
TEST(CreateProducer, ZeroThreads) {
asapo::Error err;
std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("", 0,
- asapo::RequestHandlerType::kTcp, SourceCredentials{"bt", "", ""}, 3600, &err);
+ asapo::RequestHandlerType::kTcp, SourceCredentials{"bt", "", "", ""}, 3600, &err);
ASSERT_THAT(producer, Eq(nullptr));
ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput));
}
@@ -62,7 +59,7 @@ TEST(CreateProducer, ZeroThreads) {
TEST(Producer, SimpleWorkflowWihoutConnection) {
asapo::Error err;
std::unique_ptr<asapo::Producer> producer = asapo::Producer::Create("hello", 5, asapo::RequestHandlerType::kTcp,
- SourceCredentials{"bt", "", ""}, 3600,
+ SourceCredentials{"bt", "", "", ""}, 3600,
&err);
asapo::EventHeader event_header{1, 1, "test"};
diff --git a/producer/api/cpp/unittests/test_producer_impl.cpp b/producer/api/cpp/unittests/test_producer_impl.cpp
index aadeeca48cb08f1c1ad7f7f09cbd30caf7eb8393..1442d96d04f1d469a36c0d41a6ef24ad19aa0cd2 100644
--- a/producer/api/cpp/unittests/test_producer_impl.cpp
+++ b/producer/api/cpp/unittests/test_producer_impl.cpp
@@ -72,15 +72,14 @@ class ProducerImplTests : public testing::Test {
char expected_substream[asapo::kMaxMessageSize] = "test_substream";
std::string expected_next_substream = "next_substream";
- asapo::SourceCredentials expected_credentials{
- "beamtime_id", "subname", "token"
- };
+ asapo::SourceCredentials expected_credentials{"beamtime_id", "beamline", "subname", "token"
+ };
asapo::SourceCredentials expected_default_credentials{
- "beamtime_id", "", "token"
+ "beamtime_id", "", "", "token"
};
- std::string expected_credentials_str = "beamtime_id%subname%token";
- std::string expected_default_credentials_str = "beamtime_id%detector%token";
+ std::string expected_credentials_str = "beamtime_id%beamline%subname%token";
+ std::string expected_default_credentials_str = "beamtime_id%auto%detector%token";
std::string expected_metadata = "meta";
std::string expected_fullpath = "filename";
@@ -387,7 +386,7 @@ TEST_F(ProducerImplTests, OKSendingSendFileRequestWithSubstream) {
TEST_F(ProducerImplTests, ErrorSettingBeamtime) {
std::string long_str(asapo::kMaxMessageSize * 10, 'a');
- expected_credentials = asapo::SourceCredentials{long_str, "", ""};
+ expected_credentials = asapo::SourceCredentials{long_str, "", "", ""};
EXPECT_CALL(mock_logger, Error(testing::HasSubstr("too long")));
auto err = producer.SetCredentials(expected_credentials);
@@ -398,8 +397,8 @@ TEST_F(ProducerImplTests, ErrorSettingBeamtime) {
TEST_F(ProducerImplTests, ErrorSettingSecondTime) {
EXPECT_CALL(mock_logger, Error(testing::HasSubstr("already")));
- producer.SetCredentials(asapo::SourceCredentials{"1", "2", "3"});
- auto err = producer.SetCredentials(asapo::SourceCredentials{"4", "5", "6"});
+ producer.SetCredentials(asapo::SourceCredentials{"1", "", "2", "3"});
+ auto err = producer.SetCredentials(asapo::SourceCredentials{"4", "", "5", "6"});
ASSERT_THAT(err, Eq(asapo::ProducerErrorTemplates::kWrongInput));
}
diff --git a/producer/api/cpp/unittests/test_request_handler_filesystem.cpp b/producer/api/cpp/unittests/test_request_handler_filesystem.cpp
index 4e443b971714031c162298d4e418cfc6f59682af..da7be9fba592edbd80c11cf97cf591a09d16b293 100644
--- a/producer/api/cpp/unittests/test_request_handler_filesystem.cpp
+++ b/producer/api/cpp/unittests/test_request_handler_filesystem.cpp
@@ -50,7 +50,7 @@ class RequestHandlerFilesystemTests : public testing::Test {
std::string expected_destination = "destination";
std::string expected_fullpath = expected_destination + "/" + expected_file_name;
std::string expected_origin_fullpath = std::string("origin/") + expected_file_name;
-
+ bool retry;
asapo::Opcode expected_op_code = asapo::kOpcodeTransferData;
asapo::Error callback_err;
asapo::GenericRequestHeader header{expected_op_code, expected_file_id, expected_file_size,
@@ -104,7 +104,7 @@ TEST_F(RequestHandlerFilesystemTests, CallBackErrorIfCannotSaveFile) {
);
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(callback_err, Eq(asapo::IOErrorTemplates::kUnknownIOError));
ASSERT_THAT(called, Eq(true));
@@ -119,7 +119,7 @@ TEST_F(RequestHandlerFilesystemTests, WorksWithemptyCallback) {
);
- auto success = request_handler.ProcessRequestUnlocked(&request_nocallback);
+ auto success = request_handler.ProcessRequestUnlocked(&request_nocallback, &retry);
ASSERT_THAT(called, Eq(false));
ASSERT_THAT(success, Eq(true));
@@ -135,7 +135,7 @@ TEST_F(RequestHandlerFilesystemTests, FileRequestErrorOnReadData) {
Return(nullptr)
));
- auto success = request_handler.ProcessRequestUnlocked(&request_filesend);
+ auto success = request_handler.ProcessRequestUnlocked(&request_filesend, &retry);
ASSERT_THAT(success, Eq(false));
}
@@ -154,7 +154,7 @@ TEST_F(RequestHandlerFilesystemTests, FileRequestOK) {
Return(nullptr)
);
- auto success = request_handler.ProcessRequestUnlocked(&request_filesend);
+ auto success = request_handler.ProcessRequestUnlocked(&request_filesend, &retry);
ASSERT_THAT(success, Eq(true));
}
@@ -169,7 +169,7 @@ TEST_F(RequestHandlerFilesystemTests, TransferOK) {
);
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(true));
ASSERT_THAT(callback_err, Eq(nullptr));
diff --git a/producer/api/cpp/unittests/test_request_handler_tcp.cpp b/producer/api/cpp/unittests/test_request_handler_tcp.cpp
index ab5cebbc6ddb46e94d67ad476d7596fdff2bb6c1..b8f480184955b5c1c49ab2dc161e7b63fd3e98d8 100644
--- a/producer/api/cpp/unittests/test_request_handler_tcp.cpp
+++ b/producer/api/cpp/unittests/test_request_handler_tcp.cpp
@@ -101,7 +101,8 @@ class RequestHandlerTcpTests : public testing::Test {
std::vector<asapo::SocketDescriptor> expected_sds{83942, 83943};
- Sequence seq_receive;
+ bool retry;
+ Sequence seq_receive[2];
void ExpectFailConnect(bool only_once = false);
void ExpectFailAuthorize(bool only_once = false);
void ExpectOKAuthorize(bool only_once = false);
@@ -163,11 +164,12 @@ void RequestHandlerTcpTests::ExpectFailConnect(bool only_once) {
testing::SetArgPointee<1>(asapo::IOErrorTemplates::kInvalidAddressFormat.Generate().release()),
Return(asapo::kDisconnectedSocketDescriptor)
));
- EXPECT_CALL(mock_logger, Debug(AllOf(
- HasSubstr("cannot connect"),
- HasSubstr(expected_address)
- )
- ));
+ if (only_once)
+ EXPECT_CALL(mock_logger, Debug(AllOf(
+ HasSubstr("cannot connect"),
+ HasSubstr(expected_address)
+ )
+ ));
if (only_once) break;
}
@@ -188,7 +190,7 @@ void RequestHandlerTcpTests::ExpectFailAuthorize(bool only_once) {
));
EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendDataResponse), _))
- .InSequence(seq_receive)
+ .InSequence(seq_receive[i])
.WillOnce(
DoAll(
testing::SetArgPointee<3>(nullptr),
@@ -196,18 +198,20 @@ void RequestHandlerTcpTests::ExpectFailAuthorize(bool only_once) {
testing::ReturnArg<2>()
));
EXPECT_CALL(mock_io, CloseSocket_t(expected_sd, _));
- EXPECT_CALL(mock_logger, Debug(AllOf(
- HasSubstr("disconnected"),
- HasSubstr(receivers_list[i])
- )
- ));
+ if (only_once) {
+ EXPECT_CALL(mock_logger, Debug(AllOf(
+ HasSubstr("disconnected"),
+ HasSubstr(receivers_list[i])
+ )
+ ));
- EXPECT_CALL(mock_logger, Error(AllOf(
- HasSubstr("authorization"),
- HasSubstr(expected_auth_message),
- HasSubstr(receivers_list[i])
- )
- ));
+ EXPECT_CALL(mock_logger, Error(AllOf(
+ HasSubstr("authorization"),
+ HasSubstr(expected_auth_message),
+ HasSubstr(receivers_list[i])
+ )
+ ));
+ }
if (only_once) break;
i++;
}
@@ -227,18 +231,20 @@ void RequestHandlerTcpTests::ExpectOKAuthorize(bool only_once) {
EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendDataResponse), _))
- .InSequence(seq_receive)
+ .InSequence(seq_receive[i])
.WillOnce(
DoAll(
testing::SetArgPointee<3>(nullptr),
A_WriteSendDataResponse(asapo::kNetErrorNoError, expected_auth_message),
testing::ReturnArg<2>()
));
- EXPECT_CALL(mock_logger, Info(AllOf(
- HasSubstr("authorized"),
- HasSubstr(receivers_list[i])
- )
- ));
+ if (only_once) {
+ EXPECT_CALL(mock_logger, Info(AllOf(
+ HasSubstr("authorized"),
+ HasSubstr(receivers_list[i])
+ )
+ ));
+ }
if (only_once) break;
i++;
}
@@ -258,22 +264,26 @@ void RequestHandlerTcpTests::ExpectFailSendHeader(bool only_once) {
testing::SetArgPointee<3>(asapo::IOErrorTemplates::kBadFileNumber.Generate().release()),
Return(-1)
));
- EXPECT_CALL(mock_logger, Debug(AllOf(
- HasSubstr("disconnected"),
- HasSubstr(receivers_list[i])
- )
- ));
+ if (only_once) {
+ EXPECT_CALL(mock_logger, Debug(AllOf(
+ HasSubstr("disconnected"),
+ HasSubstr(receivers_list[i])
+ )
+ ));
- EXPECT_CALL(mock_logger, Warning(AllOf(
- HasSubstr("cannot send"),
- HasSubstr(receivers_list[i])
- )
- ));
+ EXPECT_CALL(mock_logger, Warning(AllOf(
+ HasSubstr("cannot send"),
+ HasSubstr(receivers_list[i])
+ )
+ ));
+ }
EXPECT_CALL(mock_io, CloseSocket_t(expected_sd, _));
if (only_once) break;
i++;
}
- EXPECT_CALL(mock_logger, Warning(HasSubstr("put back")));
+ if (only_once) {
+ EXPECT_CALL(mock_logger, Warning(HasSubstr("put back")));
+ }
}
void RequestHandlerTcpTests::ExpectFailSendFile(const asapo::ProducerErrorTemplate& err_template, bool client_error) {
@@ -284,27 +294,26 @@ void RequestHandlerTcpTests::ExpectFailSendFile(const asapo::ProducerErrorTempla
.WillOnce(
Return(err_template.Generate().release())
);
- EXPECT_CALL(mock_logger, Debug(AllOf(
- HasSubstr("disconnected"),
- HasSubstr(receivers_list[i])
- )
- ));
+
if (client_error) {
+
+ EXPECT_CALL(mock_logger, Debug(AllOf(
+ HasSubstr("disconnected"),
+ HasSubstr(receivers_list[i])
+ )
+ ));
EXPECT_CALL(mock_logger, Error(AllOf(
HasSubstr("cannot send"),
HasSubstr(receivers_list[i]) )
));
- } else {
- EXPECT_CALL(mock_logger, Warning(AllOf(
- HasSubstr("cannot send"),
- HasSubstr(receivers_list[i]) )
- ));
+
}
+
EXPECT_CALL(mock_io, CloseSocket_t(expected_sd, _));
if (client_error) break;
i++;
}
- if (err_template != asapo::ProducerErrorTemplates::kLocalIOError.Generate()) {
+ if (client_error && err_template != asapo::ProducerErrorTemplates::kLocalIOError.Generate()) {
EXPECT_CALL(mock_logger, Warning(HasSubstr("put back")));
}
@@ -321,25 +330,27 @@ void RequestHandlerTcpTests::ExpectFailSend(uint64_t expected_size, bool only_on
testing::SetArgPointee<3>(asapo::IOErrorTemplates::kBadFileNumber.Generate().release()),
Return(-1)
));
- EXPECT_CALL(mock_logger, Debug(AllOf(
- HasSubstr("disconnected"),
- HasSubstr(receivers_list[i])
- )
- ));
+ if (only_once) {
+ EXPECT_CALL(mock_logger, Debug(AllOf(
+ HasSubstr("disconnected"),
+ HasSubstr(receivers_list[i])
+ )
+ ));
- EXPECT_CALL(mock_logger, Warning(AllOf(
- HasSubstr("cannot send"),
- HasSubstr(receivers_list[i])
- )
- ));
+ EXPECT_CALL(mock_logger, Warning(AllOf(
+ HasSubstr("cannot send"),
+ HasSubstr(receivers_list[i])
+ )
+ ));
+
+ }
EXPECT_CALL(mock_io, CloseSocket_t(expected_sd, _));
if (only_once) break;
i++;
}
- EXPECT_CALL(mock_logger, Warning(HasSubstr("put back")));
+ if (only_once) EXPECT_CALL(mock_logger, Warning(HasSubstr("put back")));
}
-
void RequestHandlerTcpTests::ExpectFailSendData(bool only_once) {
ExpectFailSend(expected_file_size, only_once);
}
@@ -354,7 +365,7 @@ void RequestHandlerTcpTests::ExpectFailReceive(bool only_once) {
int i = 0;
for (auto expected_sd : expected_sds) {
EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendDataResponse), _))
- .InSequence(seq_receive)
+ .InSequence(seq_receive[i])
.WillOnce(
DoAll(
testing::SetArgPointee<3>(asapo::IOErrorTemplates::kBadFileNumber.Generate().release()),
@@ -444,11 +455,13 @@ void RequestHandlerTcpTests::ExpectOKConnect(bool only_once) {
testing::SetArgPointee<1>(nullptr),
Return(expected_sds[i])
));
- EXPECT_CALL(mock_logger, Debug(AllOf(
- HasSubstr("connected to"),
- HasSubstr(expected_address)
- )
- ));
+ if (only_once) {
+ EXPECT_CALL(mock_logger, Debug(AllOf(
+ HasSubstr("connected to"),
+ HasSubstr(expected_address)
+ )
+ ));
+ }
if (only_once) break;
i++;
}
@@ -459,18 +472,20 @@ void RequestHandlerTcpTests::ExpectOKReceive(bool only_once, asapo::NetworkError
int i = 0;
for (auto expected_sd : expected_sds) {
EXPECT_CALL(mock_io, Receive_t(expected_sd, _, sizeof(asapo::SendDataResponse), _))
- .InSequence(seq_receive)
+ .InSequence(seq_receive[i])
.WillOnce(
DoAll(
testing::SetArgPointee<3>(nullptr),
A_WriteSendDataResponse(code, message),
testing::ReturnArg<2>()
));
- EXPECT_CALL(mock_logger, Debug(AllOf(
- HasSubstr("sent data"),
- HasSubstr(receivers_list[i])
- )
- ));
+ if (only_once) {
+ EXPECT_CALL(mock_logger, Debug(AllOf(
+ HasSubstr("sent data"),
+ HasSubstr(receivers_list[i])
+ )
+ ));
+ }
if (only_once) break;
i++;
}
@@ -495,7 +510,7 @@ void RequestHandlerTcpTests::DoSingleSend(bool connect, bool success) {
}
request_handler.PrepareProcessingRequestLocked();
- request_handler.ProcessRequestUnlocked(&request);
+ request_handler.ProcessRequestUnlocked(&request, &retry);
Mock::VerifyAndClearExpectations(&mock_io);
Mock::VerifyAndClearExpectations(&mock_logger);
@@ -531,11 +546,9 @@ TEST_F(RequestHandlerTcpTests, DoesNotGetsUriIfAlreadyConnected) {
TEST_F(RequestHandlerTcpTests, ReduceConnectionNumberAtTearDownIfError) {
n_connections = 1;
-
request_handler.TearDownProcessingRequestLocked(false);
ASSERT_THAT(n_connections, Eq(0));
-
}
TEST_F(RequestHandlerTcpTests, DoNotReduceConnectionNumberAtTearDownIfNoError) {
@@ -551,21 +564,27 @@ TEST_F(RequestHandlerTcpTests, TriesConnectWhenNotConnected) {
ExpectFailConnect();
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(false));
+ ASSERT_THAT(retry, Eq(true));
}
TEST_F(RequestHandlerTcpTests, FailsWhenCannotAuthorize) {
- ExpectOKConnect();
- ExpectFailAuthorize();
+ ExpectOKConnect(true);
+ ExpectFailAuthorize(true);
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
request_handler.TearDownProcessingRequestLocked(success);
- ASSERT_THAT(success, Eq(false));
ASSERT_THAT(n_connections, Eq(0));
+ ASSERT_THAT(callback_err, Eq(asapo::ProducerErrorTemplates::kWrongInput));
+ ASSERT_THAT(callback_called, Eq(true));
+ ASSERT_THAT(success, Eq(false));
+ ASSERT_THAT(retry, Eq(false));
+
+
}
@@ -581,11 +600,12 @@ TEST_F(RequestHandlerTcpTests, DoesNotTryConnectWhenConnected) {
ExpectFailSendHeader(true);
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(false));
-}
+ ASSERT_THAT(retry, Eq(true));
+}
TEST_F(RequestHandlerTcpTests, DoNotCloseWhenNotConnected) {
@@ -595,9 +615,11 @@ TEST_F(RequestHandlerTcpTests, DoNotCloseWhenNotConnected) {
ExpectFailSendHeader();
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(false));
+ ASSERT_THAT(retry, Eq(true));
+
}
@@ -610,9 +632,11 @@ TEST_F(RequestHandlerTcpTests, CloseConnectionWhenRebalance) {
EXPECT_CALL(mock_io, CloseSocket_t(_, _));
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(false));
+ ASSERT_THAT(retry, Eq(true));
+
}
@@ -623,9 +647,11 @@ TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendHeader) {
ExpectFailSendHeader();
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(false));
+ ASSERT_THAT(retry, Eq(true));
+
}
@@ -637,9 +663,11 @@ TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendData) {
ExpectFailSendData();
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(false));
+ ASSERT_THAT(retry, Eq(true));
+
}
TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendMetaData) {
@@ -649,9 +677,11 @@ TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendMetaData) {
ExpectFailSendMetaData();
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(false));
+ ASSERT_THAT(retry, Eq(true));
+
}
TEST_F(RequestHandlerTcpTests, ErrorWhenCannotReceiveData) {
@@ -664,9 +694,11 @@ TEST_F(RequestHandlerTcpTests, ErrorWhenCannotReceiveData) {
ExpectFailReceive(true);
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(false));
+ ASSERT_THAT(retry, Eq(true));
+
}
void RequestHandlerTcpTests::AssertImmediatelyCallBack(asapo::NetworkErrorCode error_code,
@@ -676,7 +708,7 @@ void RequestHandlerTcpTests::AssertImmediatelyCallBack(asapo::NetworkErrorCode e
ExpectOKSendAll(true);
EXPECT_CALL(mock_io, Receive_t(expected_sds[0], _, sizeof(asapo::SendDataResponse), _))
- .InSequence(seq_receive)
+ .InSequence(seq_receive[0])
.WillOnce(
DoAll(
testing::SetArgPointee<3>(nullptr),
@@ -696,10 +728,12 @@ void RequestHandlerTcpTests::AssertImmediatelyCallBack(asapo::NetworkErrorCode e
));
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(callback_err, Eq(err_template));
ASSERT_THAT(callback_called, Eq(true));
- ASSERT_THAT(success, Eq(true));
+ ASSERT_THAT(success, Eq(false));
+ ASSERT_THAT(retry, Eq(false));
+
}
void RequestHandlerTcpTests::ExpectGetFileSize(bool ok) {
@@ -732,10 +766,11 @@ TEST_F(RequestHandlerTcpTests, SendEmptyCallBack) {
ExpectOKReceive();
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request_nocallback);
+ auto success = request_handler.ProcessRequestUnlocked(&request_nocallback, &retry);
ASSERT_THAT(success, Eq(true));
ASSERT_THAT(callback_called, Eq(false));
+ ASSERT_THAT(retry, Eq(false));
}
TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendFileWithReadError) {
@@ -747,11 +782,13 @@ TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendFileWithReadError) {
ExpectFailSendFile(asapo::ProducerErrorTemplates::kLocalIOError, true);
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request_filesend);
+ auto success = request_handler.ProcessRequestUnlocked(&request_filesend, &retry);
- ASSERT_THAT(success, Eq(true));
+ ASSERT_THAT(success, Eq(false));
ASSERT_THAT(callback_called, Eq(true));
ASSERT_THAT(callback_err, Eq(asapo::ProducerErrorTemplates::kLocalIOError));
+ ASSERT_THAT(retry, Eq(false));
+
}
TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendFileWithServerError) {
@@ -763,21 +800,42 @@ TEST_F(RequestHandlerTcpTests, ErrorWhenCannotSendFileWithServerError) {
ExpectFailSendFile(asapo::ProducerErrorTemplates::kInternalServerError);
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request_filesend);
+ auto success = request_handler.ProcessRequestUnlocked(&request_filesend, &retry);
ASSERT_THAT(success, Eq(false));
ASSERT_THAT(callback_called, Eq(false));
+ ASSERT_THAT(retry, Eq(true));
}
+
+
+TEST_F(RequestHandlerTcpTests, RetryOnReauthorize) {
+ ExpectOKConnect(false);
+ ExpectOKAuthorize(false);
+ ExpectOKSendAll(false);
+ ExpectOKReceive(false, asapo::kNetErrorReauthorize);
+
+ request_handler.PrepareProcessingRequestLocked();
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
+
+ ASSERT_THAT(success, Eq(false));
+ ASSERT_THAT(callback_called, Eq(false));
+ ASSERT_THAT(retry, Eq(true));
+}
+
+
+
TEST_F(RequestHandlerTcpTests, FileRequestErrorGettingFileSize) {
ExpectGetFileSize(false);
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request_filesend);
- ASSERT_THAT(success, Eq(true));
+ auto success = request_handler.ProcessRequestUnlocked(&request_filesend, &retry);
+ ASSERT_THAT(success, Eq(false));
ASSERT_THAT(callback_called, Eq(true));
ASSERT_THAT(callback_err, Eq(asapo::ProducerErrorTemplates::kLocalIOError));
+ ASSERT_THAT(retry, Eq(false));
+
}
@@ -793,10 +851,12 @@ TEST_F(RequestHandlerTcpTests, FileRequestOK) {
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request_filesend);
+ auto success = request_handler.ProcessRequestUnlocked(&request_filesend, &retry);
ASSERT_THAT(success, Eq(true));
ASSERT_THAT(callback_called, Eq(true));
ASSERT_THAT(callback_err, Eq(nullptr));
+ ASSERT_THAT(retry, Eq(false));
+
}
@@ -808,9 +868,10 @@ TEST_F(RequestHandlerTcpTests, SendOK) {
ExpectOKReceive();
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(true));
+ ASSERT_THAT(retry, Eq(false));
ASSERT_THAT(callback_err, Eq(nullptr));
ASSERT_THAT(callback_called, Eq(true));
ASSERT_THAT(callback_header.data_size, Eq(header.data_size));
@@ -832,9 +893,11 @@ TEST_F(RequestHandlerTcpTests, SendMetadataIgnoresIngestMode) {
request.header.op_code = asapo::kOpcodeTransferMetaData;
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(true));
+ ASSERT_THAT(retry, Eq(false));
+
}
@@ -849,9 +912,10 @@ TEST_F(RequestHandlerTcpTests, SendMetaOnlyOK) {
request.header.custom_data[asapo::kPosIngestMode] = ingest_mode;
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(true));
+ ASSERT_THAT(retry, Eq(false));
ASSERT_THAT(callback_header.custom_data[asapo::kPosIngestMode], Eq(ingest_mode));
}
@@ -870,8 +934,10 @@ TEST_F(RequestHandlerTcpTests, SendMetaOnlyForFileReadOK) {
auto ingest_mode = asapo::IngestModeFlags::kTransferMetaDataOnly;
request_filesend.header.custom_data[asapo::kPosIngestMode] = ingest_mode;
- auto success = request_handler.ProcessRequestUnlocked(&request_filesend);
+ auto success = request_handler.ProcessRequestUnlocked(&request_filesend, &retry);
ASSERT_THAT(success, Eq(true));
+ ASSERT_THAT(retry, Eq(false));
+
}
@@ -902,9 +968,10 @@ TEST_F(RequestHandlerTcpTests, SendWithWarning) {
request_handler.PrepareProcessingRequestLocked();
- auto success = request_handler.ProcessRequestUnlocked(&request);
+ auto success = request_handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(true));
+ ASSERT_THAT(retry, Eq(false));
ASSERT_THAT(callback_err, Eq(asapo::ProducerErrorTemplates::kServerWarning));
ASSERT_THAT(callback_err->Explain(), HasSubstr(expected_warning));
ASSERT_THAT(callback_called, Eq(true));
@@ -914,6 +981,4 @@ TEST_F(RequestHandlerTcpTests, SendWithWarning) {
ASSERT_THAT(std::string{callback_header.message}, Eq(std::string{header.message}));
}
-
-
}
diff --git a/producer/api/python/asapo_producer.pxd b/producer/api/python/asapo_producer.pxd
index 22fdedcba6517b42f39381a17f2c19e557c448cb..b4e2386bc8bfde03406cc82a9f44751292133d96 100644
--- a/producer/api/python/asapo_producer.pxd
+++ b/producer/api/python/asapo_producer.pxd
@@ -47,6 +47,7 @@ cdef extern from "asapo_producer.h" namespace "asapo":
cdef extern from "asapo_producer.h" namespace "asapo":
struct SourceCredentials:
string beamtime_id
+ string beamline
string stream
string user_token
diff --git a/producer/api/python/asapo_producer.pyx.in b/producer/api/python/asapo_producer.pyx.in
index e606d7d892f1e2b0de8ab5c7858aa29359637f09..80b43bc3bbc641f88ed3ceb8adf808aef4c64142 100644
--- a/producer/api/python/asapo_producer.pyx.in
+++ b/producer/api/python/asapo_producer.pyx.in
@@ -278,11 +278,12 @@ cdef class PyProducer:
if self.c_producer.get() is not NULL:
self.c_producer.get().StopThreads__()
@staticmethod
- def __create_producer(endpoint,beamtime_id,stream,token,nthreads,timeout_sec):
+ def __create_producer(endpoint,beamtime_id,beamline,stream,token,nthreads,timeout_sec):
pyProd = PyProducer()
cdef Error err
cdef SourceCredentials source
source.beamtime_id = beamtime_id
+ source.beamline = beamline
source.user_token = token
source.stream = stream
pyProd.c_producer = Producer.Create(endpoint,nthreads,RequestHandlerType_Tcp,source,timeout_sec,&err)
@@ -290,12 +291,14 @@ cdef class PyProducer:
throw_exception(err)
return pyProd
-def create_producer(endpoint,beamtime_id,stream,token,nthreads,timeout_sec):
+def create_producer(endpoint,beamtime_id,beamline,stream,token,nthreads,timeout_sec):
"""
:param endpoint: server endpoint (url:port)
:type endpoint: string
- :param beamtime_id: beamtime id
+ :param beamtime_id: beamtime id, can be "auto" if beamline is given, will automatically select the current beamtime id
:type beamtime_id: string
+ :param beamline: beamline name, can be "auto" if beamtime_id is given
+ :type beamline: string
:param stream: stream to producer data to
:type stream: string
:param token: authorization token
@@ -308,7 +311,7 @@ def create_producer(endpoint,beamtime_id,stream,token,nthreads,timeout_sec):
AsapoWrongInputError: wrong input (number of threads, ,,,)
AsapoProducerError: actually should not happen
"""
- return PyProducer.__create_producer(_bytes(endpoint),_bytes(beamtime_id),_bytes(stream),_bytes(token),nthreads,timeout_sec)
+ return PyProducer.__create_producer(_bytes(endpoint),_bytes(beamtime_id),_bytes(beamline),_bytes(stream),_bytes(token),nthreads,timeout_sec)
__version__ = "@ASAPO_VERSION_PYTHON@"
diff --git a/producer/event_monitor_producer/src/main_eventmon.cpp b/producer/event_monitor_producer/src/main_eventmon.cpp
index cd5899c3c01c0992404c33e733e38c0012e0aadf..ff57a2dc3f568b0b69d7e66c275df96d957c26f0 100644
--- a/producer/event_monitor_producer/src/main_eventmon.cpp
+++ b/producer/event_monitor_producer/src/main_eventmon.cpp
@@ -39,7 +39,7 @@ std::unique_ptr<Producer> CreateProducer() {
Error err;
auto producer = Producer::Create(config->asapo_endpoint, (uint8_t) config->nthreads,
- config->mode, asapo::SourceCredentials{config->beamtime_id, config->stream, ""}, 3600, &err);
+ config->mode, asapo::SourceCredentials{config->beamtime_id, "", config->stream, ""}, 3600, &err);
if(err) {
std::cerr << "cannot create producer: " << err << std::endl;
exit(EXIT_FAILURE);
diff --git a/receiver/src/receive_file_processor.cpp b/receiver/src/receive_file_processor.cpp
index 512ca14386a88bf52dda49f6a2980830367cf84a..15311fc33e336270a70f45b72ad9f233896d5aae 100644
--- a/receiver/src/receive_file_processor.cpp
+++ b/receiver/src/receive_file_processor.cpp
@@ -16,7 +16,7 @@ Error ReceiveFileProcessor::ProcessFile(const Request* request, bool overwrite)
auto fsize = request->GetDataSize();
auto socket = request->GetSocket();
auto fname = request->GetFileName();
- auto root_folder = request->GetFullPath(GetReceiverConfig()->root_folder);
+ auto root_folder = request->GetOfflinePath();
auto err = io__->ReceiveDataToFile(socket, root_folder, fname, (size_t) fsize, true, overwrite);
if (!err) {
log__->Debug("received file of size " + std::to_string(fsize) + " to " + root_folder + kPathSeparator + fname);
diff --git a/receiver/src/receiver_config.cpp b/receiver/src/receiver_config.cpp
index 05db25037661931b6596abf59f8ce0cf5c7508a5..f82f44afe1edcd0cd5c493a178affb3127d8c690 100644
--- a/receiver/src/receiver_config.cpp
+++ b/receiver/src/receiver_config.cpp
@@ -33,7 +33,6 @@ Error ReceiverConfigFactory::SetConfig(std::string file_name) {
(err = parser.GetString("Tag", &config.tag)) ||
(err = parser.GetString("AuthorizationServer", &config.authorization_server)) ||
(err = parser.GetUInt64("AuthorizationInterval", &config.authorization_interval_ms)) ||
- (err = parser.GetString("RootFolder", &config.root_folder)) ||
(err = parser.GetString("PerformanceDbName", &config.performance_db_name)) ||
(err = parser.GetString("AdvertiseIP", &config.advertise_ip)) ||
(err = parser.GetString("LogLevel", &log_level));
diff --git a/receiver/src/receiver_config.h b/receiver/src/receiver_config.h
index 1152160264f6eb741ce028b707d59d2c8839a4ca..c69ff7990d88bec1b8db6f65194d00c25e045cbe 100644
--- a/receiver/src/receiver_config.h
+++ b/receiver/src/receiver_config.h
@@ -12,7 +12,6 @@ struct ReceiverConfig {
std::string performance_db_uri;
std::string performance_db_name;
std::string database_uri;
- std::string root_folder;
uint64_t listen_port = 0;
std::string authorization_server;
uint64_t authorization_interval_ms = 0;
diff --git a/receiver/src/receiver_data_server/receiver_data_server_request_handler.cpp b/receiver/src/receiver_data_server/receiver_data_server_request_handler.cpp
index f3d1672a8f05c144be3ac4bdd602bec2387e203b..03d8ae764549afee64c6e3e5f4a555bcf2f67ac6 100644
--- a/receiver/src/receiver_data_server/receiver_data_server_request_handler.cpp
+++ b/receiver/src/receiver_data_server/receiver_data_server_request_handler.cpp
@@ -46,7 +46,8 @@ void* ReceiverDataServerRequestHandler::GetSlot(const ReceiverDataServerRequest*
}
-bool ReceiverDataServerRequestHandler::ProcessRequestUnlocked(GenericRequest* request) {
+bool ReceiverDataServerRequestHandler::ProcessRequestUnlocked(GenericRequest* request, bool* retry) {
+ *retry = false;
auto receiver_request = dynamic_cast<ReceiverDataServerRequest*>(request);
if (!CheckRequest(receiver_request)) {
SendResponce(receiver_request, kNetErrorWrongRequest);
diff --git a/receiver/src/receiver_data_server/receiver_data_server_request_handler.h b/receiver/src/receiver_data_server/receiver_data_server_request_handler.h
index 0ca1a46b5fb0c8d789a389e54eddc092e652f057..4bae2ee45e590dc1d3e699d72e90675c04cc2b9a 100644
--- a/receiver/src/receiver_data_server/receiver_data_server_request_handler.h
+++ b/receiver/src/receiver_data_server/receiver_data_server_request_handler.h
@@ -13,7 +13,7 @@ namespace asapo {
class ReceiverDataServerRequestHandler: public RequestHandler {
public:
explicit ReceiverDataServerRequestHandler(const NetServer* server, DataCache* data_cache, Statistics* statistics);
- bool ProcessRequestUnlocked(GenericRequest* request) override;
+ bool ProcessRequestUnlocked(GenericRequest* request, bool* retry) override;
bool ReadyProcessRequest() override;
void PrepareProcessingRequestLocked() override;
void TearDownProcessingRequestLocked(bool processing_succeeded) override;
diff --git a/receiver/src/receiver_error.h b/receiver/src/receiver_error.h
index a25c9e78f79e4760e8d22e6bf81e33f16799265b..9a2fc4874ba8bd0e8696a0acce7462d602a49262 100644
--- a/receiver/src/receiver_error.h
+++ b/receiver/src/receiver_error.h
@@ -10,6 +10,7 @@ enum class ReceiverErrorType {
kBadRequest,
kAuthorizationFailure,
kInternalServerError,
+ kReAuthorizationFailure,
kWarningDuplicatedRequest
};
@@ -40,6 +41,10 @@ auto const kAuthorizationFailure = ReceiverErrorTemplate{
"authorization failure", ReceiverErrorType::kAuthorizationFailure
};
+auto const kReAuthorizationFailure = ReceiverErrorTemplate{
+ "reauthorization for auto beamtime failed", ReceiverErrorType::kReAuthorizationFailure
+};
+
};
}
diff --git a/receiver/src/request.cpp b/receiver/src/request.cpp
index c34a9f421e200f0638c438f63f9a9cdcf5070c83..2aee805842afd564c644840a363fb59add2c0039 100644
--- a/receiver/src/request.cpp
+++ b/receiver/src/request.cpp
@@ -153,30 +153,20 @@ uint64_t Request::GetMetaDataSize() const {
return request_header_.meta_size;
}
-void Request::SetFacility(std::string facility) {
- facility_ = std::move(facility);
+void Request::SetOnlinePath(std::string facility) {
+ offline_path_ = std::move(facility);
}
-void Request::SetBeamtimeYear(std::string year) {
- beamtime_year_ = std::move(year);
+void Request::SetOfflinePath(std::string path) {
+ online_path_ = std::move(path);
}
-const std::string& Request::GetFacility() const {
- return facility_;
+const std::string& Request::GetOnlinePath() const {
+ return offline_path_;
}
-const std::string& Request::GetBeamtimeYear() const {
- return beamtime_year_;
-}
-
-std::string Request::GetFullPath(std::string root_folder) const {
- return std::move(root_folder) + kPathSeparator
- + GetFacility() + kPathSeparator
- + "gpfs" + kPathSeparator
- + GetBeamline() + kPathSeparator
- + GetBeamtimeYear() + kPathSeparator
- + "data" + kPathSeparator
- + GetBeamtimeId();
+const std::string& Request::GetOfflinePath() const {
+ return online_path_;
}
bool Request::WasAlreadyProcessed() const {
diff --git a/receiver/src/request.h b/receiver/src/request.h
index 11eb6ca2a60ba7e7b3e27be5aa816fa77b7c86d5..eab5cae356cb4678d366200c2e33f1c51bc7da3a 100644
--- a/receiver/src/request.h
+++ b/receiver/src/request.h
@@ -51,13 +51,10 @@ class Request {
VIRTUAL void SetStream(std::string stream);
VIRTUAL void SetMetadata(std::string metadata);
- VIRTUAL void SetFacility(std::string facility);
- VIRTUAL void SetBeamtimeYear(std::string year);
- VIRTUAL const std::string& GetFacility() const;
- VIRTUAL const std::string& GetBeamtimeYear() const;
-
- VIRTUAL std::string GetFullPath(std::string root_folder) const;
-
+ VIRTUAL void SetOnlinePath(std::string facility);
+ VIRTUAL void SetOfflinePath(std::string path);
+ VIRTUAL const std::string& GetOnlinePath() const;
+ VIRTUAL const std::string& GetOfflinePath() const;
VIRTUAL const std::string& GetBeamline() const;
VIRTUAL const CustomRequestData& GetCustomData() const;
@@ -82,8 +79,8 @@ class Request {
std::string beamtime_id_;
std::string stream_;
std::string beamline_;
- std::string facility_;
- std::string beamtime_year_;
+ std::string offline_path_;
+ std::string online_path_;
std::string metadata_;
CacheMeta* slot_meta_ = nullptr;
bool already_processed_ = false;
diff --git a/receiver/src/request_handler_authorize.cpp b/receiver/src/request_handler_authorize.cpp
index 0aed9549de6726afc88ccf110005368a545129ad..d158882d38756fe96e90f281f98db7ac24f4023f 100644
--- a/receiver/src/request_handler_authorize.cpp
+++ b/receiver/src/request_handler_authorize.cpp
@@ -15,10 +15,13 @@ std::string RequestHandlerAuthorize::GetRequestString(const Request* request, co
return request_string;
}
-Error RequestHandlerAuthorize::ErrorFromServerResponse(const Error& err, HttpCode code) const {
+Error RequestHandlerAuthorize::ErrorFromAuthorizationServerResponse(const Error& err, HttpCode code) const {
if (err) {
return asapo::ReceiverErrorTemplates::kInternalServerError.Generate("cannot authorize request: " + err->Explain());
} else {
+ if (code != HttpCode::Unauthorized) {
+ return asapo::ReceiverErrorTemplates::kInternalServerError.Generate("return code " + std::to_string(int(code)));
+ }
return asapo::ReceiverErrorTemplates::kAuthorizationFailure.Generate("return code " + std::to_string(int(code)));
}
}
@@ -31,7 +34,7 @@ Error RequestHandlerAuthorize::Authorize(Request* request, const char* source_cr
auto response = http_client__->Post(GetReceiverConfig()->authorization_server + "/authorize", request_string, &code,
&err);
if (err || code != HttpCode::OK) {
- auto auth_error = ErrorFromServerResponse(err, code);
+ auto auth_error = ErrorFromAuthorizationServerResponse(err, code);
log__->Error("failure authorizing at " + GetReceiverConfig()->authorization_server + " request: " + request_string +
" - " +
auth_error->Explain());
@@ -39,13 +42,13 @@ Error RequestHandlerAuthorize::Authorize(Request* request, const char* source_cr
}
JsonStringParser parser{response};
- (err = parser.GetString("BeamtimeId", &beamtime_id_)) ||
- (err = parser.GetString("Stream", &stream_)) ||
- (err = parser.GetString("Facility", &facility_)) ||
- (err = parser.GetString("Year", &beamtime_year_)) ||
- (err = parser.GetString("Beamline", &beamline_));
+ (err = parser.GetString("beamtimeId", &beamtime_id_)) ||
+ (err = parser.GetString("stream", &stream_)) ||
+ (err = parser.GetString("core-path", &offline_path_)) ||
+ (err = parser.GetString("beamline-path", &online_path_)) ||
+ (err = parser.GetString("beamline", &beamline_));
if (err) {
- return ErrorFromServerResponse(err, code);
+ return ErrorFromAuthorizationServerResponse(err, code);
} else {
log__->Debug(std::string("authorized connection from ") + request->GetOriginUri() + " beamline: " +
beamline_ + ", beamtime id: " + beamtime_id_ + ", stream: " + stream_);
@@ -69,15 +72,30 @@ Error RequestHandlerAuthorize::ProcessAuthorizationRequest(Request* request) con
return Authorize(request, request->GetMessage());
}
+Error RequestHandlerAuthorize::ProcessReAuthorization(Request* request) const {
+ std::string old_beamtimeId = beamtime_id_;
+ auto err = Authorize(request, cached_source_credentials_.c_str());
+ if (err == asapo::ReceiverErrorTemplates::kAuthorizationFailure || (
+ err == nullptr && old_beamtimeId != beamtime_id_)) {
+ return asapo::ReceiverErrorTemplates::kReAuthorizationFailure.Generate();
+ }
+ return err;
+}
+
+bool RequestHandlerAuthorize::NeedReauthorize() const {
+ uint64_t elapsed_ms = (uint64_t) std::chrono::duration_cast<std::chrono::milliseconds>
+ (system_clock::now() - last_updated_).count();
+ return elapsed_ms >= GetReceiverConfig()->authorization_interval_ms;
+}
+
+
Error RequestHandlerAuthorize::ProcessOtherRequest(Request* request) const {
if (cached_source_credentials_.empty()) {
return ReceiverErrorTemplates::kAuthorizationFailure.Generate();
}
- uint64_t elapsed_ms = (uint64_t) std::chrono::duration_cast<std::chrono::milliseconds>
- (system_clock::now() - last_updated_).count();
- if (elapsed_ms >= GetReceiverConfig()->authorization_interval_ms) {
- auto err = Authorize(request, cached_source_credentials_.c_str());
+ if (NeedReauthorize()) {
+ auto err = ProcessReAuthorization(request);
if (err) {
return err;
}
@@ -85,8 +103,8 @@ Error RequestHandlerAuthorize::ProcessOtherRequest(Request* request) const {
request->SetBeamtimeId(beamtime_id_);
request->SetBeamline(beamline_);
request->SetStream(stream_);
- request->SetBeamtimeYear(beamtime_year_);
- request->SetFacility(facility_);
+ request->SetOfflinePath(offline_path_);
+ request->SetOnlinePath(online_path_);
return nullptr;
}
@@ -107,5 +125,4 @@ StatisticEntity RequestHandlerAuthorize::GetStatisticEntity() const {
return StatisticEntity::kNetwork;
}
-
}
\ No newline at end of file
diff --git a/receiver/src/request_handler_authorize.h b/receiver/src/request_handler_authorize.h
index 95c78802675c1099b09f6ea77dfc5ec78414d821..c0bcd062b046094ab4cdb26422df9489cdda7c44 100644
--- a/receiver/src/request_handler_authorize.h
+++ b/receiver/src/request_handler_authorize.h
@@ -23,15 +23,16 @@ class RequestHandlerAuthorize final: public ReceiverRequestHandler {
mutable std::string beamtime_id_;
mutable std::string stream_;
mutable std::string beamline_;
- mutable std::string facility_;
- mutable std::string beamtime_year_;
+ mutable std::string offline_path_;
+ mutable std::string online_path_;
mutable std::string cached_source_credentials_;
mutable std::chrono::system_clock::time_point last_updated_;
Error ProcessAuthorizationRequest(Request* request) const;
Error ProcessOtherRequest(Request* request) const;
Error Authorize(Request* request, const char* source_credentials) const;
- Error ErrorFromServerResponse(const Error& err, HttpCode code) const;
-
+ Error ErrorFromAuthorizationServerResponse(const Error& err, HttpCode code) const;
+ Error ProcessReAuthorization(Request* request) const;
+ bool NeedReauthorize() const;
std::string GetRequestString(const Request* request, const char* source_credentials) const;
};
diff --git a/receiver/src/request_handler_file_process.cpp b/receiver/src/request_handler_file_process.cpp
index fde5496a2e7d753b839586cb9167e0765b3ca120..e12e42b3df844a3ababe0e8de457e646463e5fc9 100644
--- a/receiver/src/request_handler_file_process.cpp
+++ b/receiver/src/request_handler_file_process.cpp
@@ -21,7 +21,7 @@ Error RequestHandlerFileProcess::ProcessFileExistSituation(Request* request) con
auto err_duplicate = request->CheckForDuplicates();
if (err_duplicate == nullptr) {
request->SetWarningMessage("file has been overwritten");
- log__->Warning("overwriting file " + request->GetFullPath(GetReceiverConfig()->root_folder));
+ log__->Warning(std::string("overwriting file " ) + request->GetOfflinePath() + kPathSeparator + request->GetFileName());
return file_processor_->ProcessFile(request, true);
}
diff --git a/receiver/src/requests_dispatcher.cpp b/receiver/src/requests_dispatcher.cpp
index 93c1cade3aaa67958c4f674d41c24b5b27eef0da..73efb765221955ee4f774c651333e8952f8ea011 100644
--- a/receiver/src/requests_dispatcher.cpp
+++ b/receiver/src/requests_dispatcher.cpp
@@ -18,6 +18,8 @@ NetworkErrorCode GetNetworkCodeFromError(const Error& err) {
if (err) {
if (err == ReceiverErrorTemplates::kAuthorizationFailure) {
return NetworkErrorCode::kNetAuthorizationError;
+ } else if (err == ReceiverErrorTemplates::kReAuthorizationFailure) {
+ return NetworkErrorCode::kNetErrorReauthorize;
} else if (err == DBErrorTemplates::kJsonParseError || err == ReceiverErrorTemplates::kBadRequest) {
return NetworkErrorCode::kNetErrorWrongRequest;
} else {
@@ -49,7 +51,11 @@ Error RequestsDispatcher::HandleRequest(const std::unique_ptr<Request>& request)
Error handle_err;
handle_err = request->Handle(statistics__);
if (handle_err) {
- log__->Error("error processing request from " + producer_uri_ + " - " + handle_err->Explain());
+ if (handle_err == ReceiverErrorTemplates::kReAuthorizationFailure) {
+ log__->Warning("warning processing request from " + producer_uri_ + " - " + handle_err->Explain());
+ } else {
+ log__->Error("error processing request from " + producer_uri_ + " - " + handle_err->Explain());
+ }
}
return handle_err;
}
diff --git a/receiver/src/write_file_processor.cpp b/receiver/src/write_file_processor.cpp
index 0e92bfe0066c0e57d215642b9538df775c94b722..ff5822762e2d6c7d21927bb5fae6a2d7cac8b901 100644
--- a/receiver/src/write_file_processor.cpp
+++ b/receiver/src/write_file_processor.cpp
@@ -21,7 +21,7 @@ Error WriteFileProcessor::ProcessFile(const Request* request, bool overwrite) co
auto data = request->GetData();
auto fname = request->GetFileName();
- auto root_folder = request->GetFullPath(GetReceiverConfig()->root_folder);
+ auto root_folder = request->GetOfflinePath();
auto err = io__->WriteDataToFile(root_folder, fname, (uint8_t*)data, (size_t) fsize, true, overwrite);
if (!err) {
diff --git a/receiver/unittests/mock_receiver_config.cpp b/receiver/unittests/mock_receiver_config.cpp
index 8a38336d0bad4a8faa868182973ca807328d97ed..99d2873cea0722cb1f865a7cf06c05c149ffbddb 100644
--- a/receiver/unittests/mock_receiver_config.cpp
+++ b/receiver/unittests/mock_receiver_config.cpp
@@ -65,7 +65,6 @@ Error SetReceiverConfig (const ReceiverConfig& config, std::string error_field)
config_string += "," + Key("LogLevel", error_field) + "\"" + log_level + "\"";
config_string += "," + Key("AdvertiseIP", error_field) + "\"" + config.advertise_ip + "\"";
config_string += "," + Key("Tag", error_field) + "\"" + config.tag + "\"";
- config_string += "," + Key("RootFolder", error_field) + "\"" + config.root_folder + "\"";
config_string += "}";
diff --git a/receiver/unittests/receiver_data_server/test_request_handler.cpp b/receiver/unittests/receiver_data_server/test_request_handler.cpp
index 6b56beb2b9a24dd57e80461d6c1804c8de34d715..d67c63cfa8bb2f01f39ba3621b13cbf16b076f32 100644
--- a/receiver/unittests/receiver_data_server/test_request_handler.cpp
+++ b/receiver/unittests/receiver_data_server/test_request_handler.cpp
@@ -57,6 +57,7 @@ class RequestHandlerTests : public Test {
uint64_t expected_meta_size = 100;
uint64_t expected_buf_id = 12345;
uint64_t expected_source_id = 11;
+ bool retry;
asapo::GenericRequestHeader header{asapo::kOpcodeGetBufferData, expected_buf_id, expected_data_size,
expected_meta_size, ""};
asapo::ReceiverDataServerRequest request{std::move(header), expected_source_id};
@@ -97,7 +98,7 @@ TEST_F(RequestHandlerTests, ProcessRequest_WronOpCode) {
EXPECT_CALL(mock_logger, Error(HasSubstr("wrong request")));
- auto success = handler.ProcessRequestUnlocked(&request);
+ auto success = handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(true));
}
@@ -105,7 +106,7 @@ TEST_F(RequestHandlerTests, ProcessRequest_WronOpCode) {
TEST_F(RequestHandlerTests, ProcessRequestReturnsNoDataWhenCacheNotUsed) {
MockSendResponce(asapo::kNetErrorNoData, true);
- auto success = handler_no_cache.ProcessRequestUnlocked(&request);
+ auto success = handler_no_cache.ProcessRequestUnlocked(&request, &retry);
EXPECT_CALL(mock_logger, Debug(_)).Times(0);
ASSERT_THAT(success, Eq(true));
@@ -116,7 +117,7 @@ TEST_F(RequestHandlerTests, ProcessRequestReadSlotReturnsNull) {
MockSendResponce(asapo::kNetErrorNoData, true);
EXPECT_CALL(mock_logger, Debug(HasSubstr("not found")));
- auto success = handler.ProcessRequestUnlocked(&request);
+ auto success = handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(true));
}
@@ -128,7 +129,7 @@ TEST_F(RequestHandlerTests, ProcessRequestReadSlotErrorSendingResponce) {
EXPECT_CALL(mock_net, SendData_t(expected_source_id, &tmp, expected_data_size)).Times(0);
EXPECT_CALL(mock_cache, UnlockSlot(_));
- auto success = handler.ProcessRequestUnlocked(&request);
+ auto success = handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(true));
}
@@ -145,7 +146,7 @@ TEST_F(RequestHandlerTests, ProcessRequestOk) {
EXPECT_CALL(mock_logger, Debug(HasSubstr("sending")));
EXPECT_CALL(mock_stat, IncreaseRequestCounter_t());
EXPECT_CALL(mock_stat, IncreaseRequestDataVolume_t(expected_data_size));
- auto success = handler.ProcessRequestUnlocked(&request);
+ auto success = handler.ProcessRequestUnlocked(&request, &retry);
ASSERT_THAT(success, Eq(true));
}
diff --git a/receiver/unittests/receiver_mocking.h b/receiver/unittests/receiver_mocking.h
index cce334542cfaf14f523d5730e812d46fe8673dfa..c693ae066d192b933bc85f909256b6609dc76493 100644
--- a/receiver/unittests/receiver_mocking.h
+++ b/receiver/unittests/receiver_mocking.h
@@ -77,8 +77,8 @@ class MockRequest: public Request {
MOCK_CONST_METHOD0(GetOpCode, asapo::Opcode ());
MOCK_CONST_METHOD0(GetSocket, asapo::SocketDescriptor ());
- MOCK_CONST_METHOD0(GetFacility, const std::string & ());
- MOCK_CONST_METHOD0(GetBeamtimeYear, const std::string & ());
+ MOCK_CONST_METHOD0(GetOnlinePath, const std::string & ());
+ MOCK_CONST_METHOD0(GetOfflinePath, const std::string & ());
const asapo::CustomRequestData& GetCustomData() const override {
return (asapo::CustomRequestData&) * GetCustomData_t();
@@ -89,9 +89,8 @@ class MockRequest: public Request {
MOCK_METHOD1(SetBeamtimeId, void (std::string));
MOCK_METHOD1(SetStream, void (std::string));
MOCK_METHOD1(SetBeamline, void (std::string));
- MOCK_METHOD1(SetFacility, void (std::string));
- MOCK_METHOD1(SetBeamtimeYear, void (std::string));
- MOCK_CONST_METHOD1(GetFullPath, std::string (std::string));
+ MOCK_METHOD1(SetOnlinePath, void (std::string));
+ MOCK_METHOD1(SetOfflinePath, void (std::string));
MOCK_CONST_METHOD0(WasAlreadyProcessed, bool());
MOCK_METHOD0(SetAlreadyProcessedFlag, void());
diff --git a/receiver/unittests/test_config.cpp b/receiver/unittests/test_config.cpp
index 4147e0349df8321963b4ff0f628b43311748e94c..538f6f9dc03ac6791a26df9ae239c62d7c5a6010 100644
--- a/receiver/unittests/test_config.cpp
+++ b/receiver/unittests/test_config.cpp
@@ -53,7 +53,6 @@ class ConfigTests : public Test {
test_config.write_to_db = true;
test_config.database_uri = "localhost:27017";
test_config.log_level = asapo::LogLevel::Error;
- test_config.root_folder = "test_fodler";
test_config.authorization_interval_ms = 10000;
test_config.authorization_server = "AuthorizationServer/aa";
test_config.use_datacache = false;
@@ -89,7 +88,6 @@ TEST_F(ConfigTests, ReadSettings) {
ASSERT_THAT(config->write_to_db, Eq(true));
ASSERT_THAT(config->log_level, Eq(asapo::LogLevel::Error));
ASSERT_THAT(config->tag, Eq("receiver1"));
- ASSERT_THAT(config->root_folder, Eq("test_fodler"));
ASSERT_THAT(config->use_datacache, Eq(false));
ASSERT_THAT(config->datacache_reserved_share, Eq(10));
ASSERT_THAT(config->datacache_size_gb, Eq(2));
@@ -106,7 +104,7 @@ TEST_F(ConfigTests, ErrorReadSettings) {
std::vector<std::string>fields {"PerformanceDbServer", "ListenPort", "DataServer", "ListenPort", "WriteToDisk",
"WriteToDb", "DataCache", "Use", "SizeGB", "ReservedShare", "DatabaseServer", "Tag",
- "AuthorizationServer", "AuthorizationInterval", "RootFolder", "PerformanceDbName", "LogLevel",
+ "AuthorizationServer", "AuthorizationInterval", "PerformanceDbName", "LogLevel",
"NThreads", "DiscoveryServer", "AdvertiseIP", "ReceiveToDiskThresholdMB"};
for (const auto& field : fields) {
auto err = asapo::SetReceiverConfig(test_config, field);
diff --git a/receiver/unittests/test_receive_file_processor.cpp b/receiver/unittests/test_receive_file_processor.cpp
index 6a729431b4db5e67e58366b40169dbad54cb1645..746733107fb6ab0355dd32cd7ce92801ac618ef2 100644
--- a/receiver/unittests/test_receive_file_processor.cpp
+++ b/receiver/unittests/test_receive_file_processor.cpp
@@ -74,7 +74,6 @@ class ReceiveFileProcessorTests : public Test {
GenericRequestHeader request_header;
request_header.data_id = 2;
asapo::ReceiverConfig test_config;
- test_config.root_folder = expected_root_folder;
asapo::SetReceiverConfig(test_config, "none");
processor.log__ = &mock_logger;
mock_request.reset(new MockRequest{request_header, 1, "", nullptr});
@@ -95,8 +94,8 @@ void ReceiveFileProcessorTests::MockRequestData() {
EXPECT_CALL(*mock_request, GetDataSize()).Times(1)
.WillRepeatedly(Return(expected_file_size));
- EXPECT_CALL(*mock_request, GetFullPath(expected_root_folder)).Times(1)
- .WillRepeatedly(Return(expected_full_path));
+ EXPECT_CALL(*mock_request, GetOfflinePath()).Times(1)
+ .WillRepeatedly(ReturnRef(expected_full_path));
EXPECT_CALL(*mock_request, GetFileName()).Times(1)
.WillRepeatedly(Return(expected_file_name));
@@ -111,7 +110,6 @@ void ReceiveFileProcessorTests::ExpectFileWrite(const asapo::SimpleErrorTemplate
TEST_F(ReceiveFileProcessorTests, CallsReceiveFile) {
asapo::ReceiverConfig test_config;
- test_config.root_folder = expected_root_folder;
asapo::SetReceiverConfig(test_config, "none");
diff --git a/receiver/unittests/test_request.cpp b/receiver/unittests/test_request.cpp
index 2a94be104d17590b7e95a522e46e710c219303fa..5d1650018b972df11f170dfb15dea10a51223517 100644
--- a/receiver/unittests/test_request.cpp
+++ b/receiver/unittests/test_request.cpp
@@ -247,28 +247,13 @@ TEST_F(RequestTests, SetGetMetadata) {
TEST_F(RequestTests, SetGetFacility) {
- request->SetFacility("p00");
- ASSERT_THAT(request->GetFacility(), "p00");
+ request->SetOnlinePath("p00");
+ ASSERT_THAT(request->GetOnlinePath(), "p00");
}
TEST_F(RequestTests, RequestTests_SetGetBeamtimeYear_Test) {
- request->SetBeamtimeYear("2020");
- ASSERT_THAT(request->GetBeamtimeYear(), "2020");
-}
-
-TEST_F(RequestTests, RequestTests_GetFullPath) {
- request->SetFacility("test_facility");
- request->SetBeamtimeYear("2020");
- request->SetBeamtimeId("beamtime");
- request->SetBeamline("beamline");
-
- std::string expected_path = std::string("test_folder") + asapo::kPathSeparator + "test_facility" +
- asapo::kPathSeparator + "gpfs" +
- asapo::kPathSeparator + "beamline" +
- asapo::kPathSeparator + "2020" +
- asapo::kPathSeparator + "data" +
- asapo::kPathSeparator + "beamtime";
- ASSERT_THAT(request->GetFullPath("test_folder"), expected_path);
+ request->SetOfflinePath("2020");
+ ASSERT_THAT(request->GetOfflinePath(), "2020");
}
TEST_F(RequestTests, SetGetWarningMessage) {
diff --git a/receiver/unittests/test_request_handler_authorizer.cpp b/receiver/unittests/test_request_handler_authorizer.cpp
index b6e7ff1b3f7b40df5fea2e54b1472978665e472a..5f3caa0be9192165520252a6eaf33a6c3a0c06e8 100644
--- a/receiver/unittests/test_request_handler_authorizer.cpp
+++ b/receiver/unittests/test_request_handler_authorizer.cpp
@@ -63,21 +63,23 @@ class AuthorizerHandlerTests : public Test {
ReceiverConfig config;
NiceMock<asapo::MockLogger> mock_logger;
- std::string expected_source_credentials = "beamtime_id%stream%token";
std::string expected_beamtime_id = "beamtime_id";
std::string expected_stream = "stream";
std::string expected_beamline = "beamline";
- std::string expected_facility = "facility";
- std::string expected_year = "2020";
+ std::string expected_beamline_path = "/beamline/p01/current";
+ std::string expected_core_path = "/gpfs/blabla";
std::string expected_producer_uri = "producer_uri";
std::string expected_authorization_server = "authorizer_host";
- std::string expect_request_string = std::string("{\"SourceCredentials\":\"") + expected_source_credentials +
- "\",\"OriginHost\":\"" +
- expected_producer_uri + "\"}";
-
+ std::string expect_request_string;
+ std::string expected_source_credentials;
void MockRequestData();
void SetUp() override {
GenericRequestHeader request_header;
+ expected_source_credentials = expected_beamtime_id + "%stream%token";
+ expect_request_string = std::string("{\"SourceCredentials\":\"") + expected_source_credentials +
+ "\",\"OriginHost\":\"" +
+ expected_producer_uri + "\"}";
+
mock_request.reset(new MockRequest{request_header, 1, expected_producer_uri, nullptr});
handler.http_client__ = std::unique_ptr<asapo::HttpClient> {&mock_http_client};
handler.log__ = &mock_logger;
@@ -106,11 +108,11 @@ class AuthorizerHandlerTests : public Test {
WillOnce(
DoAll(SetArgPointee<3>(nullptr),
SetArgPointee<2>(code),
- Return("{\"BeamtimeId\":\"" + expected_beamtime_id +
- "\",\"Stream\":" + "\"" + expected_stream +
- "\",\"Facility\":" + "\"" + expected_facility +
- "\",\"Year\":" + "\"" + expected_year +
- "\",\"Beamline\":" + "\"" + expected_beamline + "\"}")
+ Return("{\"beamtimeId\":\"" + expected_beamtime_id +
+ "\",\"stream\":" + "\"" + expected_stream +
+ "\",\"beamline-path\":" + "\"" + expected_beamline_path +
+ "\",\"core-path\":" + "\"" + expected_core_path +
+ "\",\"beamline\":" + "\"" + expected_beamline + "\"}")
));
if (code != HttpCode::OK) {
EXPECT_CALL(mock_logger, Error(AllOf(HasSubstr("failure authorizing"),
@@ -142,15 +144,16 @@ class AuthorizerHandlerTests : public Test {
MockAuthRequest(error, code);
return handler.ProcessRequest(mock_request.get());
}
- Error MockRequestAuthorization(bool error, HttpCode code = HttpCode::OK) {
+ Error MockRequestAuthorization(bool error, HttpCode code = HttpCode::OK, bool set_request = true) {
EXPECT_CALL(*mock_request, GetOpCode())
.WillOnce(Return(asapo::kOpcodeTransferData))
;
- if (!error && code == HttpCode::OK) {
+
+ if (!error && code == HttpCode::OK && set_request) {
EXPECT_CALL(*mock_request, SetBeamtimeId(expected_beamtime_id));
EXPECT_CALL(*mock_request, SetStream(expected_stream));
- EXPECT_CALL(*mock_request, SetFacility(expected_facility));
- EXPECT_CALL(*mock_request, SetBeamtimeYear(expected_year));
+ EXPECT_CALL(*mock_request, SetOfflinePath(expected_core_path));
+ EXPECT_CALL(*mock_request, SetOnlinePath(expected_beamline_path));
EXPECT_CALL(*mock_request, SetBeamline(expected_beamline));
}
@@ -214,6 +217,7 @@ TEST_F(AuthorizerHandlerTests, ErrorOnSecondAuthorize) {
TEST_F(AuthorizerHandlerTests, ErrorOnDataTransferRequestAuthorize) {
MockFirstAuthorization(false);
+
auto err = MockRequestAuthorization(true);
ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kInternalServerError));
@@ -225,17 +229,27 @@ TEST_F(AuthorizerHandlerTests, DataTransferRequestAuthorizeReturns401) {
auto err = MockRequestAuthorization(false, HttpCode::Unauthorized);
- ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kAuthorizationFailure));
+ ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kReAuthorizationFailure));
}
-TEST_F(AuthorizerHandlerTests, DataTransferRequestAuthorizeReturnsOK) {
+TEST_F(AuthorizerHandlerTests, DataTransferRequestAuthorizeReturnsSameBeamtimeId) {
MockFirstAuthorization(false);
-
auto err = MockRequestAuthorization(false);
ASSERT_THAT(err, Eq(nullptr));
}
+TEST_F(AuthorizerHandlerTests, RequestAuthorizeReturnsDifferentBeamtimeId) {
+ MockFirstAuthorization(false);
+
+ expected_beamtime_id = "different_id";
+ auto err = MockRequestAuthorization(false, HttpCode::OK, false);
+
+ ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kReAuthorizationFailure));
+}
+
+
+
TEST_F(AuthorizerHandlerTests, DataTransferRequestAuthorizeUsesCachedValue) {
config.authorization_interval_ms = 10000;
SetReceiverConfig(config, "none");
@@ -246,8 +260,8 @@ TEST_F(AuthorizerHandlerTests, DataTransferRequestAuthorizeUsesCachedValue) {
EXPECT_CALL(*mock_request, SetBeamtimeId(expected_beamtime_id));
EXPECT_CALL(*mock_request, SetBeamline(expected_beamline));
EXPECT_CALL(*mock_request, SetStream(expected_stream));
- EXPECT_CALL(*mock_request, SetFacility(expected_facility));
- EXPECT_CALL(*mock_request, SetBeamtimeYear(expected_year));
+ EXPECT_CALL(*mock_request, SetOnlinePath(expected_beamline_path));
+ EXPECT_CALL(*mock_request, SetOfflinePath(expected_core_path));
auto err = handler.ProcessRequest(mock_request.get());
diff --git a/receiver/unittests/test_request_handler_file_process.cpp b/receiver/unittests/test_request_handler_file_process.cpp
index 0978d571e371ff62573c82e314e610a89097de0d..121c0bc764d64a2b262997c1811c120731523353 100644
--- a/receiver/unittests/test_request_handler_file_process.cpp
+++ b/receiver/unittests/test_request_handler_file_process.cpp
@@ -86,6 +86,13 @@ TEST_F(FileWriteHandlerTests, FileAlreadyExists_NoRecordInDb) {
.WillOnce(
Return(nullptr)
);
+ std::string ref_str;
+ EXPECT_CALL(*mock_request, GetOfflinePath()).WillOnce
+ (ReturnRef(ref_str));
+
+ EXPECT_CALL(*mock_request, GetFileName()).WillOnce
+ (Return(""));
+
EXPECT_CALL(mock_logger, Warning(HasSubstr("overwriting")));
diff --git a/receiver/unittests/test_requests_dispatcher.cpp b/receiver/unittests/test_requests_dispatcher.cpp
index 12bcc973aae1b599c1b9ca3a0887e0ef267d50b7..5b887b046e264f1a549221d18cae26a04414bb8e 100644
--- a/receiver/unittests/test_requests_dispatcher.cpp
+++ b/receiver/unittests/test_requests_dispatcher.cpp
@@ -150,17 +150,17 @@ class RequestsDispatcherTests : public Test {
}
- void MockHandleRequest(bool error, Error err = asapo::IOErrorTemplates::kUnknownIOError.Generate() ) {
+ void MockHandleRequest(int error_mode, Error err = asapo::IOErrorTemplates::kUnknownIOError.Generate() ) {
EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("processing request"), HasSubstr(connected_uri))));
EXPECT_CALL(mock_request, Handle_t()).WillOnce(
- Return(error ? err.release() : nullptr)
+ Return(error_mode > 0 ? err.release() : nullptr)
);
- if (error) {
+ if (error_mode == 1) {
EXPECT_CALL(mock_logger, Error(AllOf(HasSubstr("error processing request from"), HasSubstr(connected_uri))));
+ } else if (error_mode == 2) {
+ EXPECT_CALL(mock_logger, Warning(AllOf(HasSubstr("warning processing request from"), HasSubstr(connected_uri))));
}
-
-
}
void MockSendResponse(GenericNetworkResponse* response, bool error ) {
EXPECT_CALL(mock_logger, Debug(AllOf(HasSubstr("sending response to"), HasSubstr(connected_uri))));
@@ -230,7 +230,7 @@ TEST_F(RequestsDispatcherTests, OkCreatetNextRequest) {
TEST_F(RequestsDispatcherTests, ErrorProcessRequestErrorSend) {
- MockHandleRequest(true);
+ MockHandleRequest(1);
MockSendResponse(&response, true);
auto err = dispatcher->ProcessRequest(request);
@@ -240,7 +240,7 @@ TEST_F(RequestsDispatcherTests, ErrorProcessRequestErrorSend) {
TEST_F(RequestsDispatcherTests, OkProcessRequestErrorSend) {
- MockHandleRequest(false);
+ MockHandleRequest(0);
MockSendResponse(&response, true);
auto err = dispatcher->ProcessRequest(request);
@@ -250,7 +250,7 @@ TEST_F(RequestsDispatcherTests, OkProcessRequestErrorSend) {
TEST_F(RequestsDispatcherTests, OkProcessRequestSendOK) {
- MockHandleRequest(false);
+ MockHandleRequest(0);
MockSendResponse(&response, false);
auto err = dispatcher->ProcessRequest(request);
@@ -260,7 +260,7 @@ TEST_F(RequestsDispatcherTests, OkProcessRequestSendOK) {
TEST_F(RequestsDispatcherTests, ProcessRequestReturnsOkWithWarning) {
- MockHandleRequest(false);
+ MockHandleRequest(0);
MockSendResponse(&response, false);
request->SetWarningMessage("duplicate");
@@ -272,7 +272,7 @@ TEST_F(RequestsDispatcherTests, ProcessRequestReturnsOkWithWarning) {
}
TEST_F(RequestsDispatcherTests, ProcessRequestReturnsAuthorizationFailure) {
- MockHandleRequest(true, asapo::ReceiverErrorTemplates::kAuthorizationFailure.Generate());
+ MockHandleRequest(1, asapo::ReceiverErrorTemplates::kAuthorizationFailure.Generate());
MockSendResponse(&response, false);
auto err = dispatcher->ProcessRequest(request);
@@ -282,9 +282,20 @@ TEST_F(RequestsDispatcherTests, ProcessRequestReturnsAuthorizationFailure) {
ASSERT_THAT(std::string(response.message), HasSubstr("authorization"));
}
+TEST_F(RequestsDispatcherTests, ProcessRequestReturnsReAuthorizationFailure) {
+ MockHandleRequest(2, asapo::ReceiverErrorTemplates::kReAuthorizationFailure.Generate());
+ MockSendResponse(&response, false);
+
+ auto err = dispatcher->ProcessRequest(request);
+
+ ASSERT_THAT(err, Eq(asapo::ReceiverErrorTemplates::kReAuthorizationFailure));
+ ASSERT_THAT(response.error_code, Eq(asapo::kNetErrorReauthorize));
+ ASSERT_THAT(std::string(response.message), HasSubstr("reauthorization"));
+}
+
TEST_F(RequestsDispatcherTests, ProcessRequestReturnsMetaDataFailure) {
- MockHandleRequest(true, asapo::DBErrorTemplates::kJsonParseError.Generate());
+ MockHandleRequest(1, asapo::DBErrorTemplates::kJsonParseError.Generate());
MockSendResponse(&response, false);
auto err = dispatcher->ProcessRequest(request);
@@ -295,7 +306,7 @@ TEST_F(RequestsDispatcherTests, ProcessRequestReturnsMetaDataFailure) {
}
TEST_F(RequestsDispatcherTests, ProcessRequestReturnsBadRequest) {
- MockHandleRequest(true, asapo::ReceiverErrorTemplates::kBadRequest.Generate());
+ MockHandleRequest(1, asapo::ReceiverErrorTemplates::kBadRequest.Generate());
MockSendResponse(&response, false);
auto err = dispatcher->ProcessRequest(request);
diff --git a/receiver/unittests/test_write_file_processor.cpp b/receiver/unittests/test_write_file_processor.cpp
index 592de637d5ff1fbf03d0ad0e9df16951130075e0..a6f2158d3a76556f0c5d4279462bdc75534c7135 100644
--- a/receiver/unittests/test_write_file_processor.cpp
+++ b/receiver/unittests/test_write_file_processor.cpp
@@ -73,7 +73,6 @@ class WriteFileProcessorTests : public Test {
GenericRequestHeader request_header;
request_header.data_id = 2;
asapo::ReceiverConfig test_config;
- test_config.root_folder = expected_root_folder;
asapo::SetReceiverConfig(test_config, "none");
processor.log__ = &mock_logger;
mock_request.reset(new MockRequest{request_header, 1, "", nullptr});
@@ -101,8 +100,8 @@ void WriteFileProcessorTests::MockRequestData(int times) {
EXPECT_CALL(*mock_request, GetData()).Times(times)
.WillRepeatedly(Return(nullptr));
- EXPECT_CALL(*mock_request, GetFullPath(expected_root_folder)).Times(times)
- .WillRepeatedly(Return(expected_full_path));
+ EXPECT_CALL(*mock_request, GetOfflinePath()).Times(times)
+ .WillRepeatedly(ReturnRef(expected_full_path));
EXPECT_CALL(*mock_request, GetFileName()).Times(times)
.WillRepeatedly(Return(expected_file_name));
diff --git a/tests/automatic/authorizer/check_authorize/CMakeLists.txt b/tests/automatic/authorizer/check_authorize/CMakeLists.txt
index 5888d99f9c5ca199e450695e5724e272de54b9c5..ef88646421bc885f00e27bb0069094613c3cfc71 100644
--- a/tests/automatic/authorizer/check_authorize/CMakeLists.txt
+++ b/tests/automatic/authorizer/check_authorize/CMakeLists.txt
@@ -4,14 +4,21 @@ set(TARGET_NAME asapo-authorizer)
# Testing
################################
file(TO_NATIVE_PATH ${CMAKE_CURRENT_BINARY_DIR}/asap3 ASAP3_FOLDER )
+file(TO_NATIVE_PATH ${CMAKE_CURRENT_BINARY_DIR}/beamline CURRENT_BEAMLINES_FOLDER )
+
file(TO_NATIVE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/ip_bl_mapping BEAMLINES_FOLDER )
if (WIN32)
string(REPLACE "\\" "\\\\" ASAP3_FOLDER "${ASAP3_FOLDER}")
string(REPLACE "\\" "\\\\" BEAMLINES_FOLDER "${BEAMLINES_FOLDER}")
+ string(REPLACE "\\" "\\\\" CURRENT_BEAMLINES_FOLDER "${CURRENT_BEAMLINES_FOLDER}")
+
endif()
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/auth_secret.key auth_secret.key COPYONLY)
+configure_file(beamtime-metadata-11111111.json beamtime-metadata-11111111.json COPYONLY)
+
+
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/settings.json.in settings.json @ONLY)
add_script_test("${TARGET_NAME}-authorize" "$<TARGET_PROPERTY:${TARGET_NAME},EXENAME>" nomem
)
diff --git a/tests/automatic/authorizer/check_authorize/beamtime-metadata-11111111.json b/tests/automatic/authorizer/check_authorize/beamtime-metadata-11111111.json
new file mode 100644
index 0000000000000000000000000000000000000000..9270dbbefe4a12e80989c7c2faa4cc6443aee784
--- /dev/null
+++ b/tests/automatic/authorizer/check_authorize/beamtime-metadata-11111111.json
@@ -0,0 +1,36 @@
+{
+ "applicant": {
+ "email": "test",
+ "institute": "test",
+ "lastname": "test",
+ "userId": "1234",
+ "username": "test"
+ },
+ "beamline": "p07",
+ "beamline_alias": "P07",
+ "beamtimeId": "11111111",
+ "contact": "None",
+ "core-path": "/asap3/petra3/gpfs/p07/2020/data/11111111",
+ "event-end": "2020-03-03 09:00:00",
+ "event-start": "2020-03-02 09:00:00",
+ "facility": "PETRA III",
+ "generated": "2020-02-22 22:37:16",
+ "pi": {
+ "email": "test",
+ "institute": "test",
+ "lastname": "test",
+ "userId": "14",
+ "username": "test"
+ },
+ "proposalId": "12345678",
+ "proposalType": "H",
+ "title": "In-House Research (P07)",
+ "unixId": "None",
+ "users": {
+ "door-db": [
+ "test"
+ ],
+ "special": [],
+ "unknown": []
+ }
+}
diff --git a/tests/automatic/authorizer/check_authorize/check_linux.sh b/tests/automatic/authorizer/check_authorize/check_linux.sh
index ef9a2ed34e9c4b95e599f1ab7cdb0b12a863da53..ceecdffca5e6530fe0b2241452fe8398da5b410d 100644
--- a/tests/automatic/authorizer/check_authorize/check_linux.sh
+++ b/tests/automatic/authorizer/check_authorize/check_linux.sh
@@ -16,20 +16,30 @@ authorizeid=`echo $!`
mkdir -p asap3/petra3/gpfs/p01/2019/comissioning/c20180508-000-COM20181
mkdir -p asap3/petra3/gpfs/p01/2019/data/11000015
-curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%stream%","OriginHost":"127.0.0.1:5555"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep c20180508-000-COM20181
-curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%stream%","OriginHost":"127.0.0.1:5555"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep p01
-curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%stream%","OriginHost":"127.0.0.1:5555"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep stream
+mkdir -p beamline/p07/current
+cp beamtime-metadata* beamline/p07/current/
+
+curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%%stream%","OriginHost":"127.0.0.1:5555"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep c20180508-000-COM20181
+curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%%stream%","OriginHost":"127.0.0.1:5555"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep p01
+curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%%stream%","OriginHost":"127.0.0.1:5555"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep stream
token=onm80KQF8s6d2p_laW0S5IYanUUsLcnB3QO-6QQ1M90= #token for c20180508-000-COM20181
-curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%stream%onm80KQF8s6d2p_laW0S5IYanUUsLcnB3QO-6QQ1M90=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep stream
-curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%stream%onm80KQF8s6d2p_laW0S5IYanUUsLcnB3QO-6QQ1M90=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep p01
-curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%stream%bla","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep 401
+curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%%stream%onm80KQF8s6d2p_laW0S5IYanUUsLcnB3QO-6QQ1M90=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep stream
+curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%auto%stream%onm80KQF8s6d2p_laW0S5IYanUUsLcnB3QO-6QQ1M90=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep p01
+curl -v --silent --data '{"SourceCredentials":"c20180508-000-COM20181%%stream%bla","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep 401
token=dccMd3NT89i32Whz7yD4VQhmEJy6Kxc35wsBbWJLXp0= #token for 11000015
-curl -v --silent --data '{"SourceCredentials":"11000015%stream%dccMd3NT89i32Whz7yD4VQhmEJy6Kxc35wsBbWJLXp0=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep p01
+curl -v --silent --data '{"SourceCredentials":"11000015%%stream%dccMd3NT89i32Whz7yD4VQhmEJy6Kxc35wsBbWJLXp0=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep p01
token=Jaas_xTpkB0Zy5dFwjs4kCrY7yXMfbnW8Ca1aYhyKBs= #token for 11000016
-curl -v --silent --data '{"SourceCredentials":"11000016%stream%Jaas_xTpkB0Zy5dFwjs4kCrY7yXMfbnW8Ca1aYhyKBs=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep 401
+curl -v --silent --data '{"SourceCredentials":"11000016%%stream%Jaas_xTpkB0Zy5dFwjs4kCrY7yXMfbnW8Ca1aYhyKBs=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep 401
+
+
+token=-pZmisCNjAbjT2gFBKs3OB2kNOU79SNsfHud0bV8gS4= # for bl_p07
+curl -v --silent --data '{"SourceCredentials":"auto%p07%stream%-pZmisCNjAbjT2gFBKs3OB2kNOU79SNsfHud0bV8gS4=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep 11111111
+curl -v --silent --data '{"SourceCredentials":"auto%p07%stream%-pZmisCNjAbjT2gFBKs3OB2kNOU79SNsfHud0bV8gS4=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep p07
+curl -v --silent --data '{"SourceCredentials":"auto%p07%stream%-pZmisCNjAbjT2gFBKs3OB2kNOU79SNsfHud0bV8gS4=","OriginHost":"bla"}' 127.0.0.1:5007/authorize --stderr - | tee /dev/stderr | grep /asap3/petra3/gpfs/p07/2020/data/11111111
+
-rm -rf asap3
\ No newline at end of file
+rm -rf asap3 beamline
\ No newline at end of file
diff --git a/tests/automatic/authorizer/check_authorize/check_windows.bat b/tests/automatic/authorizer/check_authorize/check_windows.bat
index 5a69f49501c4aa9e6bceae1a7528df800112da50..3936894d10fa6c4b9e6359cf83edee0ee539e335 100644
--- a/tests/automatic/authorizer/check_authorize/check_windows.bat
+++ b/tests/automatic/authorizer/check_authorize/check_windows.bat
@@ -6,14 +6,17 @@ start /B "" "%full_name%" -config settings.json
ping 1.0.0.0 -n 1 -w 100 > nul
mkdir asap3\petra3\gpfs\p01\2019\comissioning\c20180508-000-COM20181
+mkdir beamline\p07\current
+copy beamtime-metadata* beamline\p07\current\ /y
-C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"c20180508-000-COM20181%%stream%%\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr c20180508-000-COM20181 || goto :error
-C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"c20180508-000-COM20181%%stream%%\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr p01 || goto :error
-C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"c20180508-000-COM20181%%stream%%\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr stream || goto :error
+C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"c20180508-000-COM20181%%%%stream%%\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr c20180508-000-COM20181 || goto :error
+C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"c20180508-000-COM20181%%auto%%stream%%\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr p01 || goto :error
+C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"c20180508-000-COM20181%%%%stream%%\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr stream || goto :error
+C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"c20180508-000-COM20181%%%%stream%%onm80KQF8s6d2p_laW0S5IYanUUsLcnB3QO-6QQ1M90=\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr stream || goto :error
+C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"c20180508-000-COM20181%%%%stream%%wrong\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr 401 || goto :error
-C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"c20180508-000-COM20181%%stream%%onm80KQF8s6d2p_laW0S5IYanUUsLcnB3QO-6QQ1M90=\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr stream || goto :error
-C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"c20180508-000-COM20181%%stream%%wrong\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr 401 || goto :error
+C:\Curl\curl.exe -v --silent --data "{\"SourceCredentials\":\"auto%%p07%%stream%%-pZmisCNjAbjT2gFBKs3OB2kNOU79SNsfHud0bV8gS4=\",\"OriginHost\":\"127.0.0.1:5555\"}" 127.0.0.1:5007/authorize --stderr - | findstr 11111111 || goto :error
goto :clean
@@ -24,3 +27,4 @@ exit /b 1
:clean
Taskkill /IM "%short_name%" /F
rmdir /S /Q asap3
+rmdir /S /Q beamline
diff --git a/tests/automatic/authorizer/check_authorize/settings.json.in b/tests/automatic/authorizer/check_authorize/settings.json.in
index 57c915d54f75e2251938e94e357dd781753221df..c344b09099def4170c924dab9864180d8a7f72eb 100644
--- a/tests/automatic/authorizer/check_authorize/settings.json.in
+++ b/tests/automatic/authorizer/check_authorize/settings.json.in
@@ -2,6 +2,7 @@
"Port": 5007,
"LogLevel":"debug",
"RootBeamtimesFolder":"@ASAP3_FOLDER@",
+ "CurrentBeamlinesFolder":"@CURRENT_BEAMLINES_FOLDER@",
"IpBeamlineMappingFolder":"@BEAMLINES_FOLDER@",
"SecretFile":"auth_secret.key"
}
diff --git a/tests/automatic/broker/read_config/settings_bad.json b/tests/automatic/broker/read_config/settings_bad.json
index 88157fb1b06da72f529ea9f42a808a58e91533cb..65fe983f44ac95e2c35cdab60350486eba59b387 100644
--- a/tests/automatic/broker/read_config/settings_bad.json
+++ b/tests/automatic/broker/read_config/settings_bad.json
@@ -1,4 +1,4 @@
{
"BrokerDbAddres":"127.0.0.1:27017",
- "port":12
+ "Port":12
}
\ No newline at end of file
diff --git a/tests/automatic/bug_fixes/error-sending-data-using-callback-method/bugfix_callback.py b/tests/automatic/bug_fixes/error-sending-data-using-callback-method/bugfix_callback.py
index 8f8864647e2c59cc86b4b7a24b7d25b25a30ad29..0f048aeed2d6f2236d10dbb2197490b5cb038cfc 100644
--- a/tests/automatic/bug_fixes/error-sending-data-using-callback-method/bugfix_callback.py
+++ b/tests/automatic/bug_fixes/error-sending-data-using-callback-method/bugfix_callback.py
@@ -25,7 +25,7 @@ class AsapoSender:
def _callback(self, header, err):
print ("hello self callback")
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads, 600)
+producer = asapo_producer.create_producer(endpoint,beamtime,'auto', stream, token, nthreads, 600)
producer.set_log_level("debug")
sender = AsapoSender(producer)
diff --git a/tests/automatic/consumer/consumer_api/consumer_api.cpp b/tests/automatic/consumer/consumer_api/consumer_api.cpp
index 9a51b41a1a331689f4336e65a64fb2e670308858..2a85a1d351632ccba6ebf6ae131c2b589c172fb6 100644
--- a/tests/automatic/consumer/consumer_api/consumer_api.cpp
+++ b/tests/automatic/consumer/consumer_api/consumer_api.cpp
@@ -187,7 +187,7 @@ void TestDataset(const std::unique_ptr<asapo::DataBroker>& broker, const std::st
void TestAll(const Args& args) {
asapo::Error err;
auto broker = asapo::DataBrokerFactory::CreateServerBroker(args.server, ".",
- asapo::SourceCredentials{args.run_name, "", args.token}, &err);
+ asapo::SourceCredentials{args.run_name, "", "", args.token}, &err);
broker->SetTimeout(100);
auto group_id = broker->GenerateNewGroupId(&err);
diff --git a/tests/automatic/consumer/next_multithread_broker/next_multithread_broker.cpp b/tests/automatic/consumer/next_multithread_broker/next_multithread_broker.cpp
index e623774e20f9798278c3a213baead395333107d4..3f06dda46b641e892bc57064e1d506ee7fde8a59 100644
--- a/tests/automatic/consumer/next_multithread_broker/next_multithread_broker.cpp
+++ b/tests/automatic/consumer/next_multithread_broker/next_multithread_broker.cpp
@@ -49,7 +49,7 @@ Args GetArgs(int argc, char* argv[]) {
void TestAll(const Args& args) {
asapo::Error err;
- auto broker = asapo::DataBrokerFactory::CreateServerBroker(args.server, "dummy", asapo::SourceCredentials{args.run_name, "", args.token}, &err);
+ auto broker = asapo::DataBrokerFactory::CreateServerBroker(args.server, "dummy", asapo::SourceCredentials{args.run_name, "", "", args.token}, &err);
auto group_id = broker->GenerateNewGroupId(&err);
broker->SetTimeout(10000);
std::vector<asapo::FileInfos>file_infos(args.nthreads);
diff --git a/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp b/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp
index 8fb1df7623077f4f0644fd0135bde61f85184906..49efa4afae83f9f87a6a00f93e10a697126ed803 100644
--- a/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp
+++ b/tests/automatic/curl_http_client/curl_http_client_command/curl_httpclient_command.cpp
@@ -32,7 +32,7 @@ int main(int argc, char* argv[]) {
auto args = GetArgs(argc, argv);
asapo::Error err;
- auto broker = asapo::DataBrokerFactory::CreateServerBroker(args.uri, "", asapo::SourceCredentials{"", "", ""}, &err);
+ auto broker = asapo::DataBrokerFactory::CreateServerBroker(args.uri, "", asapo::SourceCredentials{"", "", "", ""}, &err);
auto server_broker = static_cast<asapo::ServerDataBroker*>(broker.get());
asapo::HttpCode code;
diff --git a/tests/automatic/full_chain/send_recv_substreams/send_recv_substreams.cpp b/tests/automatic/full_chain/send_recv_substreams/send_recv_substreams.cpp
index d4eb41a5024faba5da17adf2480152eac1550d76..34a1623ad0c80f60e60b985f9aacfd4f53aa3895 100644
--- a/tests/automatic/full_chain/send_recv_substreams/send_recv_substreams.cpp
+++ b/tests/automatic/full_chain/send_recv_substreams/send_recv_substreams.cpp
@@ -36,7 +36,7 @@ void ProcessAfterSend(asapo::GenericRequestHeader header, asapo::Error err) {
BrokerPtr CreateBrokerAndGroup(const Args& args, Error* err) {
auto broker = asapo::DataBrokerFactory::CreateServerBroker(args.server, ".",
- asapo::SourceCredentials{args.beamtime_id, "", args.token}, err);
+ asapo::SourceCredentials{args.beamtime_id, "", "", args.token}, err);
if (*err) {
return nullptr;
}
@@ -56,7 +56,7 @@ ProducerPtr CreateProducer(const Args& args) {
asapo::Error err;
auto producer = asapo::Producer::Create(args.server, 1,
asapo::RequestHandlerType::kTcp,
- asapo::SourceCredentials{args.beamtime_id, "", args.token }, 60, &err);
+ asapo::SourceCredentials{args.beamtime_id, "", "", args.token }, 60, &err);
if(err) {
std::cerr << "Cannot start producer. ProducerError: " << err << std::endl;
exit(EXIT_FAILURE);
diff --git a/tests/automatic/full_chain/send_recv_substreams_python/send_recv_substreams.py b/tests/automatic/full_chain/send_recv_substreams_python/send_recv_substreams.py
index f7c959370bdc4f68d6f5512af9fba4c65761bb63..f52b8f4460d60254bbdbbccec174f9d0ccc84c1c 100644
--- a/tests/automatic/full_chain/send_recv_substreams_python/send_recv_substreams.py
+++ b/tests/automatic/full_chain/send_recv_substreams_python/send_recv_substreams.py
@@ -28,7 +28,7 @@ def callback(header,err):
source, beamtime, token = sys.argv[1:]
broker = asapo_consumer.create_server_broker(source,".", beamtime,"",token,timeout)
-producer = asapo_producer.create_producer(source,beamtime, "", token, 1, 600)
+producer = asapo_producer.create_producer(source,beamtime,'auto', "", token, 1, 600)
producer.set_log_level("debug")
group_id = broker.generate_group_id()
diff --git a/tests/automatic/json_parser/parse_config_file/config.json b/tests/automatic/json_parser/parse_config_file/config.json
index 06f8798f2fd07e3558e274f32fef4234e9dd46b3..a12c161f4441ba8fec93e4e94b2682ae112a6f78 100644
--- a/tests/automatic/json_parser/parse_config_file/config.json
+++ b/tests/automatic/json_parser/parse_config_file/config.json
@@ -1,7 +1,7 @@
{
"server":"some_string",
- "port": 10,
+ "Port": 10,
"//": "This is a way to make comments",
"some_array":[1,2,3],
"//": "Another comment",
diff --git a/tests/automatic/json_parser/parse_config_file/config_bad.json b/tests/automatic/json_parser/parse_config_file/config_bad.json
index b960fb0ce8985f07ccadeb5180aade059fd89cca..9ff6ce5948233380e73c2d900b35d7c0e57e2e00 100644
--- a/tests/automatic/json_parser/parse_config_file/config_bad.json
+++ b/tests/automatic/json_parser/parse_config_file/config_bad.json
@@ -1,6 +1,6 @@
{
"server":"some_string",
- "port": 10,
+ "Port": 10,
"some_array":1,
"some_string_array":["s1","s2","s3"],
"embedded_stuff":
diff --git a/tests/automatic/json_parser/parse_config_file/parse_config_file.cpp b/tests/automatic/json_parser/parse_config_file/parse_config_file.cpp
index 0733caddec79f57ad2b7f78519f5f60ac85c4953..eb4bb5903f42af50f1856f5abea9a4ec458c9cf4 100644
--- a/tests/automatic/json_parser/parse_config_file/parse_config_file.cpp
+++ b/tests/automatic/json_parser/parse_config_file/parse_config_file.cpp
@@ -44,7 +44,7 @@ Settings Parse(const std::string& fname, Error* err) {
Settings settings;
- (*err = parser.GetUInt64("port", &settings.port)) ||
+ (*err = parser.GetUInt64("Port", &settings.port)) ||
(*err = parser.GetArrayString("some_string_array", &settings.stringarray)) ||
(*err = parser.GetArrayUInt64("some_array", &settings.intarray)) ||
(*err = parser.Embedded("embedded_stuff").GetString("some_string", &settings.embedded_string)) ||
diff --git a/tests/automatic/producer/CMakeLists.txt b/tests/automatic/producer/CMakeLists.txt
index 5974fe371170505407bc7ed3cec20ff7a1929202..e96695ebbea23663ada2f8218199ac3bd06f7ef2 100644
--- a/tests/automatic/producer/CMakeLists.txt
+++ b/tests/automatic/producer/CMakeLists.txt
@@ -3,4 +3,6 @@ add_subdirectory(beamtime_metadata)
if (UNIX OR CMAKE_BUILD_TYPE STREQUAL "Release")
add_subdirectory(python_api)
+ add_subdirectory(aai)
+
endif()
diff --git a/tests/automatic/producer/aai/CMakeLists.txt b/tests/automatic/producer/aai/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2330026be2ac46e58f307196622770ea32f82403
--- /dev/null
+++ b/tests/automatic/producer/aai/CMakeLists.txt
@@ -0,0 +1,30 @@
+set(TARGET_NAME producer_python_aai)
+
+################################
+# Testing
+################################
+prepare_asapo()
+
+prepare_asapo()
+
+if (UNIX)
+ get_target_property(PYTHON_LIBS python-lib-producer BINARY_DIR)
+else()
+ get_target_property(PYTHON_LIBS asapo_producer BINARY_DIR)
+endif()
+
+file(TO_NATIVE_PATH ${CMAKE_CURRENT_BINARY_DIR}/beamline CURRENT_BEAMLINES_FOLDER )
+
+if (WIN32)
+ string(REPLACE "\\" "\\\\" CURRENT_BEAMLINES_FOLDER "${CURRENT_BEAMLINES_FOLDER}")
+endif()
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/settings.json.tpl.in authorizer.json.tpl @ONLY)
+configure_file(beamtime-metadata-11111111.json beamline/p07/current/beamtime-metadata-11111111.json COPYONLY)
+
+file(TO_NATIVE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/producer_aai.py TEST_SCRIPT )
+
+add_script_test("${TARGET_NAME}-python" "${Python_EXECUTABLE} ${PYTHON_LIBS} ${TEST_SCRIPT} " nomem)
+
+
+
diff --git a/tests/automatic/producer/aai/beamtime-metadata-11111111.json b/tests/automatic/producer/aai/beamtime-metadata-11111111.json
new file mode 100644
index 0000000000000000000000000000000000000000..65820fcb015bfc78d10f20e483d28d3ba42e3746
--- /dev/null
+++ b/tests/automatic/producer/aai/beamtime-metadata-11111111.json
@@ -0,0 +1,36 @@
+{
+ "applicant": {
+ "email": "test",
+ "institute": "test",
+ "lastname": "test",
+ "userId": "1234",
+ "username": "test"
+ },
+ "beamline": "p07",
+ "beamline_alias": "P07",
+ "beamtimeId": "11111111",
+ "contact": "None",
+ "core-path": "/tmp/asapo/receiver/files/test_facility/gpfs/p07/2019/data/11111111",
+ "event-end": "2020-03-03 09:00:00",
+ "event-start": "2020-03-02 09:00:00",
+ "facility": "PETRA III",
+ "generated": "2020-02-22 22:37:16",
+ "pi": {
+ "email": "test",
+ "institute": "test",
+ "lastname": "test",
+ "userId": "14",
+ "username": "test"
+ },
+ "proposalId": "12345678",
+ "proposalType": "H",
+ "title": "In-House Research (P07)",
+ "unixId": "None",
+ "users": {
+ "door-db": [
+ "test"
+ ],
+ "special": [],
+ "unknown": []
+ }
+}
diff --git a/tests/automatic/producer/aai/check_linux.sh b/tests/automatic/producer/aai/check_linux.sh
new file mode 100644
index 0000000000000000000000000000000000000000..553319166de9bb1bfa933bac07b0da0eb141c33e
--- /dev/null
+++ b/tests/automatic/producer/aai/check_linux.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -e
+
+trap Cleanup EXIT
+
+beamtime_id=11111111
+beamtime_id2=22222222
+stream=python
+beamline=p07
+receiver_root_folder=/tmp/asapo/receiver/files
+facility=test_facility
+year=2019
+receiver_folder=${receiver_root_folder}/${facility}/gpfs/${beamline}/${year}/data/${beamtime_id}
+receiver_folder2=${receiver_root_folder}/${facility}/gpfs/${beamline}/${year}/data/${beamtime_id2}
+token=-pZmisCNjAbjT2gFBKs3OB2kNOU79SNsfHud0bV8gS4= # for bl_p07
+
+
+Cleanup() {
+ echo cleanup
+ rm -rf ${receiver_root_folder}
+ nomad stop receiver >/dev/null
+ nomad stop discovery >/dev/null
+ nomad stop authorizer >/dev/null
+ nomad stop nginx >/dev/null
+ nomad run nginx_kill.nmd && nomad stop -yes -purge nginx_kill > /dev/null
+ echo "db.dropDatabase()" | mongo ${beamtime_id}_${stream} >/dev/null
+ echo "db.dropDatabase()" | mongo ${beamtime_id2}_${stream} >/dev/null
+}
+
+export PYTHONPATH=$2:${PYTHONPATH}
+
+nomad run authorizer.nmd >/dev/null
+nomad run nginx.nmd >/dev/null
+nomad run receiver.nmd >/dev/null
+nomad run discovery.nmd >/dev/null
+
+mkdir -p ${receiver_folder} ${receiver_folder2}
+
+sleep 1
+
+echo test > file1
+
+
+$1 $3 $beamline $token $stream "127.0.0.1:8400" > out || cat out
+cat out
+cat out | grep "successfuly sent" | wc -l | grep 3
+cat out | grep "reauthorization" | wc -l | grep 1
+cat out | grep "duplicated" | wc -l | grep 2
+
diff --git a/tests/automatic/producer/aai/check_windows.bat b/tests/automatic/producer/aai/check_windows.bat
new file mode 100644
index 0000000000000000000000000000000000000000..ea077946cbae3ecd7a9f29da0f914ca244bf5a7e
--- /dev/null
+++ b/tests/automatic/producer/aai/check_windows.bat
@@ -0,0 +1,61 @@
+SET mongo_exe="c:\Program Files\MongoDB\Server\4.2\bin\mongo.exe"
+SET beamtime_id=11111111
+SET beamtime_id2=22222222
+SET beamline=p07
+SET stream=python
+SET receiver_root_folder=c:\tmp\asapo\receiver\files
+SET receiver_folder="%receiver_root_folder%\test_facility\gpfs\%beamline%\2019\data\%beamtime_id%"
+SET receiver_folder2="%receiver_root_folder%\test_facility\gpfs\%beamline%\2019\data\%beamtime_id2%"
+SET dbname=%beamtime_id%_%stream%
+SET dbname2=%beamtime_id2%_%stream%
+SET token=-pZmisCNjAbjT2gFBKs3OB2kNOU79SNsfHud0bV8gS4=
+
+echo db.%dbname%.insert({dummy:1})" | %mongo_exe% %dbname%
+
+c:\opt\consul\nomad run receiver.nmd
+c:\opt\consul\nomad run authorizer.nmd
+c:\opt\consul\nomad run discovery.nmd
+c:\opt\consul\nomad run nginx.nmd
+
+ping 1.0.0.0 -n 5 -w 100 > nul
+
+mkdir %receiver_folder%
+mkdir %receiver_folder2%
+
+echo test > file1
+
+ping 1.0.0.0 -n 1 -w 100 > nul
+
+set PYTHONPATH=%2
+
+"%1" "%3" %beamline% %token% %stream% "127.0.0.1:8400" > out
+
+type out
+set NUM=0
+for /F %%N in ('find /C "successfuly sent" ^< "out"') do set NUM=%%N
+echo %NUM% | findstr 3 || goto error
+
+for /F %%N in ('find /C "reauthorization" ^< "out"') do set NUM=%%N
+echo %NUM% | findstr 1 || goto error
+
+for /F %%N in ('find /C "} server warning: duplicated request" ^< "out"') do set NUM=%%N
+echo %NUM% | findstr 1 || goto error
+
+goto :clean
+
+:error
+call :clean
+exit /b 1
+
+:clean
+c:\opt\consul\nomad stop receiver
+c:\opt\consul\nomad stop discovery
+c:\opt\consul\nomad stop nginx
+c:\opt\consul\nomad run nginx_kill.nmd && c:\opt\consul\nomad stop -yes -purge nginx_kill
+c:\opt\consul\nomad stop authorizer
+rmdir /S /Q %receiver_root_folder%
+rmdir /S /Q %receiver_root_folder2%
+echo db.dropDatabase() | %mongo_exe% %dbname%
+echo db.dropDatabase() | %mongo_exe% %dbname2%
+
+
diff --git a/tests/automatic/producer/aai/producer_aai.py b/tests/automatic/producer/aai/producer_aai.py
new file mode 100644
index 0000000000000000000000000000000000000000..54bc6f7758aa18a9e72de3fa91005df18037581a
--- /dev/null
+++ b/tests/automatic/producer/aai/producer_aai.py
@@ -0,0 +1,76 @@
+from __future__ import print_function
+
+import asapo_producer
+import sys
+import time
+import numpy as np
+import threading
+lock = threading.Lock()
+import json
+
+beamline = sys.argv[1]
+token = sys.argv[2]
+stream = sys.argv[3]
+endpoint = sys.argv[4]
+
+nthreads = 1
+
+def callback(header,err):
+ lock.acquire() # to print
+ if isinstance(err,asapo_producer.AsapoServerWarning):
+ print("successfuly sent, but with warning from server: ",header,err)
+ elif err is not None:
+ print("could not sent: ",header,err)
+ else:
+ print ("successfuly sent: ",header)
+ lock.release()
+
+
+producer = asapo_producer.create_producer(endpoint,'auto',beamline, stream, token, nthreads, 60)
+
+producer.set_log_level("debug")
+
+#send single file
+producer.send_file(1, local_path = "./file1", exposed_path = stream+"/"+"file1", user_meta = '{"test_key":"test_val"}', callback = callback)
+
+producer.wait_requests_finished(10000)
+
+time.sleep(2)
+
+#send single file to other beamtime - should be warning on duplicated request (same beamtime, no reauthorization)
+producer.send_file(1, local_path = "./file1", exposed_path = stream+"/"+"file1", user_meta = '{"test_key":"test_val"}', callback = callback)
+producer.wait_requests_finished(10000)
+
+
+fname = 'beamline/p07/current/beamtime-metadata-11111111.json'
+with open(fname) as json_file:
+ data = json.load(json_file)
+data['beamtimeId']='22222222'
+data['core-path']=data['core-path'].replace('11111111','22222222')
+
+with open(fname, 'w') as outfile:
+ json.dump(data, outfile)
+
+time.sleep(2)
+
+#send single file to other beamtime - now ok since receiver authorization timed out
+producer.send_file(1, local_path = "./file1", exposed_path = stream+"/"+"file1", user_meta = '{"test_key":"test_val"}', callback = callback)
+
+producer.wait_requests_finished(10000)
+
+data['beamtimeId']='11111111'
+data['core-path']=data['core-path'].replace('22222222','11111111')
+
+with open(fname, 'w') as outfile:
+ json.dump(data, outfile)
+
+
+n = producer.get_requests_queue_size()
+if n!=0:
+ print("number of remaining requests should be zero, got ",n)
+ sys.exit(1)
+
+
+
+
+
diff --git a/tests/automatic/producer/aai/settings.json.tpl.in b/tests/automatic/producer/aai/settings.json.tpl.in
new file mode 100644
index 0000000000000000000000000000000000000000..319ef7063706b37efcbe6c62bbdadb9fe7bfe8b9
--- /dev/null
+++ b/tests/automatic/producer/aai/settings.json.tpl.in
@@ -0,0 +1,8 @@
+{
+ "Port": {{ env "NOMAD_PORT_authorizer" }},
+ "LogLevel":"debug",
+ "CurrentBeamlinesFolder":"@CURRENT_BEAMLINES_FOLDER@",
+ "SecretFile":"auth_secret.key"
+}
+
+
diff --git a/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp b/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp
index 2b72594b76e8e6106a8bb993b1e7f21234f0fecc..d3f31d010d0b3f98af8cdc4575eb6a784a2b03a0 100644
--- a/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp
+++ b/tests/automatic/producer/beamtime_metadata/beamtime_metadata.cpp
@@ -69,7 +69,7 @@ std::unique_ptr<asapo::Producer> CreateProducer(const Args& args) {
auto producer = asapo::Producer::Create(args.receiver_address, 1,
args.mode == 0 ? asapo::RequestHandlerType::kTcp
: asapo::RequestHandlerType::kFilesystem,
- asapo::SourceCredentials{args.beamtime_id, "", ""}, 60, &err);
+ asapo::SourceCredentials{args.beamtime_id, "", "", ""}, 60, &err);
if (err) {
std::cerr << "Cannot start producer. ProducerError: " << err << std::endl;
exit(EXIT_FAILURE);
diff --git a/tests/automatic/producer/python_api/producer_api.py b/tests/automatic/producer/python_api/producer_api.py
index 084af55680c72cb65808c84fef9c30f1245a3f11..a0f559f6f50a69e4e9bf17ea8af062308c3e8212 100644
--- a/tests/automatic/producer/python_api/producer_api.py
+++ b/tests/automatic/producer/python_api/producer_api.py
@@ -24,7 +24,7 @@ def callback(header,err):
print ("successfuly sent: ",header)
lock.release()
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads,60)
+producer = asapo_producer.create_producer(endpoint,beamtime,'auto', stream, token, nthreads,60)
producer.set_log_level("debug")
@@ -96,7 +96,7 @@ if n!=0:
# create with error
try:
- producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, 0,0)
+ producer = asapo_producer.create_producer(endpoint,beamtime,'auto', stream, token, 0,0)
except asapo_producer.AsapoWrongInputError as e:
print(e)
else:
diff --git a/tests/automatic/settings/authorizer_settings.json.tpl b/tests/automatic/settings/authorizer_settings.json.tpl
deleted file mode 100644
index f6d57da90da2a18c01b519e0cf0ef51f3ab667dc..0000000000000000000000000000000000000000
--- a/tests/automatic/settings/authorizer_settings.json.tpl
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "Port": {{ env "NOMAD_PORT_authorizer" }},
- "LogLevel":"debug",
- "AlwaysAllowedBeamtimes":[{"BeamtimeId":"asapo_test","Beamline":"test","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test1","Beamline":"test1","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test2","Beamline":"test2","Year":"2019","Facility":"test_facility"}],
- "SecretFile":"auth_secret.key"
-}
-
-
diff --git a/tests/automatic/settings/authorizer_settings.json.tpl.lin b/tests/automatic/settings/authorizer_settings.json.tpl.lin
new file mode 100644
index 0000000000000000000000000000000000000000..07988026d1779f11f3ae53d801eafd26abe6b553
--- /dev/null
+++ b/tests/automatic/settings/authorizer_settings.json.tpl.lin
@@ -0,0 +1,8 @@
+{
+ "Port": {{ env "NOMAD_PORT_authorizer" }},
+ "LogLevel":"debug",
+ "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","core-path":"/tmp/asapo/receiver/files/test_facility/gpfs/test/2019/data/asapo_test"},
+ {"beamtimeId":"asapo_test1","beamline":"test1","core-path":"/tmp/asapo/receiver/files/test_facility/gpfs/test1/2019/data/asapo_test1"},
+ {"beamtimeId":"asapo_test2","beamline":"test2","core-path":"/tmp/asapo/receiver/files/test_facility/gpfs/test2/2019/data/asapo_test2"}],
+ "SecretFile":"auth_secret.key"
+}
diff --git a/tests/automatic/settings/authorizer_settings.json.tpl.win b/tests/automatic/settings/authorizer_settings.json.tpl.win
new file mode 100644
index 0000000000000000000000000000000000000000..418da1031bbdd84616f635be9d83b3635232e036
--- /dev/null
+++ b/tests/automatic/settings/authorizer_settings.json.tpl.win
@@ -0,0 +1,8 @@
+{
+ "Port": {{ env "NOMAD_PORT_authorizer" }},
+ "LogLevel":"debug",
+ "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","core-path":"c:\\tmp\\asapo\\receiver\\files\\test_facility\\gpfs\\test\\2019\\data\\asapo_test"},
+ {"beamtimeId":"asapo_test1","beamline":"test1","core-path":"c:\\tmp\\asapo\\receiver\\files\\test_facility\\gpfs\\test1\\2019\\data\\asapo_test1"},
+ {"beamtimeId":"asapo_test2","beamline":"test2","core-path":"c:\\tmp\\asapo\\receiver\\files\\test_facility\\gpfs\\test2\\2019\\data\\asapo_test2"}],
+ "SecretFile":"auth_secret.key"
+}
diff --git a/tests/automatic/settings/broker_settings.json b/tests/automatic/settings/broker_settings.json
index dc249838039cab80bc38414e200c1911d154b591..769f171b7924765b97faf72819aa5690e9902cb9 100644
--- a/tests/automatic/settings/broker_settings.json
+++ b/tests/automatic/settings/broker_settings.json
@@ -2,7 +2,7 @@
"DatabaseServer":"127.0.0.1:27017",
"PerformanceDbServer": "localhost:8086",
"PerformanceDbName": "db_test",
- "port":5005,
+ "Port":5005,
"LogLevel":"info",
"SecretFile":"auth_secret.key"
}
\ No newline at end of file
diff --git a/tests/automatic/settings/broker_settings.json.tpl b/tests/automatic/settings/broker_settings.json.tpl
index 0ca5866f9d51339e15ed19f6dbcd1c3a5e1c3c92..56694d8538be26718c9e32bc69a538ea20abc987 100644
--- a/tests/automatic/settings/broker_settings.json.tpl
+++ b/tests/automatic/settings/broker_settings.json.tpl
@@ -3,7 +3,7 @@
"DiscoveryServer": "localhost:8400/discovery",
"PerformanceDbServer": "localhost:8086",
"PerformanceDbName": "db_test",
- "port":{{ env "NOMAD_PORT_broker" }},
+ "Port":{{ env "NOMAD_PORT_broker" }},
"LogLevel":"info",
"SecretFile":"auth_secret.key"
}
\ No newline at end of file
diff --git a/tests/automatic/settings/receiver.json.tpl.lin.in b/tests/automatic/settings/receiver.json.tpl.lin.in
index 3ba55dd6ca3da1f05d518b4ab6fe9437eaa36931..b910579330a6849a654cafef00918a119628a8e1 100644
--- a/tests/automatic/settings/receiver.json.tpl.lin.in
+++ b/tests/automatic/settings/receiver.json.tpl.lin.in
@@ -14,12 +14,11 @@
"ReservedShare": 10
},
"AuthorizationServer": "localhost:8400/authorizer",
- "AuthorizationInterval": 10000,
+ "AuthorizationInterval": 1000,
"ListenPort": {{ env "NOMAD_PORT_recv" }},
"Tag": "{{ env "NOMAD_ADDR_recv" }}",
"WriteToDisk": @RECEIVER_WRITE_TO_DISK@,
"ReceiveToDiskThresholdMB":50,
"WriteToDb": true,
- "LogLevel" : "debug",
- "RootFolder" : "/tmp/asapo/receiver/files"
-}
+ "LogLevel" : "debug"
+ }
diff --git a/tests/automatic/settings/receiver.json.tpl.win.in b/tests/automatic/settings/receiver.json.tpl.win.in
index 6cff01f5904fd9cfbcc4880e2a0b4b0c97dc78c5..f46bd5bdf5349be12114870036853518059cbd6b 100644
--- a/tests/automatic/settings/receiver.json.tpl.win.in
+++ b/tests/automatic/settings/receiver.json.tpl.win.in
@@ -5,7 +5,7 @@
"DatabaseServer":"auto",
"DiscoveryServer": "localhost:8400/discovery",
"AuthorizationServer": "localhost:8400/authorizer",
- "AuthorizationInterval": 10000,
+ "AuthorizationInterval": 1000,
"ListenPort": {{ env "NOMAD_PORT_recv" }},
"DataServer": {
"NThreads": 2,
@@ -20,6 +20,5 @@
"WriteToDisk": @RECEIVER_WRITE_TO_DISK@,
"ReceiveToDiskThresholdMB":50,
"WriteToDb": true,
- "LogLevel" : "debug",
- "RootFolder" : "c:\\tmp\\asapo\\receiver\\files"
+ "LogLevel" : "debug"
}
diff --git a/tests/manual/broker_debug_local/authorizer.json.tpl b/tests/manual/broker_debug_local/authorizer.json.tpl
index f6d57da90da2a18c01b519e0cf0ef51f3ab667dc..9ad08b7e6111f19abd222aa13fa07abf1fb9025e 100644
--- a/tests/manual/broker_debug_local/authorizer.json.tpl
+++ b/tests/manual/broker_debug_local/authorizer.json.tpl
@@ -1,9 +1,9 @@
{
"Port": {{ env "NOMAD_PORT_authorizer" }},
"LogLevel":"debug",
- "AlwaysAllowedBeamtimes":[{"BeamtimeId":"asapo_test","Beamline":"test","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test1","Beamline":"test1","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test2","Beamline":"test2","Year":"2019","Facility":"test_facility"}],
+ "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","Year":"2019","Facility":"test_facility"},
+ {"beamtimeId":"asapo_test1","beamline":"test1","Year":"2019","Facility":"test_facility"},
+ {"beamtimeId":"asapo_test2","beamline":"test2","Year":"2019","Facility":"test_facility"}],
"SecretFile":"auth_secret.key"
}
diff --git a/tests/manual/broker_debug_local/broker.json b/tests/manual/broker_debug_local/broker.json
index 2827fe3d41de76cd52030923f3456ceb58bd09da..df0effdb32e30400a8c2b0b8168cee944d171749 100644
--- a/tests/manual/broker_debug_local/broker.json
+++ b/tests/manual/broker_debug_local/broker.json
@@ -3,7 +3,7 @@
"DiscoveryServer": "localhost:8400/discovery",
"PerformanceDbServer": "localhost:8086",
"PerformanceDbName": "db_test",
- "port": 5005,
+ "Port": 5005,
"LogLevel":"info",
"SecretFile":"auth_secret.key"
}
diff --git a/tests/manual/broker_debug_local/receiver.json b/tests/manual/broker_debug_local/receiver.json
index ace0fbecf3ab6726516b2e530ecbabe4ea6da2ad..6a5d12f59d24f354bfba8354a61341fcafbad518 100644
--- a/tests/manual/broker_debug_local/receiver.json
+++ b/tests/manual/broker_debug_local/receiver.json
@@ -20,6 +20,5 @@
"WriteToDisk": true,
"ReceiveToDiskThresholdMB":50,
"WriteToDb": true,
- "LogLevel" : "debug",
- "RootFolder" : "/tmp/asapo/receiver/files"
+ "LogLevel" : "debug"
}
diff --git a/tests/manual/broker_debug_local/receiver.json.tpl b/tests/manual/broker_debug_local/receiver.json.tpl
index 215df3b161a1b3957ddabe70c0ffb83828a6365a..1f7cbc7c1b75af4b38c23fc7fdba9f158c145b8e 100644
--- a/tests/manual/broker_debug_local/receiver.json.tpl
+++ b/tests/manual/broker_debug_local/receiver.json.tpl
@@ -20,6 +20,5 @@
"WriteToDisk": true,
"ReceiveToDiskThresholdMB":50,
"WriteToDb": true,
- "LogLevel" : "debug",
- "RootFolder" : "/tmp/asapo/receiver/files"
+ "LogLevel" : "debug"
}
diff --git a/tests/manual/performance_broker/settings.json b/tests/manual/performance_broker/settings.json
index 72e89b21fb3a4eb09f4914c63773e0df5343324c..e982871610bb8de89de2f0ef140793e51361f82e 100644
--- a/tests/manual/performance_broker/settings.json
+++ b/tests/manual/performance_broker/settings.json
@@ -2,7 +2,7 @@
"DatabaseServer":"localhost:27017",
"PerformanceDbServer": "localhost:8086",
"PerformanceDbName": "db_test",
- "port":5005,
+ "Port":5005,
"LogLevel":"info",
"SecretFile":"auth_secret.key"
}
\ No newline at end of file
diff --git a/tests/manual/performance_broker_receiver/getlast_broker.cpp b/tests/manual/performance_broker_receiver/getlast_broker.cpp
index 8aaa9d9aa122ecf7faa9ba521ed63a1229328a27..4272ae17c239cd8a7139d03835619af80761046a 100644
--- a/tests/manual/performance_broker_receiver/getlast_broker.cpp
+++ b/tests/manual/performance_broker_receiver/getlast_broker.cpp
@@ -48,7 +48,7 @@ std::vector<std::thread> StartThreads(const Args& params,
asapo::FileInfo fi;
Error err;
auto broker = asapo::DataBrokerFactory::CreateServerBroker(params.server, params.file_path,
- asapo::SourceCredentials{params.beamtime_id, "", params.token}, &err);
+ asapo::SourceCredentials{params.beamtime_id, "", "", params.token}, &err);
broker->SetTimeout((uint64_t) params.timeout_ms);
asapo::FileData data;
diff --git a/tests/manual/performance_full_chain_simple/authorizer.json b/tests/manual/performance_full_chain_simple/authorizer.json
index 30e5ca3bad8797b8f94cb03eba3e9a28339f07a4..8400681c58b4a9f2dde30e227686e4e9388b5bc8 100644
--- a/tests/manual/performance_full_chain_simple/authorizer.json
+++ b/tests/manual/performance_full_chain_simple/authorizer.json
@@ -1,7 +1,7 @@
{
"Port": 5007,
"LogLevel":"info",
- "AlwaysAllowedBeamtimes":[{"BeamtimeId":"asapo_test","Beamline":"test","Year":"2019","Facility":"test_facility"}],
+ "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","Year":"2019","Facility":"test_facility"}],
"SecretFile":"auth_secret.key"
}
diff --git a/tests/manual/performance_full_chain_simple/broker.json b/tests/manual/performance_full_chain_simple/broker.json
index 72e89b21fb3a4eb09f4914c63773e0df5343324c..e982871610bb8de89de2f0ef140793e51361f82e 100644
--- a/tests/manual/performance_full_chain_simple/broker.json
+++ b/tests/manual/performance_full_chain_simple/broker.json
@@ -2,7 +2,7 @@
"DatabaseServer":"localhost:27017",
"PerformanceDbServer": "localhost:8086",
"PerformanceDbName": "db_test",
- "port":5005,
+ "Port":5005,
"LogLevel":"info",
"SecretFile":"auth_secret.key"
}
\ No newline at end of file
diff --git a/tests/manual/performance_full_chain_simple/receiver.json b/tests/manual/performance_full_chain_simple/receiver.json
index 2d88da1f9a43f51528e0097c76f79d7ffd53bac5..476958c50d503934b42d71e49055465e4fdd85a7 100644
--- a/tests/manual/performance_full_chain_simple/receiver.json
+++ b/tests/manual/performance_full_chain_simple/receiver.json
@@ -19,6 +19,5 @@
"ReceiveToDiskThresholdMB":50,
"WriteToDb":true,
"LogLevel":"info",
- "Tag": "test_receiver",
- "RootFolder" : "/gpfs/petra3/scratch/yakubov/receiver_tests/files"
+ "Tag": "test_receiver"
}
diff --git a/tests/manual/performance_producer_receiver/authorizer.json b/tests/manual/performance_producer_receiver/authorizer.json
index 30e5ca3bad8797b8f94cb03eba3e9a28339f07a4..8400681c58b4a9f2dde30e227686e4e9388b5bc8 100644
--- a/tests/manual/performance_producer_receiver/authorizer.json
+++ b/tests/manual/performance_producer_receiver/authorizer.json
@@ -1,7 +1,7 @@
{
"Port": 5007,
"LogLevel":"info",
- "AlwaysAllowedBeamtimes":[{"BeamtimeId":"asapo_test","Beamline":"test","Year":"2019","Facility":"test_facility"}],
+ "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","Year":"2019","Facility":"test_facility"}],
"SecretFile":"auth_secret.key"
}
diff --git a/tests/manual/performance_producer_receiver/receiver.json b/tests/manual/performance_producer_receiver/receiver.json
index 9d3609563eb5f2789e8c5d0c1d83a8386feecafd..f4871c8325730d3dcfd5c0edc65090297814c7c3 100644
--- a/tests/manual/performance_producer_receiver/receiver.json
+++ b/tests/manual/performance_producer_receiver/receiver.json
@@ -19,6 +19,5 @@
"ReceiveToDiskThresholdMB":50,
"WriteToDb":true,
"LogLevel":"info",
- "Tag": "test_receiver",
- "RootFolder" : "/gpfs/petra3/scratch/yakubov/receiver_tests/files"
+ "Tag": "test_receiver"
}
diff --git a/tests/manual/python_tests/producer/authorizer.json.tpl b/tests/manual/python_tests/producer/authorizer.json.tpl
index f6d57da90da2a18c01b519e0cf0ef51f3ab667dc..9ad08b7e6111f19abd222aa13fa07abf1fb9025e 100644
--- a/tests/manual/python_tests/producer/authorizer.json.tpl
+++ b/tests/manual/python_tests/producer/authorizer.json.tpl
@@ -1,9 +1,9 @@
{
"Port": {{ env "NOMAD_PORT_authorizer" }},
"LogLevel":"debug",
- "AlwaysAllowedBeamtimes":[{"BeamtimeId":"asapo_test","Beamline":"test","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test1","Beamline":"test1","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test2","Beamline":"test2","Year":"2019","Facility":"test_facility"}],
+ "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","Year":"2019","Facility":"test_facility"},
+ {"beamtimeId":"asapo_test1","beamline":"test1","Year":"2019","Facility":"test_facility"},
+ {"beamtimeId":"asapo_test2","beamline":"test2","Year":"2019","Facility":"test_facility"}],
"SecretFile":"auth_secret.key"
}
diff --git a/tests/manual/python_tests/producer/receiver.json.tpl b/tests/manual/python_tests/producer/receiver.json.tpl
index bb59485af142c6d380d23f94b5e89b96c76f5e64..a235fec0d273fea59a9e8537f5f65324c38498a8 100644
--- a/tests/manual/python_tests/producer/receiver.json.tpl
+++ b/tests/manual/python_tests/producer/receiver.json.tpl
@@ -20,6 +20,5 @@
"WriteToDisk": true,
"ReceiveToDiskThresholdMB":50,
"WriteToDb": true,
- "LogLevel" : "debug",
- "RootFolder" : "/tmp/asapo/receiver/files"
+ "LogLevel" : "debug"
}
diff --git a/tests/manual/python_tests/producer/test.py b/tests/manual/python_tests/producer/test.py
index b5354b29daf550583db5cd6a3efc6fa4000bdf2e..2d364a7d7e4827f655e79b38229d88b7db457214 100644
--- a/tests/manual/python_tests/producer/test.py
+++ b/tests/manual/python_tests/producer/test.py
@@ -27,7 +27,7 @@ def assert_err(err):
print(err)
sys.exit(1)
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads ,0)
+producer = asapo_producer.create_producer(endpoint,beamtime,'auto', stream, token, nthreads ,0)
producer.set_log_level("info")
diff --git a/tests/manual/python_tests/producer_wait_bug_mongo/authorizer.json.tpl b/tests/manual/python_tests/producer_wait_bug_mongo/authorizer.json.tpl
index f6d57da90da2a18c01b519e0cf0ef51f3ab667dc..9ad08b7e6111f19abd222aa13fa07abf1fb9025e 100644
--- a/tests/manual/python_tests/producer_wait_bug_mongo/authorizer.json.tpl
+++ b/tests/manual/python_tests/producer_wait_bug_mongo/authorizer.json.tpl
@@ -1,9 +1,9 @@
{
"Port": {{ env "NOMAD_PORT_authorizer" }},
"LogLevel":"debug",
- "AlwaysAllowedBeamtimes":[{"BeamtimeId":"asapo_test","Beamline":"test","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test1","Beamline":"test1","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test2","Beamline":"test2","Year":"2019","Facility":"test_facility"}],
+ "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","Year":"2019","Facility":"test_facility"},
+ {"beamtimeId":"asapo_test1","beamline":"test1","Year":"2019","Facility":"test_facility"},
+ {"beamtimeId":"asapo_test2","beamline":"test2","Year":"2019","Facility":"test_facility"}],
"SecretFile":"auth_secret.key"
}
diff --git a/tests/manual/python_tests/producer_wait_bug_mongo/receiver.json.tpl b/tests/manual/python_tests/producer_wait_bug_mongo/receiver.json.tpl
index bb59485af142c6d380d23f94b5e89b96c76f5e64..a235fec0d273fea59a9e8537f5f65324c38498a8 100644
--- a/tests/manual/python_tests/producer_wait_bug_mongo/receiver.json.tpl
+++ b/tests/manual/python_tests/producer_wait_bug_mongo/receiver.json.tpl
@@ -20,6 +20,5 @@
"WriteToDisk": true,
"ReceiveToDiskThresholdMB":50,
"WriteToDb": true,
- "LogLevel" : "debug",
- "RootFolder" : "/tmp/asapo/receiver/files"
+ "LogLevel" : "debug"
}
diff --git a/tests/manual/python_tests/producer_wait_bug_mongo/test.py b/tests/manual/python_tests/producer_wait_bug_mongo/test.py
index 99d063b467a4f2fe9cfc498a94ab1252e333f638..9e420f33ccef2879ae05e46fa3a616edf241d88d 100644
--- a/tests/manual/python_tests/producer_wait_bug_mongo/test.py
+++ b/tests/manual/python_tests/producer_wait_bug_mongo/test.py
@@ -27,7 +27,7 @@ def assert_err(err):
print(err)
sys.exit(1)
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads, 600)
+producer = asapo_producer.create_producer(endpoint,beamtime,'auto', stream, token, nthreads, 600)
producer.set_log_level("debug")
diff --git a/tests/manual/python_tests/producer_wait_threads/producer_api.py b/tests/manual/python_tests/producer_wait_threads/producer_api.py
index e15ce89b8b6214fe763b6eb8f66697e9404e0dc9..85ccd36c07dfe93c0018d1bb017fedbb8e18f11b 100644
--- a/tests/manual/python_tests/producer_wait_threads/producer_api.py
+++ b/tests/manual/python_tests/producer_wait_threads/producer_api.py
@@ -22,7 +22,7 @@ def callback(header,err):
print ("successfuly sent: ",header)
lock.release()
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads, 600)
+producer = asapo_producer.create_producer(endpoint,beamtime, 'auto', stream, token, nthreads, 600)
producer.set_log_level("info")
@@ -63,7 +63,7 @@ if n!=0:
# create with error
try:
- producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, 0, 600)
+ producer = asapo_producer.create_producer(endpoint,beamtime,'auto', stream, token, 0, 600)
except Exception as Asapo:
print(e)
else:
diff --git a/tests/manual/python_tests/producer_wait_threads/test.py b/tests/manual/python_tests/producer_wait_threads/test.py
index 70c04381ed06d5b10819b35146a6f064b2c26f79..5ebe7b95caec871caed2240f232a8494b05857bf 100644
--- a/tests/manual/python_tests/producer_wait_threads/test.py
+++ b/tests/manual/python_tests/producer_wait_threads/test.py
@@ -22,7 +22,7 @@ def callback(header,err):
print ("successfuly sent: ",header)
lock.release()
-producer = asapo_producer.create_producer(endpoint,beamtime, stream, token, nthreads, 600)
+producer = asapo_producer.create_producer(endpoint,beamtime,'auto', stream, token, nthreads, 600)
producer.set_log_level("info")
diff --git a/tests/manual/receiver_debug_local/authorizer.json.tpl b/tests/manual/receiver_debug_local/authorizer.json.tpl
index f6d57da90da2a18c01b519e0cf0ef51f3ab667dc..9ad08b7e6111f19abd222aa13fa07abf1fb9025e 100644
--- a/tests/manual/receiver_debug_local/authorizer.json.tpl
+++ b/tests/manual/receiver_debug_local/authorizer.json.tpl
@@ -1,9 +1,9 @@
{
"Port": {{ env "NOMAD_PORT_authorizer" }},
"LogLevel":"debug",
- "AlwaysAllowedBeamtimes":[{"BeamtimeId":"asapo_test","Beamline":"test","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test1","Beamline":"test1","Year":"2019","Facility":"test_facility"},
- {"BeamtimeId":"asapo_test2","Beamline":"test2","Year":"2019","Facility":"test_facility"}],
+ "AlwaysAllowedBeamtimes":[{"beamtimeId":"asapo_test","beamline":"test","Year":"2019","Facility":"test_facility"},
+ {"beamtimeId":"asapo_test1","beamline":"test1","Year":"2019","Facility":"test_facility"},
+ {"beamtimeId":"asapo_test2","beamline":"test2","Year":"2019","Facility":"test_facility"}],
"SecretFile":"auth_secret.key"
}
diff --git a/tests/manual/receiver_debug_local/broker.json.tpl b/tests/manual/receiver_debug_local/broker.json.tpl
index 0ca5866f9d51339e15ed19f6dbcd1c3a5e1c3c92..56694d8538be26718c9e32bc69a538ea20abc987 100644
--- a/tests/manual/receiver_debug_local/broker.json.tpl
+++ b/tests/manual/receiver_debug_local/broker.json.tpl
@@ -3,7 +3,7 @@
"DiscoveryServer": "localhost:8400/discovery",
"PerformanceDbServer": "localhost:8086",
"PerformanceDbName": "db_test",
- "port":{{ env "NOMAD_PORT_broker" }},
+ "Port":{{ env "NOMAD_PORT_broker" }},
"LogLevel":"info",
"SecretFile":"auth_secret.key"
}
\ No newline at end of file
diff --git a/tests/manual/receiver_debug_local/receiver.json b/tests/manual/receiver_debug_local/receiver.json
index ace0fbecf3ab6726516b2e530ecbabe4ea6da2ad..6a5d12f59d24f354bfba8354a61341fcafbad518 100644
--- a/tests/manual/receiver_debug_local/receiver.json
+++ b/tests/manual/receiver_debug_local/receiver.json
@@ -20,6 +20,5 @@
"WriteToDisk": true,
"ReceiveToDiskThresholdMB":50,
"WriteToDb": true,
- "LogLevel" : "debug",
- "RootFolder" : "/tmp/asapo/receiver/files"
+ "LogLevel" : "debug"
}