From 9f38b48053772b0a5b216496b0b28ebe3f95b9d4 Mon Sep 17 00:00:00 2001 From: Sergey Yakubov <sergey.yakubov@desy.de> Date: Tue, 2 Apr 2019 17:05:22 +0200 Subject: [PATCH] refactor broker --- broker/src/asapo_broker/database/database.go | 2 +- .../asapo_broker/database/database_test.go | 2 +- .../asapo_broker/database/mock_database.go | 2 +- broker/src/asapo_broker/database/mongodb.go | 2 +- .../src/asapo_broker/database/mongodb_test.go | 31 ++++++++++--- broker/src/asapo_broker/server/get_id.go | 2 +- broker/src/asapo_broker/server/get_id_test.go | 4 +- broker/src/asapo_broker/server/get_last.go | 2 +- .../src/asapo_broker/server/get_last_test.go | 4 +- broker/src/asapo_broker/server/get_next.go | 2 +- .../src/asapo_broker/server/get_next_test.go | 4 +- broker/src/asapo_broker/server/get_size.go | 9 ++++ .../src/asapo_broker/server/get_size_test.go | 43 +++++++++++++++++++ broker/src/asapo_broker/server/listroutes.go | 6 +++ .../{get_image.go => process_request.go} | 10 ++--- ..._image_test.go => process_request_test.go} | 38 ++++++++-------- .../asapo_broker/server/statistics_writers.go | 1 - 17 files changed, 120 insertions(+), 44 deletions(-) create mode 100644 broker/src/asapo_broker/server/get_size.go create mode 100644 broker/src/asapo_broker/server/get_size_test.go rename broker/src/asapo_broker/server/{get_image.go => process_request.go} (81%) rename broker/src/asapo_broker/server/{get_image_test.go => process_request_test.go} (68%) diff --git a/broker/src/asapo_broker/database/database.go b/broker/src/asapo_broker/database/database.go index efbfec776..5512b8ee8 100644 --- a/broker/src/asapo_broker/database/database.go +++ b/broker/src/asapo_broker/database/database.go @@ -1,7 +1,7 @@ package database type Agent interface { - GetRecordFromDb(db_name string, group_id string, op string, id int) ([]byte, error) + ProcessRequest(db_name string, group_id string, op string, id int) ([]byte, error) Connect(string) error Close() Copy() Agent diff --git a/broker/src/asapo_broker/database/database_test.go b/broker/src/asapo_broker/database/database_test.go index d61896433..db511efdc 100644 --- a/broker/src/asapo_broker/database/database_test.go +++ b/broker/src/asapo_broker/database/database_test.go @@ -11,7 +11,7 @@ func TestMockDataBase(t *testing.T) { db.On("Connect", mock.AnythingOfType("string")).Return(nil) db.On("Close").Return() db.On("Copy").Return(nil) - db.On("GetRecordFromDb", "", "", 0).Return([]byte(""), nil) + db.On("ProcessRequest", "", "", 0).Return([]byte(""), nil) db.Connect("") db.Close() diff --git a/broker/src/asapo_broker/database/mock_database.go b/broker/src/asapo_broker/database/mock_database.go index 2f6b66828..eaa756192 100644 --- a/broker/src/asapo_broker/database/mock_database.go +++ b/broker/src/asapo_broker/database/mock_database.go @@ -24,7 +24,7 @@ func (db *MockedDatabase) Copy() Agent { return db } -func (db *MockedDatabase) GetRecordFromDb(db_name string, group_id string, op string, id int) (answer []byte, err error) { +func (db *MockedDatabase) ProcessRequest(db_name string, group_id string, op string, id int) (answer []byte, err error) { args := db.Called(db_name, group_id, op, id) return args.Get(0).([]byte), args.Error(1) } diff --git a/broker/src/asapo_broker/database/mongodb.go b/broker/src/asapo_broker/database/mongodb.go index 3582a0ca9..1890dbeb8 100644 --- a/broker/src/asapo_broker/database/mongodb.go +++ b/broker/src/asapo_broker/database/mongodb.go @@ -282,7 +282,7 @@ func (db *Mongodb) GetLastRecord(db_name string, group_id string) ([]byte, error return res, err } -func (db *Mongodb) GetRecordFromDb(db_name string, group_id string, op string, id int) (answer []byte, err error) { +func (db *Mongodb) ProcessRequest(db_name string, group_id string, op string, id int) (answer []byte, err error) { switch op { case "next": return db.GetNextRecord(db_name, group_id) diff --git a/broker/src/asapo_broker/database/mongodb_test.go b/broker/src/asapo_broker/database/mongodb_test.go index 038dea77e..b74c37ba7 100644 --- a/broker/src/asapo_broker/database/mongodb_test.go +++ b/broker/src/asapo_broker/database/mongodb_test.go @@ -15,6 +15,11 @@ type TestRecord struct { FName string `bson:"fname" json:"fname"` } +type TestSizeRecord struct { + Size int `bson:"size" json:"size"` +} + + var db Mongodb const dbname = "run1" @@ -28,6 +33,10 @@ var rec1_expect, _ = json.Marshal(rec1) var rec2_expect, _ = json.Marshal(rec2) var rec3_expect, _ = json.Marshal(rec3) +var recs1 = TestSizeRecord{3} +var recs1_expect, _ = json.Marshal(recs1) + + func cleanup() { db.DeleteAllRecords(dbname) db.db_pointers_created = nil @@ -178,7 +187,7 @@ func TestMongoDBGetRecordNext(t *testing.T) { db.Connect(dbaddress) defer cleanup() db.InsertRecord(dbname, &rec1) - res, err := db.GetRecordFromDb(dbname, groupId, "next", 0) + res, err := db.ProcessRequest(dbname, groupId, "next", 0) assert.Nil(t, err) assert.Equal(t, string(rec1_expect), string(res)) } @@ -187,7 +196,7 @@ func TestMongoDBGetRecordID(t *testing.T) { db.Connect(dbaddress) defer cleanup() db.InsertRecord(dbname, &rec1) - res, err := db.GetRecordFromDb(dbname, groupId, "id", 1) + res, err := db.ProcessRequest(dbname, groupId, "id", 1) assert.Nil(t, err) assert.Equal(t, string(rec1_expect), string(res)) } @@ -196,7 +205,7 @@ func TestMongoDBWrongOp(t *testing.T) { db.Connect(dbaddress) defer cleanup() db.InsertRecord(dbname, &rec1) - _, err := db.GetRecordFromDb(dbname, groupId, "bla", 0) + _, err := db.ProcessRequest(dbname, groupId, "bla", 0) assert.NotNil(t, err) } @@ -206,7 +215,7 @@ func TestMongoDBGetRecordLast(t *testing.T) { db.InsertRecord(dbname, &rec1) db.InsertRecord(dbname, &rec2) - res, err := db.GetRecordFromDb(dbname, groupId, "last", 0) + res, err := db.ProcessRequest(dbname, groupId, "last", 0) assert.Nil(t, err) assert.Equal(t, string(rec2_expect), string(res)) } @@ -217,14 +226,24 @@ func TestMongoDBGetNextAfterGetLastCorrect(t *testing.T) { db.InsertRecord(dbname, &rec1) db.InsertRecord(dbname, &rec2) - res, err := db.GetRecordFromDb(dbname, groupId, "last", 0) + res, err := db.ProcessRequest(dbname, groupId, "last", 0) assert.Nil(t, err) assert.Equal(t, string(rec2_expect), string(res)) db.InsertRecord(dbname, &rec3) - res, err = db.GetRecordFromDb(dbname, groupId, "next", 0) + res, err = db.ProcessRequest(dbname, groupId, "next", 0) assert.Nil(t, err) assert.Equal(t, string(rec3_expect), string(res)) } + +/* +func TestMongoDBGetSize(t *testing.T) { + db.Connect(dbaddress) + defer cleanup() + db.InsertRecord(dbname, &rec1) + res, err := db.ProcessRequest(dbname, "", "size", 0) + assert.Nil(t, err) + assert.Equal(t, string(recs1_expect), string(res)) +}*/ \ No newline at end of file diff --git a/broker/src/asapo_broker/server/get_id.go b/broker/src/asapo_broker/server/get_id.go index 80a30ddbe..d5b888dd5 100644 --- a/broker/src/asapo_broker/server/get_id.go +++ b/broker/src/asapo_broker/server/get_id.go @@ -22,5 +22,5 @@ func routeGetByID(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) return } - getImage(w, r, "id", id, false) + processRequest(w, r, "id", id, false) } diff --git a/broker/src/asapo_broker/server/get_id_test.go b/broker/src/asapo_broker/server/get_id_test.go index dc09a5a7e..984f1f3f1 100644 --- a/broker/src/asapo_broker/server/get_id_test.go +++ b/broker/src/asapo_broker/server/get_id_test.go @@ -45,8 +45,8 @@ func TestGetIDTestSuite(t *testing.T) { } func (suite *GetIDTestSuite) TestGetIdCallsCorrectRoutine() { - suite.mock_db.On("GetRecordFromDb", expectedBeamtimeId, "", "id", 1).Return([]byte("Hello"), nil) - logger.MockLog.On("Debug", mock.MatchedBy(containsMatcher("get id request"))) + suite.mock_db.On("ProcessRequest", expectedBeamtimeId, "", "id", 1).Return([]byte("Hello"), nil) + logger.MockLog.On("Debug", mock.MatchedBy(containsMatcher("processing request"))) ExpectCopyClose(suite.mock_db) w := doRequest("/database/" + expectedBeamtimeId + "/1" + correctTokenSuffix) diff --git a/broker/src/asapo_broker/server/get_last.go b/broker/src/asapo_broker/server/get_last.go index 1242ad6bb..414540ba0 100644 --- a/broker/src/asapo_broker/server/get_last.go +++ b/broker/src/asapo_broker/server/get_last.go @@ -5,5 +5,5 @@ import ( ) func routeGetLast(w http.ResponseWriter, r *http.Request) { - getImage(w, r, "last", 0, true) + processRequest(w, r, "last", 0, true) } diff --git a/broker/src/asapo_broker/server/get_last_test.go b/broker/src/asapo_broker/server/get_last_test.go index ea355ca7c..3b6673ff8 100644 --- a/broker/src/asapo_broker/server/get_last_test.go +++ b/broker/src/asapo_broker/server/get_last_test.go @@ -33,8 +33,8 @@ func TestGetLastTestSuite(t *testing.T) { } func (suite *GetLastTestSuite) TestGetLastCallsCorrectRoutine() { - suite.mock_db.On("GetRecordFromDb", expectedBeamtimeId, expectedGroupID, "last", 0).Return([]byte("Hello"), nil) - logger.MockLog.On("Debug", mock.MatchedBy(containsMatcher("get last request"))) + suite.mock_db.On("ProcessRequest", expectedBeamtimeId, expectedGroupID, "last", 0).Return([]byte("Hello"), nil) + logger.MockLog.On("Debug", mock.MatchedBy(containsMatcher("processing request last"))) ExpectCopyClose(suite.mock_db) w := doRequest("/database/" + expectedBeamtimeId + "/" + expectedGroupID + "/last" + correctTokenSuffix) diff --git a/broker/src/asapo_broker/server/get_next.go b/broker/src/asapo_broker/server/get_next.go index 535c54954..5456d4c4c 100644 --- a/broker/src/asapo_broker/server/get_next.go +++ b/broker/src/asapo_broker/server/get_next.go @@ -5,5 +5,5 @@ import ( ) func routeGetNext(w http.ResponseWriter, r *http.Request) { - getImage(w, r, "next", 0, true) + processRequest(w, r, "next", 0, true) } diff --git a/broker/src/asapo_broker/server/get_next_test.go b/broker/src/asapo_broker/server/get_next_test.go index 45cead5a1..5418327d9 100644 --- a/broker/src/asapo_broker/server/get_next_test.go +++ b/broker/src/asapo_broker/server/get_next_test.go @@ -33,8 +33,8 @@ func TestGetNextTestSuite(t *testing.T) { } func (suite *GetNextTestSuite) TestGetNextCallsCorrectRoutine() { - suite.mock_db.On("GetRecordFromDb", expectedBeamtimeId, expectedGroupID, "next", 0).Return([]byte("Hello"), nil) - logger.MockLog.On("Debug", mock.MatchedBy(containsMatcher("get next request"))) + suite.mock_db.On("ProcessRequest", expectedBeamtimeId, expectedGroupID, "next", 0).Return([]byte("Hello"), nil) + logger.MockLog.On("Debug", mock.MatchedBy(containsMatcher("processing request next"))) ExpectCopyClose(suite.mock_db) w := doRequest("/database/" + expectedBeamtimeId + "/" + expectedGroupID + "/next" + correctTokenSuffix) diff --git a/broker/src/asapo_broker/server/get_size.go b/broker/src/asapo_broker/server/get_size.go new file mode 100644 index 000000000..963e74826 --- /dev/null +++ b/broker/src/asapo_broker/server/get_size.go @@ -0,0 +1,9 @@ +package server + +import ( + "net/http" +) + +func routeGetSize(w http.ResponseWriter, r *http.Request) { + processRequest(w, r, "size", 0, false) +} diff --git a/broker/src/asapo_broker/server/get_size_test.go b/broker/src/asapo_broker/server/get_size_test.go new file mode 100644 index 000000000..739abe27a --- /dev/null +++ b/broker/src/asapo_broker/server/get_size_test.go @@ -0,0 +1,43 @@ +package server + +import ( + "asapo_broker/database" + "asapo_common/logger" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "net/http" + "testing" +) + +type GetSizeTestSuite struct { + suite.Suite + mock_db *database.MockedDatabase +} + +func (suite *GetSizeTestSuite) SetupTest() { + statistics.Reset() + suite.mock_db = new(database.MockedDatabase) + db = suite.mock_db + prepareTestAuth() + logger.SetMockLog() +} + +func (suite *GetSizeTestSuite) TearDownTest() { + assertExpectations(suite.T(), suite.mock_db) + logger.UnsetMockLog() + db = nil +} + +func TestGetSizeTestSuite(t *testing.T) { + suite.Run(t, new(GetSizeTestSuite)) +} + +func (suite *GetSizeTestSuite) TestGetSizeOK() { + suite.mock_db.On("ProcessRequest", expectedBeamtimeId, "", "size", 0).Return([]byte("{\"size\":10}"), nil) + logger.MockLog.On("Debug", mock.MatchedBy(containsMatcher("processing request size"))) + ExpectCopyClose(suite.mock_db) + + w := doRequest("/database/" + expectedBeamtimeId + "/size" + correctTokenSuffix) + suite.Equal(http.StatusOK, w.Code, "GetSize OK") + suite.Equal("{\"size\":10}", string(w.Body.Bytes()), "GetSize sends size") +} diff --git a/broker/src/asapo_broker/server/listroutes.go b/broker/src/asapo_broker/server/listroutes.go index 486905245..cfd0a4b86 100644 --- a/broker/src/asapo_broker/server/listroutes.go +++ b/broker/src/asapo_broker/server/listroutes.go @@ -11,6 +11,12 @@ var listRoutes = utils.Routes{ "/database/{dbname}/{groupid}/next", routeGetNext, }, + utils.Route{ + "GetSize", + "Get", + "/database/{dbname}/size", + routeGetSize, + }, utils.Route{ "GetLast", "Get", diff --git a/broker/src/asapo_broker/server/get_image.go b/broker/src/asapo_broker/server/process_request.go similarity index 81% rename from broker/src/asapo_broker/server/get_image.go rename to broker/src/asapo_broker/server/process_request.go index 66a8bd8f7..31a422c86 100644 --- a/broker/src/asapo_broker/server/get_image.go +++ b/broker/src/asapo_broker/server/process_request.go @@ -37,7 +37,7 @@ func checkGroupID(w http.ResponseWriter, needGroupID bool, group_id string, db_n return true } -func getImage(w http.ResponseWriter, r *http.Request, op string, id int, needGroupID bool) { +func processRequest(w http.ResponseWriter, r *http.Request, op string, id int, needGroupID bool) { r.Header.Set("Content-type", "application/json") w.Header().Set("Access-Control-Allow-Origin", "*") db_name, group_id, ok := extractRequestParameters(r, needGroupID) @@ -55,7 +55,7 @@ func getImage(w http.ResponseWriter, r *http.Request, op string, id int, needGro return } - answer, code := getRecord(db_name, group_id, op, id) + answer, code := processRequestInDb(db_name, group_id, op, id) w.WriteHeader(code) w.Write(answer) } @@ -74,12 +74,12 @@ func returnError(err error, log_str string) (answer []byte, code int) { return []byte(err.Error()), code } -func getRecord(db_name string, group_id string, op string, id int) (answer []byte, code int) { +func processRequestInDb(db_name string, group_id string, op string, id int) (answer []byte, code int) { db_new := db.Copy() defer db_new.Close() statistics.IncreaseCounter() - answer, err := db_new.GetRecordFromDb(db_name, group_id, op, id) - log_str := "processing get " + op + " request in " + db_name + " at " + settings.BrokerDbAddress + answer, err := db_new.ProcessRequest(db_name, group_id, op, id) + log_str := "processing request " + op + " in " + db_name + " at " + settings.BrokerDbAddress if err != nil { return returnError(err, log_str) } diff --git a/broker/src/asapo_broker/server/get_image_test.go b/broker/src/asapo_broker/server/process_request_test.go similarity index 68% rename from broker/src/asapo_broker/server/get_image_test.go rename to broker/src/asapo_broker/server/process_request_test.go index 6d583c4cf..694a05edc 100644 --- a/broker/src/asapo_broker/server/get_image_test.go +++ b/broker/src/asapo_broker/server/process_request_test.go @@ -62,7 +62,7 @@ func doRequest(path string, method ...string) *httptest.ResponseRecorder { return w } -func TestGetImageWithoutDatabaseName(t *testing.T) { +func TestProcessRequestWithoutDatabaseName(t *testing.T) { w := doRequest("/database/next") assert.Equal(t, http.StatusNotFound, w.Code, "no database name") } @@ -72,12 +72,12 @@ func ExpectCopyClose(mock_db *database.MockedDatabase) { mock_db.On("Close").Return() } -type GetImageTestSuite struct { +type ProcessRequestTestSuite struct { suite.Suite mock_db *database.MockedDatabase } -func (suite *GetImageTestSuite) SetupTest() { +func (suite *ProcessRequestTestSuite) SetupTest() { statistics.Reset() suite.mock_db = new(database.MockedDatabase) db = suite.mock_db @@ -85,17 +85,17 @@ func (suite *GetImageTestSuite) SetupTest() { logger.SetMockLog() } -func (suite *GetImageTestSuite) TearDownTest() { +func (suite *ProcessRequestTestSuite) TearDownTest() { assertExpectations(suite.T(), suite.mock_db) logger.UnsetMockLog() db = nil } -func TestGetImageTestSuite(t *testing.T) { - suite.Run(t, new(GetImageTestSuite)) +func TestProcessRequestTestSuite(t *testing.T) { + suite.Run(t, new(ProcessRequestTestSuite)) } -func (suite *GetImageTestSuite) TestGetImageWithWrongToken() { +func (suite *ProcessRequestTestSuite) TestProcessRequestWithWrongToken() { logger.MockLog.On("Error", mock.MatchedBy(containsMatcher("wrong token"))) w := doRequest("/database/" + expectedBeamtimeId + "/" + expectedGroupID + "/next" + suffixWithWrongToken) @@ -103,7 +103,7 @@ func (suite *GetImageTestSuite) TestGetImageWithWrongToken() { suite.Equal(http.StatusUnauthorized, w.Code, "wrong token") } -func (suite *GetImageTestSuite) TestGetImageWithNoToken() { +func (suite *ProcessRequestTestSuite) TestProcessRequestWithNoToken() { logger.MockLog.On("Error", mock.MatchedBy(containsMatcher("cannot extract"))) w := doRequest("/database/" + expectedBeamtimeId + "/" + expectedGroupID + "/next" + wrongTokenSuffix) @@ -111,11 +111,11 @@ func (suite *GetImageTestSuite) TestGetImageWithNoToken() { suite.Equal(http.StatusUnauthorized, w.Code, "no token") } -func (suite *GetImageTestSuite) TestGetImageWithWrongDatabaseName() { - suite.mock_db.On("GetRecordFromDb", expectedBeamtimeId, expectedGroupID, "next", 0).Return([]byte(""), +func (suite *ProcessRequestTestSuite) TestProcessRequestWithWrongDatabaseName() { + suite.mock_db.On("ProcessRequest", expectedBeamtimeId, expectedGroupID, "next", 0).Return([]byte(""), &database.DBError{utils.StatusWrongInput, ""}) - logger.MockLog.On("Error", mock.MatchedBy(containsMatcher("get next request"))) + logger.MockLog.On("Error", mock.MatchedBy(containsMatcher("processing request next"))) ExpectCopyClose(suite.mock_db) w := doRequest("/database/" + expectedBeamtimeId + "/" + expectedGroupID + "/next" + correctTokenSuffix) @@ -123,25 +123,25 @@ func (suite *GetImageTestSuite) TestGetImageWithWrongDatabaseName() { suite.Equal(http.StatusBadRequest, w.Code, "wrong database name") } -func (suite *GetImageTestSuite) TestGetImageWithInternalDBError() { - suite.mock_db.On("GetRecordFromDb", expectedBeamtimeId, expectedGroupID, "next", 0).Return([]byte(""), errors.New("")) - logger.MockLog.On("Error", mock.MatchedBy(containsMatcher("get next request"))) +func (suite *ProcessRequestTestSuite) TestProcessRequestWithInternalDBError() { + suite.mock_db.On("ProcessRequest", expectedBeamtimeId, expectedGroupID, "next", 0).Return([]byte(""), errors.New("")) + logger.MockLog.On("Error", mock.MatchedBy(containsMatcher("processing request next"))) ExpectCopyClose(suite.mock_db) w := doRequest("/database/" + expectedBeamtimeId + "/" + expectedGroupID + "/next" + correctTokenSuffix) suite.Equal(http.StatusInternalServerError, w.Code, "internal error") } -func (suite *GetImageTestSuite) TestGetImageAddsCounter() { - suite.mock_db.On("GetRecordFromDb", expectedBeamtimeId, expectedGroupID, "next", 0).Return([]byte("Hello"), nil) - logger.MockLog.On("Debug", mock.MatchedBy(containsMatcher("get next request in "+expectedBeamtimeId))) +func (suite *ProcessRequestTestSuite) TestProcessRequestAddsCounter() { + suite.mock_db.On("ProcessRequest", expectedBeamtimeId, expectedGroupID, "next", 0).Return([]byte("Hello"), nil) + logger.MockLog.On("Debug", mock.MatchedBy(containsMatcher("processing request next in "+expectedBeamtimeId))) ExpectCopyClose(suite.mock_db) doRequest("/database/" + expectedBeamtimeId + "/" + expectedGroupID + "/next" + correctTokenSuffix) - suite.Equal(1, statistics.GetCounter(), "GetImage increases counter") + suite.Equal(1, statistics.GetCounter(), "ProcessRequest increases counter") } -func (suite *GetImageTestSuite) TestGetImageWrongGroupID() { +func (suite *ProcessRequestTestSuite) TestProcessRequestWrongGroupID() { logger.MockLog.On("Error", mock.MatchedBy(containsMatcher("wrong groupid"))) w := doRequest("/database/" + expectedBeamtimeId + "/" + wrongGroupID + "/next" + correctTokenSuffix) suite.Equal(http.StatusBadRequest, w.Code, "wrong group id") diff --git a/broker/src/asapo_broker/server/statistics_writers.go b/broker/src/asapo_broker/server/statistics_writers.go index db6e46fb2..f3ad28d2e 100644 --- a/broker/src/asapo_broker/server/statistics_writers.go +++ b/broker/src/asapo_broker/server/statistics_writers.go @@ -3,7 +3,6 @@ package server import ( - "github.com/influxdata/influxdb/client/v2" "log" "time" ) -- GitLab