Skip to content
Snippets Groups Projects
Commit 43877250 authored by Sergey Yakubov's avatar Sergey Yakubov
Browse files

Merge pull request #24 in ASAPO/asapo from feature/aai to develop

* commit '631491e7': (27 commits)
  use more files for test
  update manual tests
  start adding tokens to worker
  use HMAC token instead of JWT
  create asapo_tools, generating token
  fix
  fix tests
  fix tests
  fix
  update authorizer service - read from files, return beamline
  updated manual tests, fix windows test
  test failure on wrong beamtime_id
  update tests for authorizer
  fix memory leaks
  add authorizer
  restructure go files
  set beamtime_id only once at producer create
  finished auth handler
  more work on authorization handler
  more work on authorization handler
  ...
parents e1743759 631491e7
No related branches found
No related tags found
No related merge requests found
Showing
with 773 additions and 1 deletion
......@@ -122,4 +122,7 @@ doxygen
#GO
broker/pkg
discovery/pkg
\ No newline at end of file
discovery/pkg
common/go/pkg
authorizer/pkg
asapo_tools/pkg
......@@ -68,6 +68,10 @@ add_subdirectory(receiver)
add_subdirectory(discovery)
add_subdirectory(authorizer)
add_subdirectory(asapo_tools)
if(BUILD_INTEGRATION_TESTS)
add_subdirectory(tests)
......
......@@ -2,6 +2,7 @@ function(prepare_asapo)
get_target_property(RECEIVER_DIR receiver-bin BINARY_DIR)
get_target_property(RECEIVER_NAME receiver-bin OUTPUT_NAME)
get_target_property(DISCOVERY_FULLPATH asapo-discovery EXENAME)
get_target_property(AUTHORIZER_FULLPATH asapo-authorizer EXENAME)
get_target_property(BROKER_FULLPATH asapo-broker EXENAME)
set(WORK_DIR ${CMAKE_CURRENT_BINARY_DIR})
if (WIN32)
......@@ -11,9 +12,12 @@ function(prepare_asapo)
endif()
configure_file(${CMAKE_SOURCE_DIR}/config/nomad/receiver.nmd.in receiver.nmd @ONLY)
configure_file(${CMAKE_SOURCE_DIR}/config/nomad/discovery.nmd.in discovery.nmd @ONLY)
configure_file(${CMAKE_SOURCE_DIR}/config/nomad/authorizer.nmd.in authorizer.nmd @ONLY)
configure_file(${CMAKE_SOURCE_DIR}/config/nomad/broker.nmd.in broker.nmd @ONLY)
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/discovery_settings.json.tpl discovery.json.tpl COPYONLY)
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/authorizer_settings.json.tpl authorizer.json.tpl COPYONLY)
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/broker_settings.json.tpl broker.json.tpl COPYONLY)
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/broker_secret.key broker_secret.key COPYONLY)
configure_file(${CMAKE_SOURCE_DIR}/tests/automatic/settings/nginx.conf.tpl nginx.conf.tpl COPYONLY)
configure_file(${CMAKE_SOURCE_DIR}/config/nomad/nginx.nmd.in nginx.nmd @ONLY)
......
set (TARGET_NAME asapo)
if (NOT "$ENV{GOPATH}" STREQUAL "")
set(GOPATH $ENV{GOPATH})
endif()
if (NOT GOPATH)
message (FATAL_ERROR "GOPATH not set")
endif()
message(STATUS "global gopath ${GOPATH}")
IF(WIN32)
set (gopath "${GOPATH}\;${CMAKE_CURRENT_SOURCE_DIR}\;${CMAKE_SOURCE_DIR}/common/go")
set (exe_name "${TARGET_NAME}.exe")
ELSE()
set (gopath ${GOPATH}:${CMAKE_CURRENT_SOURCE_DIR}:${CMAKE_SOURCE_DIR}/common/go)
set (exe_name "${TARGET_NAME}")
ENDIF()
include(testing_go)
add_custom_target(asapo ALL
COMMAND ${CMAKE_COMMAND} -E env GOPATH=${gopath}
go build ${GO_OPTS} -o ${exe_name} asapo_tools/main
VERBATIM)
define_property(TARGET PROPERTY EXENAME
BRIEF_DOCS <executable name>
FULL_DOCS <full-doc>)
set_target_properties(${TARGET_NAME} PROPERTIES EXENAME ${CMAKE_CURRENT_BINARY_DIR}/${exe_name})
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${exe_name} DESTINATION bin)
gotest(${TARGET_NAME} "./...")
// Package contains asapo commands that can be executed from command line.
// Every CommandXxxx function that is a member of a cmd struct processes asapo xxxx command
package cli
import (
"errors"
"flag"
"fmt"
"io"
"os"
"reflect"
"strings"
)
var flHelp bool
var outBuf io.Writer = os.Stdout
func printHelp(f *flag.FlagSet) bool {
if flHelp {
f.Usage()
return true
} else {
return false
}
}
// DoCommand takes command name as a parameter and executes corresponding to this name cmd method
func DoCommand(name string, args []string) error {
commandName := "Command" + strings.ToUpper(name[:1]) + strings.ToLower(name[1:])
cmd := new(command)
methodVal := reflect.ValueOf(cmd).MethodByName(commandName)
if !methodVal.IsValid() {
return errors.New("wrong asapo command: " + name + "\nType 'asapo --help'")
}
cmd.name = name
cmd.args = args
method := methodVal.Interface().(func() error)
return method()
}
// PrintAllCommands prints all available commands (found wihtin methods of cmd)
func PrintAllCommands() {
fmt.Fprintln(outBuf, "\nCommands:")
cmd := new(command)
CmdType := reflect.TypeOf(cmd)
for i := 0; i < CmdType.NumMethod(); i++ {
methodVal := CmdType.Method(i)
if strings.HasPrefix(methodVal.Name, "Command") {
method := methodVal.Func.Interface().(func(*command) error)
cmd.name = strings.ToLower(methodVal.Name)[7:]
cmd.args = []string{"description"}
method(cmd)
}
}
}
package cli
import (
"errors"
"flag"
"fmt"
)
// A command consists of a command name and arguments, passed to this command (all after asapo name ...)
type command struct {
name string
args []string
}
// description prints description line and returns true if first command argument is "description".
func (cmd *command) description(d string) bool {
if len(cmd.args) == 1 && cmd.args[0] == "description" {
fmt.Fprintf(outBuf, " %-10s %s\n", cmd.name, d)
return true
}
return false
}
func (cmd *command) errBadOptions(err string) error {
return errors.New("asapo " + cmd.name + ": " + err + "\nType 'asapo " + cmd.name + " --help'")
}
// createDefaultFlagset creates new flagset and adds default help behaviour.
func (cmd *command) createDefaultFlagset(description, args string) *flag.FlagSet {
flags := flag.NewFlagSet(cmd.name, flag.ExitOnError)
flags.BoolVar(&flHelp, "help", false, "Print usage")
flags.Usage = func() {
fmt.Fprintf(outBuf, "Usage:\t\nasapo %s "+args, cmd.name)
fmt.Fprintf(outBuf, "\n\n%s\n", description)
flags.PrintDefaults()
}
return flags
}
package cli
import (
"bytes"
"testing"
"github.com/stretchr/testify/assert"
)
var CommandTests = []struct {
cmd command
answer string
}{
{command{"token", []string{"-secret", "secret_file", "beamtime"}}, "secret"},
{command{"dummy", []string{"description"}}, "wrong"},
}
func TestCommand(t *testing.T) {
outBuf = new(bytes.Buffer)
for _, test := range CommandTests {
outBuf.(*bytes.Buffer).Reset()
err := DoCommand(test.cmd.name, test.cmd.args)
assert.Contains(t, err.Error(), test.answer, "")
assert.NotNil(t, err, "Should be error")
}
}
func TestPrintAllCommands(t *testing.T) {
outBuf = new(bytes.Buffer)
PrintAllCommands()
assert.Contains(t, outBuf.(*bytes.Buffer).String(), "token", "all commands must have token")
}
package cli
import (
"errors"
"os"
"fmt"
"asapo_common/utils"
)
type tokenFlags struct {
BeamtimeID string
SecretFile string
}
func generateToken(id string,secret string) string {
hmac := utils.NewHMACAuth(secret)
token,err := hmac.GenerateToken(&id)
if (err!=nil) {
fmt.Println(err.Error())
}
return token
}
// GenerateToken generates token for workers
func (cmd *command) CommandToken() error {
message_string := "Generate token"
if cmd.description(message_string) {
return nil
}
flags, err := cmd.parseTokenFlags(message_string)
if err != nil {
return err
}
secret, err := utils.ReadFirstStringFromFile(flags.SecretFile)
if err !=nil {
return err
}
fmt.Fprintf(outBuf, "%s\n", generateToken(flags.BeamtimeID,secret))
return nil
}
func (cmd *command) parseTokenFlags(message_string string) (tokenFlags, error) {
var flags tokenFlags
flagset := cmd.createDefaultFlagset(message_string, "<beamtime id>")
flagset.StringVar(&flags.SecretFile, "secret", "", "path to file with secret")
flagset.Parse(cmd.args)
if printHelp(flagset) {
os.Exit(0)
}
flags.BeamtimeID = flagset.Arg(0)
if flags.BeamtimeID == "" {
return flags, errors.New("beamtime id missed ")
}
if flags.SecretFile == "" {
return flags, errors.New("secret file missed ")
}
return flags, nil
}
package cli
import (
"testing"
"github.com/stretchr/testify/assert"
"bytes"
"io/ioutil"
"os"
)
var tokenTests = []struct {
cmd command
answer string
msg string
}{
{command{args: []string{"beamtime_id"}}, "secret", "no secret parameter"},
{command{args: []string{"-secret","secret.tmp"}}, "beamtime id", "no file"},
{command{args: []string{"-secret","not_existing_file","beamtime_id"}}, "not_existing_file", "no file"},
{command{args: []string{"-secret","secret.tmp","beamtime_id"}}, "eodk3s5ZXwACLGyVA63MZYcOTWuWE4bceI9Vxl9zejI=", "no file"},
}
func TestParseTokenFlags(t *testing.T) {
ioutil.WriteFile("secret.tmp", []byte("secret"), 0644)
outBuf = new(bytes.Buffer)
for _, test := range tokenTests {
err := test.cmd.CommandToken()
if err == nil {
assert.Contains(t, outBuf.(*bytes.Buffer).String(), test.answer, test.msg)
} else {
assert.Contains(t, err.Error(), test.answer, test.msg)
}
}
os.Remove("secret.tmp")
}
package main
import (
"flag"
"fmt"
"os"
"asapo_tools/version"
"asapo_tools/cli"
)
var (
flHelp = flag.Bool("help", false, "Print usage")
)
func main() {
if ret := version.ShowVersion(os.Stdout, "asapo"); ret {
return
}
flag.Parse()
if *flHelp || flag.NArg() == 0 {
flag.Usage()
cli.PrintAllCommands()
return
}
if err := cli.DoCommand(flag.Arg(0), flag.Args()[1:]); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
package version
import (
"flag"
"fmt"
"io"
"os"
)
var version, buildTime, gitCommit, shortVersion string
func ShowVersion(w io.Writer, name string) bool {
flags := flag.NewFlagSet("version", flag.ExitOnError)
flag.Bool("version", false, "Print version information") // to have it in main help
flVersion := flags.Bool("version", false, "Print version information")
flags.Bool("help", false, "Print usage") // define help flag but ignore it
flags.Parse(os.Args[1:])
if *flVersion {
fmt.Fprintf(w, "%s version %s, build time %s\n", name, version, buildTime)
return true
}
return false
}
package version
// Default build-time variable for library-import.
// This file is overridden on build with build-time informations.
func init(){
gitCommit = "@VERSION_SHA1@"
version = "@VERSION@"
shortVersion = "@VERSION_SHORT@"
buildTime = "@TIMESTAMP@"
}
set (TARGET_NAME asapo-authorizer)
if (NOT "$ENV{GOPATH}" STREQUAL "")
set(GOPATH $ENV{GOPATH})
endif()
if (NOT GOPATH)
message (FATAL_ERROR "GOPATH not set")
endif()
message(STATUS "global gopath ${GOPATH}")
IF(WIN32)
set (gopath "${GOPATH}\;${CMAKE_CURRENT_SOURCE_DIR}\;${CMAKE_SOURCE_DIR}/common/go")
set (exe_name "${TARGET_NAME}.exe")
ELSE()
set (gopath ${GOPATH}:${CMAKE_CURRENT_SOURCE_DIR}:${CMAKE_SOURCE_DIR}/common/go)
set (exe_name "${TARGET_NAME}")
ENDIF()
include(testing_go)
add_custom_target(asapo-authorizer ALL
COMMAND ${CMAKE_COMMAND} -E env GOPATH=${gopath}
go build ${GO_OPTS} -o ${exe_name} asapo_authorizer/main
VERBATIM)
define_property(TARGET PROPERTY EXENAME
BRIEF_DOCS <executable name>
FULL_DOCS <full-doc>)
set_target_properties(asapo-authorizer PROPERTIES EXENAME ${CMAKE_CURRENT_BINARY_DIR}/${exe_name})
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${exe_name} DESTINATION bin)
gotest(${TARGET_NAME} "./...")
#go_integration_test(${TARGET_NAME}-connectdb "./..." "MongoDBConnect")
#go_integration_test(${TARGET_NAME}-nextrecord "./..." "MongoDBNext")
//+build !test
package main
import (
log "asapo_common/logger"
"asapo_authorizer/server"
"flag"
"os"
)
func PrintUsage() {
log.Fatal("Usage: " + os.Args[0] + " -config <config file>")
}
func main() {
var fname = flag.String("config", "", "config file path")
flag.Parse()
if *fname == "" {
PrintUsage()
}
logLevel, err := server.ReadConfig(*fname)
if err != nil {
log.Fatal(err.Error())
}
log.SetLevel(logLevel)
server.Start()
}
package server
import (
"net/http"
"encoding/json"
"asapo_common/utils"
"path/filepath"
"strings"
log "asapo_common/logger"
"errors"
)
type authorizationRequest struct {
BeamtimeId string
OriginHost string
}
func extractRequest(r *http.Request) (request authorizationRequest, err error) {
decoder := json.NewDecoder(r.Body)
err = decoder.Decode(&request)
return
}
func splitHost(hostPort string) string {
s := strings.Split(hostPort, ":")
return s[0]
}
func getBeamlineFromIP(ip string) (string, error) {
host := splitHost(ip)
lines, err := utils.ReadStringsFromFile(settings.IpBeamlineMappingFolder + string(filepath.Separator) + host)
if err != nil {
return "", err
}
if len(lines) < 1 || len(lines[0]) == 0 {
return "", errors.New("file is empty")
}
return lines[0], nil
}
func checkBeamtimeExistsInStrings(info beamtimeInfo, lines []string) bool {
for _, line := range lines {
words := strings.Fields(line)
if len(words) < 3 {
continue
}
if words[1] == info.Beamline && words[2] == info.BeamtimeId {
return true
}
}
return false
}
func beamtimeExists(info beamtimeInfo) bool {
lines, err := utils.ReadStringsFromFile(settings.BeamtimeBeamlineMappingFile)
if err != nil || len(lines) < 3 {
return false
}
lines = lines[2:]
return checkBeamtimeExistsInStrings(info, lines)
}
func authorize(request authorizationRequest) (bool, beamtimeInfo) {
for _, pair := range settings.AlwaysAllowedBeamtimes {
if pair.BeamtimeId == request.BeamtimeId {
return true, pair
}
}
var answer beamtimeInfo
beamline, err := getBeamlineFromIP(request.OriginHost)
if err != nil {
log.Error("cannot find beamline for " + request.OriginHost + " - " + err.Error())
return false, beamtimeInfo{}
}
answer.Beamline = beamline
answer.BeamtimeId = request.BeamtimeId
if (!beamtimeExists(answer)) {
log.Error("cannot authorize beamtime " + answer.BeamtimeId + " for " + request.OriginHost + " in " + answer.Beamline)
return false, beamtimeInfo{}
}
log.Debug("authorized beamtime " + answer.BeamtimeId + " for " + request.OriginHost + " in " + answer.Beamline)
return true, answer
}
func routeAuthorize(w http.ResponseWriter, r *http.Request) {
request, err := extractRequest(r)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
ok, beamtimeInfo := authorize(request)
if (!ok) {
w.WriteHeader(http.StatusUnauthorized)
return
}
res, err := utils.MapToJson(&beamtimeInfo)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte(res))
}
package server
import (
"asapo_common/utils"
"github.com/stretchr/testify/assert"
"net/http"
"net/http/httptest"
"strings"
"testing"
"io/ioutil"
"os"
)
type request struct {
path string
cmd string
answer int
message string
}
func allowBeamlines(beamlines []beamtimeInfo) {
settings.AlwaysAllowedBeamtimes=beamlines
}
func containsMatcher(substr string) func(str string) bool {
return func(str string) bool { return strings.Contains(str, substr) }
}
func makeRequest(request authorizationRequest) string {
buf, _ := utils.MapToJson(request)
return string(buf)
}
func doAuthorizeRequest(path string,buf string) *httptest.ResponseRecorder {
mux := utils.NewRouter(listRoutes)
req, _ := http.NewRequest("POST", path, strings.NewReader(buf))
w := httptest.NewRecorder()
mux.ServeHTTP(w, req)
return w
}
func TestAuthorizeOK(t *testing.T) {
allowBeamlines([]beamtimeInfo{{"asapo_test","beamline"}})
request := makeRequest(authorizationRequest{"asapo_test","host"})
w := doAuthorizeRequest("/authorize",request)
body, _ := ioutil.ReadAll(w.Body)
assert.Contains(t, string(body), "asapo_test", "")
assert.Contains(t, string(body), "beamline", "")
assert.Equal(t, http.StatusOK, w.Code, "")
}
func TestNotAuthorized(t *testing.T) {
request := makeRequest(authorizationRequest{"any_id","host"})
w := doAuthorizeRequest("/authorize",request)
assert.Equal(t, http.StatusUnauthorized, w.Code, "")
}
func TestAuthorizeWrongRequest(t *testing.T) {
w := doAuthorizeRequest("/authorize","babla")
assert.Equal(t, http.StatusBadRequest, w.Code, "")
}
func TestAuthorizeWrongPath(t *testing.T) {
w := doAuthorizeRequest("/authorized","")
assert.Equal(t, http.StatusNotFound, w.Code, "")
}
func TestAlwaysAuthorizeAllowed(t *testing.T) {
allowBeamlines([]beamtimeInfo{{"test","beamline"}})
request := authorizationRequest{"asapo_test","host"}
ok,_ := authorize(request)
assert.Equal(t,false, ok, "")
}
func TestSplitHost(t *testing.T) {
host := splitHost("127.0.0.1:112")
assert.Equal(t,"127.0.0.1", host, "")
}
func TestSplitHostNoPort(t *testing.T) {
host := splitHost("127.0.0.1")
assert.Equal(t,"127.0.0.1", host, "")
}
func TestGetBeamlineFromIP(t *testing.T) {
beamline, err := getBeamlineFromIP("127.0.0.1:112")
assert.NotNil(t,err, "")
assert.Empty(t,beamline, "")
}
func TestCheckBeamtimeExistsInStringsFalse(t *testing.T) {
beamInfo := beamtimeInfo{"123","bl"}
lines:=[]string{"111","flash pg2 11003932 beamtime start: 2018-06-11","petra3 p01 c20180508-000-COM20181 commissioning"}
ok := checkBeamtimeExistsInStrings(beamInfo,lines)
assert.False(t,ok, "")
}
func TestCheckBeamtimeExistsInStringsOk(t *testing.T) {
beamInfo := beamtimeInfo{"11003932","pg2"}
lines:=[]string{"111","flash pg2 11003932 beamtime start: 2018-06-11","petra3 p01 c20180508-000-COM20181 commissioning"}
ok := checkBeamtimeExistsInStrings(beamInfo,lines)
assert.True(t,ok, "")
}
func TestAuthorizeWithFile(t *testing.T) {
settings.IpBeamlineMappingFolder="."
settings.BeamtimeBeamlineMappingFile="file.tmp"
lines :=`
Open beam times as of Thursday, 2018/06/21 11:32
Faclty BL BeamTime Id kind
flash bl1 11003924 beamtime start: 2018-04-24
flash bl2 11003921 beamtime start: 2018-06-08
flash fl24 11001734 beamtime start: 2018-06-13
flash pg2 11003932 beamtime start: 2018-06-11
flash thz 11005667 beamtime start: 2018-05-24
petra3 ext 50000181 beamtime start: 2017-04-12
petra3 ext 50000193 beamtime start: 2017-10-12
petra3 ext 50000202 beamtime start: 2017-12-06
petra3 ext 50000209 beamtime start: 2018-02-19
petra3 ext 50000211 beamtime start: 2018-02-19
petra3 ext 50000214 beamtime start: 2018-04-23
petra3 ext 50000215 beamtime start: 2018-03-23
petra3 ext 50000216 beamtime start: 2018-03-23
petra3 ext 50000217 beamtime start: 2018-03-23
petra3 ext 50000218 beamtime start: 2018-03-23
petra3 ext 50000219 beamtime start: 2018-04-24
petra3 ext 50000221 beamtime start: 2018-06-14
petra3 p01 11004172 beamtime start: 2018-06-20
petra3 p01 c20180508-000-COM20181 commissioning
petra3 p02.1 11004341 beamtime start: 2018-06-18
`
ioutil.WriteFile("file.tmp", []byte(lines), 0644)
ioutil.WriteFile("127.0.0.1", []byte("bl1"), 0644)
request := authorizationRequest{"11003924","127.0.0.1"}
w := doAuthorizeRequest("/authorize",makeRequest(request))
body, _ := ioutil.ReadAll(w.Body)
assert.Contains(t, string(body), request.BeamtimeId, "")
assert.Contains(t, string(body), "bl1", "")
assert.Equal(t, http.StatusOK, w.Code, "")
request = authorizationRequest{"wrong","127.0.0.1"}
w = doAuthorizeRequest("/authorize",makeRequest(request))
assert.Equal(t, http.StatusUnauthorized, w.Code, "")
os.Remove("127.0.0.1")
os.Remove("file.tmp")
}
package server
import (
"net/http"
)
func routeGetHealth(w http.ResponseWriter, r *http.Request) {
r.Header.Set("Content-type", "application/json")
w.WriteHeader(http.StatusNoContent)
}
package server
import (
"github.com/stretchr/testify/assert"
"net/http"
"testing"
"net/http/httptest"
"asapo_common/utils"
)
func TestGetNext(t *testing.T) {
mux := utils.NewRouter(listRoutes)
req, _ := http.NewRequest("GET", "/health-check", nil)
w := httptest.NewRecorder()
mux.ServeHTTP(w, req)
assert.Equal(t, http.StatusNoContent, w.Code)
}
package server
import (
"asapo_common/utils"
)
var listRoutes = utils.Routes{
utils.Route{
"Authorize",
"POST",
"/authorize",
routeAuthorize,
},
utils.Route{
"HealthCheck",
"Get",
"/health-check",
routeGetHealth,
},
}
package server
type beamtimeInfo struct {
BeamtimeId string
Beamline string
}
type serverSettings struct {
Port int
LogLevel string
IpBeamlineMappingFolder string
BeamtimeBeamlineMappingFile string
AlwaysAllowedBeamtimes []beamtimeInfo
}
var settings serverSettings
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment