MG-2235 - Check gRPC service is healthy during setup (#2245)

Signed-off-by: Rodney Osodo <28790446+rodneyosodo@users.noreply.github.com>
This commit is contained in:
b1ackd0t
2024-06-12 12:46:42 +03:00
committed by GitHub
parent 75a66a4f9b
commit 487dcc643c
25 changed files with 123 additions and 53 deletions
+15 -15
View File
@@ -63,7 +63,7 @@ func startGRPCServer(svc auth.Service, port int) {
}
func TestIssue(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -132,7 +132,7 @@ func TestIssue(t *testing.T) {
}
func TestRefresh(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -178,7 +178,7 @@ func TestRefresh(t *testing.T) {
}
func TestIdentify(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -222,7 +222,7 @@ func TestIdentify(t *testing.T) {
}
func TestAuthorize(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -344,7 +344,7 @@ func TestAuthorize(t *testing.T) {
}
func TestAddPolicy(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -398,7 +398,7 @@ func TestAddPolicy(t *testing.T) {
}
func TestAddPolicies(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -460,7 +460,7 @@ func TestAddPolicies(t *testing.T) {
}
func TestDeletePolicy(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -513,7 +513,7 @@ func TestDeletePolicy(t *testing.T) {
}
func TestDeletePolicies(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -576,7 +576,7 @@ func TestDeletePolicies(t *testing.T) {
}
func TestListObjects(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -626,7 +626,7 @@ func TestListObjects(t *testing.T) {
}
func TestListAllObjects(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -676,7 +676,7 @@ func TestListAllObjects(t *testing.T) {
}
func TestCountObects(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -726,7 +726,7 @@ func TestCountObects(t *testing.T) {
}
func TestListSubjects(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -776,7 +776,7 @@ func TestListSubjects(t *testing.T) {
}
func TestListAllSubjects(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf(`"Unexpected error creating client connection %s"`, err))
client := grpcapi.NewClient(conn, time.Second)
@@ -826,7 +826,7 @@ func TestListAllSubjects(t *testing.T) {
}
func TestCountSubjects(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
@@ -881,7 +881,7 @@ func TestCountSubjects(t *testing.T) {
}
func TestListPermissions(t *testing.T) {
conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
client := grpcapi.NewClient(conn, time.Second)
+1 -1
View File
@@ -107,7 +107,7 @@ func main() {
return
}
authClient, authHandler, err := auth.Setup(authConfig)
authClient, authHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+2 -2
View File
@@ -76,7 +76,7 @@ func main() {
return
}
ac, acHandler, err := auth.Setup(authConfig)
ac, acHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
@@ -93,7 +93,7 @@ func main() {
return
}
tc, tcHandler, err := auth.SetupAuthz(authConfig)
tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -126,7 +126,7 @@ func main() {
return
}
authClient, authHandler, err := auth.Setup(authConfig)
authClient, authHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -94,7 +94,7 @@ func main() {
return
}
authClient, authHandler, err := auth.SetupAuthz(authConfig)
authClient, authHandler, err := auth.SetupAuthz(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -94,7 +94,7 @@ func main() {
return
}
authClient, authHandler, err := auth.SetupAuthz(authConfig)
authClient, authHandler, err := auth.SetupAuthz(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+2 -2
View File
@@ -75,7 +75,7 @@ func main() {
return
}
ac, acHandler, err := auth.Setup(authConfig)
ac, acHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
@@ -92,7 +92,7 @@ func main() {
return
}
tc, tcHandler, err := auth.SetupAuthz(authConfig)
tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -98,7 +98,7 @@ func main() {
exitCode = 1
return
}
authClient, authHandler, err := auth.Setup(authConfig)
authClient, authHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+2 -2
View File
@@ -84,7 +84,7 @@ func main() {
return
}
ac, acHandler, err := auth.Setup(authConfig)
ac, acHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
@@ -101,7 +101,7 @@ func main() {
return
}
tc, tcHandler, err := auth.SetupAuthz(authConfig)
tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -171,7 +171,7 @@ func main() {
return
}
authClient, authHandler, err := auth.SetupAuthz(authConfig)
authClient, authHandler, err := auth.SetupAuthz(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+2 -2
View File
@@ -90,7 +90,7 @@ func main() {
return
}
ac, acHandler, err := auth.Setup(authConfig)
ac, acHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
@@ -107,7 +107,7 @@ func main() {
return
}
tc, tcHandler, err := auth.SetupAuthz(authConfig)
tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -139,7 +139,7 @@ func main() {
return
}
authClient, authHandler, err := auth.Setup(authConfig)
authClient, authHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -140,7 +140,7 @@ func main() {
return
}
authClient, authHandler, err := auth.Setup(authConfig)
authClient, authHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -158,7 +158,7 @@ func main() {
return
}
authServiceClient, authHandler, err := auth.Setup(authConfig)
authServiceClient, authHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+2 -2
View File
@@ -90,7 +90,7 @@ func main() {
return
}
ac, acHandler, err := auth.Setup(authConfig)
ac, acHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
@@ -107,7 +107,7 @@ func main() {
return
}
tc, tcHandler, err := auth.SetupAuthz(authConfig)
tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -134,7 +134,7 @@ func main() {
return
}
authServiceClient, authHandler, err := auth.Setup(authConfig)
authServiceClient, authHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -154,7 +154,7 @@ func main() {
return
}
authClient, authHandler, err := auth.Setup(authConfig)
authClient, authHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+1 -1
View File
@@ -96,7 +96,7 @@ func main() {
return
}
authClient, authHandler, err := auth.SetupAuthz(authConfig)
authClient, authHandler, err := auth.SetupAuthz(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
+2
View File
@@ -175,6 +175,7 @@ services:
container_name: magistrala-invitations
restart: on-failure
depends_on:
- auth
- invitations-db
environment:
MG_INVITATIONS_LOG_LEVEL: ${MG_INVITATIONS_LOG_LEVEL}
@@ -255,6 +256,7 @@ services:
env_file:
- .env
depends_on:
- auth
- things
- users
- mqtt-adapter
+8 -3
View File
@@ -18,16 +18,17 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/health"
grpchealth "google.golang.org/grpc/health/grpc_health_v1"
)
const (
stopWaitTime = 5 * time.Second
)
const stopWaitTime = 5 * time.Second
type Server struct {
server.BaseServer
server *grpc.Server
registerService serviceRegister
health *health.Server
}
type serviceRegister func(srv *grpc.Server)
@@ -116,7 +117,10 @@ func (s *Server) Start() error {
grpcServerOptions = append(grpcServerOptions, creds)
s.server = grpc.NewServer(grpcServerOptions...)
s.health = health.NewServer()
grpchealth.RegisterHealthServer(s.server, s.health)
s.registerService(s.server)
s.health.SetServingStatus(s.Name, grpchealth.HealthCheckResponse_SERVING)
go func() {
errCh <- s.server.Serve(listener)
@@ -136,6 +140,7 @@ func (s *Server) Stop() error {
c := make(chan bool)
go func() {
defer close(c)
s.health.Shutdown()
s.server.GracefulStop()
}()
select {
+26 -4
View File
@@ -4,22 +4,36 @@
package auth
import (
"context"
"github.com/absmach/magistrala"
authgrpc "github.com/absmach/magistrala/auth/api/grpc"
"github.com/absmach/magistrala/pkg/errors"
thingsauth "github.com/absmach/magistrala/things/api/grpc"
grpchealth "google.golang.org/grpc/health/grpc_health_v1"
)
var errSvcNotServing = errors.New("service is not serving")
// Setup loads Auth gRPC configuration and creates new Auth gRPC client.
//
// For example:
//
// authClient, authHandler, err := auth.Setup(auth.Config{})
func Setup(cfg Config) (magistrala.AuthServiceClient, Handler, error) {
// authClient, authHandler, err := auth.Setup(ctx, auth.Config{})
func Setup(ctx context.Context, cfg Config) (magistrala.AuthServiceClient, Handler, error) {
client, err := newHandler(cfg)
if err != nil {
return nil, nil, err
}
health := grpchealth.NewHealthClient(client.Connection())
resp, err := health.Check(ctx, &grpchealth.HealthCheckRequest{
Service: "auth",
})
if err != nil || resp.GetStatus() != grpchealth.HealthCheckResponse_SERVING {
return nil, nil, errSvcNotServing
}
return authgrpc.NewClient(client.Connection(), cfg.Timeout), client, nil
}
@@ -27,12 +41,20 @@ func Setup(cfg Config) (magistrala.AuthServiceClient, Handler, error) {
//
// For example:
//
// authzClient, authzHandler, err := auth.Setup(auth.Config{})
func SetupAuthz(cfg Config) (magistrala.AuthzServiceClient, Handler, error) {
// authzClient, authzHandler, err := auth.Setup(ctx, auth.Config{})
func SetupAuthz(ctx context.Context, cfg Config) (magistrala.AuthzServiceClient, Handler, error) {
client, err := newHandler(cfg)
if err != nil {
return nil, nil, err
}
health := grpchealth.NewHealthClient(client.Connection())
resp, err := health.Check(ctx, &grpchealth.HealthCheckRequest{
Service: "things",
})
if err != nil || resp.GetStatus() != grpchealth.HealthCheckResponse_SERVING {
return nil, nil, errSvcNotServing
}
return thingsauth.NewClient(client.Connection(), cfg.Timeout), client, nil
}
+46 -6
View File
@@ -4,16 +4,41 @@
package auth_test
import (
"context"
"fmt"
"testing"
"time"
"github.com/absmach/magistrala"
authgrpcapi "github.com/absmach/magistrala/auth/api/grpc"
"github.com/absmach/magistrala/auth/mocks"
"github.com/absmach/magistrala/internal/server"
grpcserver "github.com/absmach/magistrala/internal/server/grpc"
mglog "github.com/absmach/magistrala/logger"
"github.com/absmach/magistrala/pkg/auth"
"github.com/absmach/magistrala/pkg/errors"
thingsgrpcapi "github.com/absmach/magistrala/things/api/grpc"
thmocks "github.com/absmach/magistrala/things/mocks"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
)
func TestSetupAuth(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
registerAuthServiceServer := func(srv *grpc.Server) {
magistrala.RegisterAuthServiceServer(srv, authgrpcapi.NewServer(new(mocks.Service)))
}
gs := grpcserver.New(ctx, cancel, "auth", server.Config{Port: "12345"}, registerAuthServiceServer, mglog.NewMock())
go func() {
err := gs.Start()
assert.Nil(t, err, fmt.Sprintf(`"Unexpected error creating server %s"`, err))
}()
defer func() {
err := gs.Stop()
assert.Nil(t, err, fmt.Sprintf(`"Unexpected error stopping server %s"`, err))
}()
cases := []struct {
desc string
config auth.Config
@@ -22,7 +47,7 @@ func TestSetupAuth(t *testing.T) {
{
desc: "successful",
config: auth.Config{
URL: "localhost:8080",
URL: "localhost:12345",
Timeout: time.Second,
},
err: nil,
@@ -33,13 +58,13 @@ func TestSetupAuth(t *testing.T) {
URL: "",
Timeout: time.Second,
},
err: errors.New("failed to connect to grpc server"),
err: errors.New("service is not serving"),
},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
client, handler, err := auth.Setup(c.config)
client, handler, err := auth.Setup(context.Background(), c.config)
assert.True(t, errors.Contains(err, c.err), fmt.Sprintf("expected %s to contain %s", err, c.err))
if err == nil {
assert.NotNil(t, client)
@@ -50,6 +75,21 @@ func TestSetupAuth(t *testing.T) {
}
func TestSetupAuthz(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
registerAuthaServiceServer := func(srv *grpc.Server) {
magistrala.RegisterAuthzServiceServer(srv, thingsgrpcapi.NewServer(new(thmocks.Service)))
}
gs := grpcserver.New(ctx, cancel, "things", server.Config{Port: "12345"}, registerAuthaServiceServer, mglog.NewMock())
go func() {
err := gs.Start()
assert.Nil(t, err, fmt.Sprintf(`"Unexpected error creating server %s"`, err))
}()
defer func() {
err := gs.Stop()
assert.Nil(t, err, fmt.Sprintf(`"Unexpected error stopping server %s"`, err))
}()
cases := []struct {
desc string
config auth.Config
@@ -58,7 +98,7 @@ func TestSetupAuthz(t *testing.T) {
{
desc: "successful",
config: auth.Config{
URL: "localhost:8080",
URL: "localhost:12345",
Timeout: time.Second,
},
err: nil,
@@ -69,13 +109,13 @@ func TestSetupAuthz(t *testing.T) {
URL: "",
Timeout: time.Second,
},
err: errors.New("failed to connect to grpc server"),
err: errors.New("service is not serving"),
},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
client, handler, err := auth.SetupAuthz(c.config)
client, handler, err := auth.SetupAuthz(context.Background(), c.config)
assert.True(t, errors.Contains(err, c.err), fmt.Sprintf("expected %s to contain %s", err, c.err))
if err == nil {
assert.NotNil(t, client)
+2 -1
View File
@@ -143,9 +143,10 @@ func connect(cfg Config) (*grpc.ClientConn, security, error) {
grpc.WithWriteBufferSize(buffSize),
)
conn, err := grpc.Dial(cfg.URL, opts...)
conn, err := grpc.NewClient(cfg.URL, opts...)
if err != nil {
return nil, secure, errors.Wrap(errGrpcConnect, err)
}
return conn, secure, nil
}
+1 -1
View File
@@ -56,7 +56,7 @@ func TestHandler(t *testing.T) {
URL: "",
Timeout: time.Second,
},
err: errors.New("failed to connect to grpc server"),
secure: "without TLS",
},
{
desc: "failed with invalid server CA file",
+1 -1
View File
@@ -51,7 +51,7 @@ func TestAuthorize(t *testing.T) {
svc := new(mocks.Service)
startGRPCServer(svc, port)
authAddr := fmt.Sprintf("localhost:%d", port)
conn, _ := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, _ := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
client := grpcapi.NewClient(conn, time.Second)
cases := []struct {