From 938dd6cb7830ec22bbeef3d3f906b02238ec82c9 Mon Sep 17 00:00:00 2001 From: Sammy Kerata Oina <44265300+SammyOina@users.noreply.github.com> Date: Thu, 8 Feb 2024 12:07:51 +0300 Subject: [PATCH] NOISSUE - mTLS support across services (#71) * Implemented mTLS support across services Extended gRPC configuration to support mutual TLS (mTLS) in agent and manager components for enhanced security. This includes the loading of Certificate Authority (CA) certificates, server, and client certificates, and keys. Updated README documentation to reflect the new environment variables required for mTLS configuration. Additionally, streamlined secure gRPC client connection setup and logging messages to indicate whether a service is running with TLS, mTLS, or without TLS. The change ensures secure communication between services by verifying both client and server identities, thus addressing potential security concerns in network-level interactions. Signed-off-by: SammyOina * Enhance agent cert handling and update copyright - Implement function to create certificate files for the agent configuration dynamically, ensuring file paths are updated to reflect newly created files. This improves the agent's setup process by automating the certificate handling. - Update copyright clause to reflect the new owning entity, Ultraviolet, affirming correct attribution and compliance with legal requirements. - Refactor gRPC client connection code to remove redundant package alias, streamlining the codebase and improving readability. Signed-off-by: SammyOina * Refactor cert loading with fallbacks Removed redundant certificate file creation logic in the agent module and introduced a more robust loading mechanism in the gRPC server module to support direct byte content aside from file paths. This change simplifies the initial setup process for the agent by removing the need to create certificate files preemptively, thereby streamlining deployment in environments with varying filesystem access. It supports using certificate contents directly, enhancing compatibility with in-memory configurations or environments where file storage may not be ideal. Signed-off-by: SammyOina * fix lint Signed-off-by: SammyOina --------- Signed-off-by: SammyOina --- agent/README.md | 3 ++ agent/computations.go | 12 +++-- cmd/agent/main.go | 10 ++-- internal/server/grpc/grpc.go | 99 ++++++++++++++++++++++++++++++++---- internal/server/server.go | 10 ++-- manager/README.md | 3 ++ manager/manager.proto | 2 +- manager/service.go | 12 +++-- pkg/clients/grpc/connect.go | 85 +++++++++++++++++++++++-------- 9 files changed, 184 insertions(+), 52 deletions(-) diff --git a/agent/README.md b/agent/README.md index 078ae862..3a63a371 100644 --- a/agent/README.md +++ b/agent/README.md @@ -13,6 +13,9 @@ The service is configured using the environment variables from the following tab | AGENT_GRPC_PORT | Agent service gRPC port | 7002 | | AGENT_GRPC_SERVER_CERT | Path to gRPC server certificate in pem format | "" | | AGENT_GRPC_SERVER_KEY | Path to gRPC server key in pem format | "" | +| AGENT_GRPC_SERVER_CA_CERTS | Path to gRPC server CA certificate | "" | +| AGENT_GRPC_CLIENT_CA_CERTS | Path to gRPC client CA certificate | "" | +| COCOS_NOTIFICATION_SERVER_URL | Server to receive notification events from agent. | http:/localhost:9000 | ## Deployment diff --git a/agent/computations.go b/agent/computations.go index 8170cad8..579384b6 100644 --- a/agent/computations.go +++ b/agent/computations.go @@ -14,11 +14,13 @@ var ( ) type AgentConfig struct { - LogLevel string `json:"log_level"` - Host string `json:"host"` - Port string `json:"port"` - CertFile string `json:"cert_file"` - KeyFile string `json:"server_key"` + LogLevel string `json:"log_level"` + Host string `json:"host"` + Port string `json:"port"` + CertFile string `json:"cert_file"` + KeyFile string `json:"server_key"` + ServerCAFile string `json:"server_ca_file"` + ClientCAFile string `json:"client_ca_file"` } type Computation struct { diff --git a/cmd/agent/main.go b/cmd/agent/main.go index c802aae7..ee75b651 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -66,10 +66,12 @@ func main() { } grpcServerConfig := server.Config{ - Port: cfg.AgentConfig.Port, - Host: cfg.AgentConfig.Host, - CertFile: cfg.AgentConfig.CertFile, - KeyFile: cfg.AgentConfig.KeyFile, + Port: cfg.AgentConfig.Port, + Host: cfg.AgentConfig.Host, + CertFile: cfg.AgentConfig.CertFile, + KeyFile: cfg.AgentConfig.KeyFile, + ServerCAFile: cfg.AgentConfig.ServerCAFile, + ClientCAFile: cfg.AgentConfig.ClientCAFile, } registerAgentServiceServer := func(srv *grpc.Server) { diff --git a/internal/server/grpc/grpc.go b/internal/server/grpc/grpc.go index 0103df70..55cc9de3 100644 --- a/internal/server/grpc/grpc.go +++ b/internal/server/grpc/grpc.go @@ -1,18 +1,23 @@ // Copyright (c) Ultraviolet // SPDX-License-Identifier: Apache-2.0 + package grpc import ( "context" + "crypto/tls" + "crypto/x509" "fmt" "log/slog" "net" + "os" "time" "github.com/ultravioletrs/cocos/internal/server" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" ) const ( @@ -31,7 +36,6 @@ var _ server.Server = (*Server)(nil) func New(ctx context.Context, cancel context.CancelFunc, name string, config server.Config, registerService serviceRegister, logger *slog.Logger) server.Server { listenFullAddress := fmt.Sprintf("%s:%s", config.Host, config.Port) - return &Server{ BaseServer: server.BaseServer{ Ctx: ctx, @@ -47,30 +51,71 @@ func New(ctx context.Context, cancel context.CancelFunc, name string, config ser func (s *Server) Start() error { errCh := make(chan error) + grpcServerOptions := []grpc.ServerOption{ + grpc.StatsHandler(otelgrpc.NewServerHandler()), + } listener, err := net.Listen("tcp", s.Address) if err != nil { return fmt.Errorf("failed to listen on port %s: %w", s.Address, err) } + creds := grpc.Creds(insecure.NewCredentials()) switch { case s.Config.CertFile != "" || s.Config.KeyFile != "": - creds, err := credentials.NewServerTLSFromFile(s.Config.CertFile, s.Config.KeyFile) + certificate, err := loadX509KeyPair(s.Config.CertFile, s.Config.KeyFile) if err != nil { return fmt.Errorf("failed to load auth certificates: %w", err) } - s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS cert %s and key %s", s.Name, s.Address, s.Config.CertFile, s.Config.KeyFile)) - s.server = grpc.NewServer( - grpc.Creds(creds), - grpc.StatsHandler(otelgrpc.NewServerHandler()), - ) + tlsConfig := &tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{certificate}, + } + + var mtlsCA string + // Loading Server CA file + rootCA, err := loadCertFile(s.Config.ServerCAFile) + if err != nil { + return fmt.Errorf("failed to load root ca file: %w", err) + } + if len(rootCA) > 0 { + if tlsConfig.RootCAs == nil { + tlsConfig.RootCAs = x509.NewCertPool() + } + if !tlsConfig.RootCAs.AppendCertsFromPEM(rootCA) { + return fmt.Errorf("failed to append root ca to tls.Config") + } + mtlsCA = fmt.Sprintf("root ca %s", s.Config.ServerCAFile) + } + + // Loading Client CA File + clientCA, err := loadCertFile(s.Config.ClientCAFile) + if err != nil { + return fmt.Errorf("failed to load client ca file: %w", err) + } + if len(clientCA) > 0 { + if tlsConfig.ClientCAs == nil { + tlsConfig.ClientCAs = x509.NewCertPool() + } + if !tlsConfig.ClientCAs.AppendCertsFromPEM(clientCA) { + return fmt.Errorf("failed to append client ca to tls.Config") + } + mtlsCA = fmt.Sprintf("%s client ca %s", mtlsCA, s.Config.ClientCAFile) + } + creds = grpc.Creds(credentials.NewTLS(tlsConfig)) + switch { + case mtlsCA != "": + s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS/mTLS cert %s , key %s and %s", s.Name, s.Address, s.Config.CertFile, s.Config.KeyFile, mtlsCA)) + default: + s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS cert %s and key %s", s.Name, s.Address, s.Config.CertFile, s.Config.KeyFile)) + } default: s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s without TLS", s.Name, s.Address)) - s.server = grpc.NewServer( - grpc.StatsHandler(otelgrpc.NewServerHandler()), - ) } + grpcServerOptions = append(grpcServerOptions, creds) + + s.server = grpc.NewServer(grpcServerOptions...) s.registerService(s.server) go func() { @@ -82,7 +127,6 @@ func (s *Server) Start() error { return s.Stop() case err := <-errCh: s.Cancel() - return err } } @@ -102,3 +146,36 @@ func (s *Server) Stop() error { return nil } + +func loadCertFile(certFile string) ([]byte, error) { + if certFile != "" { + return os.ReadFile(certFile) + } + return []byte{}, nil +} + +func loadX509KeyPair(certfile, keyfile string) (tls.Certificate, error) { + var cert, key []byte + var err error + if _, err = os.Stat(certfile); err == nil { + cert, err = os.ReadFile(certfile) + if err != nil { + return tls.Certificate{}, err + } + } else if os.IsNotExist(err) { + cert = []byte(certfile) + } else { + return tls.Certificate{}, err + } + if _, err := os.Stat(keyfile); err == nil { + cert, err = os.ReadFile(keyfile) + if err != nil { + return tls.Certificate{}, err + } + } else if os.IsNotExist(err) { + key = []byte(keyfile) + } else { + return tls.Certificate{}, err + } + return tls.X509KeyPair(cert, key) +} diff --git a/internal/server/server.go b/internal/server/server.go index 55a08d6a..41d6eeed 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -17,10 +17,12 @@ type Server interface { } type Config struct { - Host string `env:"HOST" envDefault:""` - Port string `env:"PORT" envDefault:""` - CertFile string `env:"SERVER_CERT" envDefault:""` - KeyFile string `env:"SERVER_KEY" envDefault:""` + Host string `env:"HOST" envDefault:""` + Port string `env:"PORT" envDefault:""` + CertFile string `env:"SERVER_CERT" envDefault:""` + KeyFile string `env:"SERVER_KEY" envDefault:""` + ServerCAFile string `env:"SERVER_CA_CERTS" envDefault:""` + ClientCAFile string `env:"CLIENT_CA_CERTS" envDefault:""` } type BaseServer struct { diff --git a/manager/README.md b/manager/README.md index 63ed2a3e..353a35a0 100644 --- a/manager/README.md +++ b/manager/README.md @@ -13,6 +13,8 @@ The service is configured using the environment variables from the following tab | MANAGER_GRPC_PORT | Manager service gRPC port | 7001 | | MANAGER_GRPC_SERVER_CERT | Path to server certificate in pem format | | | MANAGER_GRPC_SERVER_KEY | Path to server key in pem format | | +| MANAGER_GRPC_SERVER_CA_CERTS | Path to gRPC server CA certificate | | +| MANAGER_GRPC_CLIENT_CA_CERTS | Path to gRPC client CA certificate | | | COCOS_JAEGER_URL | Jaeger server URL | http://localhost:14268/api/traces | | MANAGER_INSTANCE_ID | Manager service instance ID | | @@ -94,6 +96,7 @@ qemu-system-x86_64 \ Once the VM is booted press enter and on the login use username `root`. #### Build and run Agent +Agent is started automatically in the VM. ```sh # List running processes and use 'grep' to filter for processes containing 'agent' in their names. ps aux | grep cocos-agent diff --git a/manager/manager.proto b/manager/manager.proto index c07dca2b..f24a31ec 100644 --- a/manager/manager.proto +++ b/manager/manager.proto @@ -72,4 +72,4 @@ message AgentConfig { string client_ca_file = 5; string server_ca_file = 6; string log_level = 7; -} \ No newline at end of file +} diff --git a/manager/service.go b/manager/service.go index ffcd66fe..6e7b70d7 100644 --- a/manager/service.go +++ b/manager/service.go @@ -69,11 +69,13 @@ func (ms *managerService) Run(ctx context.Context, c *ComputationRunReq) (string Description: c.Description, ResultConsumers: c.ResultConsumers, AgentConfig: agent.AgentConfig{ - Port: c.AgentConfig.Port, - Host: c.AgentConfig.Host, - KeyFile: c.AgentConfig.KeyFile, - CertFile: c.AgentConfig.CertFile, - LogLevel: c.AgentConfig.LogLevel, + Port: c.AgentConfig.Port, + Host: c.AgentConfig.Host, + KeyFile: c.AgentConfig.KeyFile, + CertFile: c.AgentConfig.CertFile, + ServerCAFile: c.AgentConfig.ServerCaFile, + ClientCAFile: c.AgentConfig.ClientCaFile, + LogLevel: c.AgentConfig.LogLevel, }, } for _, algo := range c.Algorithms { diff --git a/pkg/clients/grpc/connect.go b/pkg/clients/grpc/connect.go index 86b1d443..2e7bbb56 100644 --- a/pkg/clients/grpc/connect.go +++ b/pkg/clients/grpc/connect.go @@ -3,25 +3,38 @@ package grpc import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" "time" "github.com/absmach/magistrala/pkg/errors" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - gogrpc "google.golang.org/grpc" + "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" ) +type security int + +const ( + withoutTLS security = iota + withTLS + withmTLS +) + var ( errGrpcConnect = errors.New("failed to connect to grpc server") errGrpcClose = errors.New("failed to close grpc connection") ) type Config struct { - ClientTLS bool `env:"CLIENT_TLS" envDefault:"false"` - CACerts string `env:"CA_CERTS" envDefault:""` - URL string `env:"URL" envDefault:"localhost:7001"` - Timeout time.Duration `env:"TIMEOUT" envDefault:"60s"` + ClientCert string `env:"CLIENT_CERT" envDefault:""` + ClientKey string `env:"CLIENT_KEY" envDefault:""` + ServerCAFile string `env:"SERVER_CA_CERTS" envDefault:""` + URL string `env:"URL" envDefault:"localhost:7001"` + Timeout time.Duration `env:"TIMEOUT" envDefault:"60s"` } type Client interface { @@ -32,13 +45,13 @@ type Client interface { Secure() string // Connection returns the gRPC connection. - Connection() *gogrpc.ClientConn + Connection() *grpc.ClientConn } type client struct { - *gogrpc.ClientConn + *grpc.ClientConn cfg Config - secure bool + secure security } var _ Client = (*client)(nil) @@ -65,37 +78,65 @@ func (c *client) Close() error { } func (c *client) Secure() string { - if c.secure { + switch c.secure { + case withTLS: return "with TLS" + case withmTLS: + return "with mTLS" + case withoutTLS: + fallthrough + default: + return "without TLS" } - return "without TLS" } -func (c *client) Connection() *gogrpc.ClientConn { +func (c *client) Connection() *grpc.ClientConn { return c.ClientConn } // connect creates new gRPC client and connect to gRPC server. -func connect(cfg Config) (*gogrpc.ClientConn, bool, error) { - var opts []gogrpc.DialOption - secure := false +func connect(cfg Config) (*grpc.ClientConn, security, error) { + opts := []grpc.DialOption{ + grpc.WithStatsHandler(otelgrpc.NewClientHandler()), + } + secure := withoutTLS tc := insecure.NewCredentials() - if cfg.ClientTLS && cfg.CACerts != "" { - var err error - tc, err = credentials.NewClientTLSFromFile(cfg.CACerts, "") + if cfg.ServerCAFile != "" { + tlsConfig := &tls.Config{} + + // Loading root ca certificates file + rootCA, err := os.ReadFile(cfg.ServerCAFile) if err != nil { - return nil, secure, err + return nil, secure, fmt.Errorf("failed to load root ca file: %w", err) } - secure = true + if len(rootCA) > 0 { + capool := x509.NewCertPool() + if !capool.AppendCertsFromPEM(rootCA) { + return nil, secure, fmt.Errorf("failed to append root ca to tls.Config") + } + tlsConfig.RootCAs = capool + secure = withTLS + } + + // Loading mtls certificates file + if cfg.ClientCert != "" || cfg.ClientKey != "" { + certificate, err := tls.LoadX509KeyPair(cfg.ClientCert, cfg.ClientKey) + if err != nil { + return nil, secure, fmt.Errorf("failed to client certificate and key %w", err) + } + tlsConfig.Certificates = []tls.Certificate{certificate} + secure = withmTLS + } + + tc = credentials.NewTLS(tlsConfig) } - opts = append(opts, gogrpc.WithTransportCredentials(tc), gogrpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor())) + opts = append(opts, grpc.WithTransportCredentials(tc)) - conn, err := gogrpc.Dial(cfg.URL, opts...) + conn, err := grpc.Dial(cfg.URL, opts...) if err != nil { return nil, secure, errors.Wrap(errGrpcConnect, err) } - return conn, secure, nil }