NOISSUE - mTLS support across services (#71)

* Implemented mTLS support across services

Extended gRPC configuration to support mutual TLS (mTLS) in agent and manager components for enhanced security. This includes the loading of Certificate Authority (CA) certificates, server, and client certificates, and keys. Updated README documentation to reflect the new environment variables required for mTLS configuration. Additionally, streamlined secure gRPC client connection setup and logging messages to indicate whether a service is running with TLS, mTLS, or without TLS.

The change ensures secure communication between services by verifying both client and server identities, thus addressing potential security concerns in network-level interactions.

Signed-off-by: SammyOina <sammyoina@gmail.com>

* Enhance agent cert handling and update copyright

- Implement function to create certificate files for the agent configuration dynamically, ensuring file paths are updated to reflect newly created files. This improves the agent's setup process by automating the certificate handling.
- Update copyright clause to reflect the new owning entity, Ultraviolet, affirming correct attribution and compliance with legal requirements.
- Refactor gRPC client connection code to remove redundant package alias, streamlining the codebase and improving readability.

Signed-off-by: SammyOina <sammyoina@gmail.com>

* Refactor cert loading with fallbacks

Removed redundant certificate file creation logic in the agent module and introduced a more robust loading mechanism in the gRPC server module to support direct byte content aside from file paths. This change simplifies the initial setup process for the agent by removing the need to create certificate files preemptively, thereby streamlining deployment in environments with varying filesystem access. It supports using certificate contents directly, enhancing compatibility with in-memory configurations or environments where file storage may not be ideal.

Signed-off-by: SammyOina <sammyoina@gmail.com>

* fix lint

Signed-off-by: SammyOina <sammyoina@gmail.com>

---------

Signed-off-by: SammyOina <sammyoina@gmail.com>

agent/README.md

@@ -13,6 +13,9 @@ The service is configured using the environment variables from the following tab
 | AGENT_GRPC_PORT               | Agent service gRPC port                                | 7002                           |
 | AGENT_GRPC_SERVER_CERT        | Path to gRPC server certificate in pem format          | ""                             |
 | AGENT_GRPC_SERVER_KEY         | Path to gRPC server key in pem format                  | ""                             |
+| AGENT_GRPC_SERVER_CA_CERTS    | Path to gRPC server CA certificate                     | ""                             |
+| AGENT_GRPC_CLIENT_CA_CERTS    | Path to gRPC client CA certificate                     | ""                             |
+| COCOS_NOTIFICATION_SERVER_URL | Server to receive notification events from agent.      | http:/localhost:9000           |
 
 
 ## Deployment

agent/computations.go

@@ -14,11 +14,13 @@ var (
 )
 
 type AgentConfig struct {
-	LogLevel string `json:"log_level"`
-	Host     string `json:"host"`
-	Port     string `json:"port"`
-	CertFile string `json:"cert_file"`
-	KeyFile  string `json:"server_key"`
+	LogLevel     string `json:"log_level"`
+	Host         string `json:"host"`
+	Port         string `json:"port"`
+	CertFile     string `json:"cert_file"`
+	KeyFile      string `json:"server_key"`
+	ServerCAFile string `json:"server_ca_file"`
+	ClientCAFile string `json:"client_ca_file"`
 }
 
 type Computation struct {

cmd/agent/main.go

@@ -66,10 +66,12 @@ func main() {
 	}
 
 	grpcServerConfig := server.Config{
-		Port:     cfg.AgentConfig.Port,
-		Host:     cfg.AgentConfig.Host,
-		CertFile: cfg.AgentConfig.CertFile,
-		KeyFile:  cfg.AgentConfig.KeyFile,
+		Port:         cfg.AgentConfig.Port,
+		Host:         cfg.AgentConfig.Host,
+		CertFile:     cfg.AgentConfig.CertFile,
+		KeyFile:      cfg.AgentConfig.KeyFile,
+		ServerCAFile: cfg.AgentConfig.ServerCAFile,
+		ClientCAFile: cfg.AgentConfig.ClientCAFile,
 	}
 
 	registerAgentServiceServer := func(srv *grpc.Server) {

internal/server/grpc/grpc.go

@@ -1,18 +1,23 @@
 // Copyright (c) Ultraviolet
 // SPDX-License-Identifier: Apache-2.0
+
 package grpc
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"log/slog"
 	"net"
+	"os"
 	"time"
 
 	"github.com/ultravioletrs/cocos/internal/server"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
 )
 
 const (
@@ -31,7 +36,6 @@ var _ server.Server = (*Server)(nil)
 
 func New(ctx context.Context, cancel context.CancelFunc, name string, config server.Config, registerService serviceRegister, logger *slog.Logger) server.Server {
 	listenFullAddress := fmt.Sprintf("%s:%s", config.Host, config.Port)
-
 	return &Server{
 		BaseServer: server.BaseServer{
 			Ctx:     ctx,
@@ -47,30 +51,71 @@ func New(ctx context.Context, cancel context.CancelFunc, name string, config ser
 
 func (s *Server) Start() error {
 	errCh := make(chan error)
+	grpcServerOptions := []grpc.ServerOption{
+		grpc.StatsHandler(otelgrpc.NewServerHandler()),
+	}
 
 	listener, err := net.Listen("tcp", s.Address)
 	if err != nil {
 		return fmt.Errorf("failed to listen on port %s: %w", s.Address, err)
 	}
+	creds := grpc.Creds(insecure.NewCredentials())
 
 	switch {
 	case s.Config.CertFile != "" || s.Config.KeyFile != "":
-		creds, err := credentials.NewServerTLSFromFile(s.Config.CertFile, s.Config.KeyFile)
+		certificate, err := loadX509KeyPair(s.Config.CertFile, s.Config.KeyFile)
 		if err != nil {
 			return fmt.Errorf("failed to load auth certificates: %w", err)
 		}
-		s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS cert %s and key %s", s.Name, s.Address, s.Config.CertFile, s.Config.KeyFile))
-		s.server = grpc.NewServer(
-			grpc.Creds(creds),
-			grpc.StatsHandler(otelgrpc.NewServerHandler()),
-		)
+		tlsConfig := &tls.Config{
+			ClientAuth:   tls.RequireAndVerifyClientCert,
+			Certificates: []tls.Certificate{certificate},
+		}
+
+		var mtlsCA string
+		// Loading Server CA file
+		rootCA, err := loadCertFile(s.Config.ServerCAFile)
+		if err != nil {
+			return fmt.Errorf("failed to load root ca file: %w", err)
+		}
+		if len(rootCA) > 0 {
+			if tlsConfig.RootCAs == nil {
+				tlsConfig.RootCAs = x509.NewCertPool()
+			}
+			if !tlsConfig.RootCAs.AppendCertsFromPEM(rootCA) {
+				return fmt.Errorf("failed to append root ca to tls.Config")
+			}
+			mtlsCA = fmt.Sprintf("root ca %s", s.Config.ServerCAFile)
+		}
+
+		// Loading Client CA File
+		clientCA, err := loadCertFile(s.Config.ClientCAFile)
+		if err != nil {
+			return fmt.Errorf("failed to load client ca file: %w", err)
+		}
+		if len(clientCA) > 0 {
+			if tlsConfig.ClientCAs == nil {
+				tlsConfig.ClientCAs = x509.NewCertPool()
+			}
+			if !tlsConfig.ClientCAs.AppendCertsFromPEM(clientCA) {
+				return fmt.Errorf("failed to append client ca to tls.Config")
+			}
+			mtlsCA = fmt.Sprintf("%s client ca %s", mtlsCA, s.Config.ClientCAFile)
+		}
+		creds = grpc.Creds(credentials.NewTLS(tlsConfig))
+		switch {
+		case mtlsCA != "":
+			s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS/mTLS cert %s , key %s and %s", s.Name, s.Address, s.Config.CertFile, s.Config.KeyFile, mtlsCA))
+		default:
+			s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS cert %s and key %s", s.Name, s.Address, s.Config.CertFile, s.Config.KeyFile))
+		}
 	default:
 		s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s without TLS", s.Name, s.Address))
-		s.server = grpc.NewServer(
-			grpc.StatsHandler(otelgrpc.NewServerHandler()),
-		)
 	}
 
+	grpcServerOptions = append(grpcServerOptions, creds)
+
+	s.server = grpc.NewServer(grpcServerOptions...)
 	s.registerService(s.server)
 
 	go func() {
@@ -82,7 +127,6 @@ func (s *Server) Start() error {
 		return s.Stop()
 	case err := <-errCh:
 		s.Cancel()
-
 		return err
 	}
 }
@@ -102,3 +146,36 @@ func (s *Server) Stop() error {
 
 	return nil
 }
+
+func loadCertFile(certFile string) ([]byte, error) {
+	if certFile != "" {
+		return os.ReadFile(certFile)
+	}
+	return []byte{}, nil
+}
+
+func loadX509KeyPair(certfile, keyfile string) (tls.Certificate, error) {
+	var cert, key []byte
+	var err error
+	if _, err = os.Stat(certfile); err == nil {
+		cert, err = os.ReadFile(certfile)
+		if err != nil {
+			return tls.Certificate{}, err
+		}
+	} else if os.IsNotExist(err) {
+		cert = []byte(certfile)
+	} else {
+		return tls.Certificate{}, err
+	}
+	if _, err := os.Stat(keyfile); err == nil {
+		cert, err = os.ReadFile(keyfile)
+		if err != nil {
+			return tls.Certificate{}, err
+		}
+	} else if os.IsNotExist(err) {
+		key = []byte(keyfile)
+	} else {
+		return tls.Certificate{}, err
+	}
+	return tls.X509KeyPair(cert, key)
+}

internal/server/server.go

@@ -17,10 +17,12 @@ type Server interface {
 }
 
 type Config struct {
-	Host     string `env:"HOST"           envDefault:""`
-	Port     string `env:"PORT"           envDefault:""`
-	CertFile string `env:"SERVER_CERT"    envDefault:""`
-	KeyFile  string `env:"SERVER_KEY"     envDefault:""`
+	Host         string `env:"HOST"            envDefault:""`
+	Port         string `env:"PORT"            envDefault:""`
+	CertFile     string `env:"SERVER_CERT"     envDefault:""`
+	KeyFile      string `env:"SERVER_KEY"      envDefault:""`
+	ServerCAFile string `env:"SERVER_CA_CERTS" envDefault:""`
+	ClientCAFile string `env:"CLIENT_CA_CERTS" envDefault:""`
 }
 
 type BaseServer struct {

manager/README.md

@@ -13,6 +13,8 @@ The service is configured using the environment variables from the following tab
 | MANAGER_GRPC_PORT             | Manager service gRPC port                                | 7001                              |
 | MANAGER_GRPC_SERVER_CERT      | Path to server certificate in pem format                 |                                   |
 | MANAGER_GRPC_SERVER_KEY       | Path to server key in pem format                         |                                   |
+| MANAGER_GRPC_SERVER_CA_CERTS  | Path to gRPC server CA certificate                       |                                   |
+| MANAGER_GRPC_CLIENT_CA_CERTS  | Path to gRPC client CA certificate                       |                                   |
 | COCOS_JAEGER_URL              | Jaeger server URL                                        | http://localhost:14268/api/traces |
 | MANAGER_INSTANCE_ID           | Manager service instance ID                              |                                   |
 
@@ -94,6 +96,7 @@ qemu-system-x86_64 \
 Once the VM is booted press enter and on the login use username `root`.
 
 #### Build and run Agent
+Agent is started automatically in the VM.
 ```sh
 # List running processes and use 'grep' to filter for processes containing 'agent' in their names.
 ps aux | grep cocos-agent

manager/manager.proto

@@ -72,4 +72,4 @@ message AgentConfig {
   string client_ca_file = 5;
   string server_ca_file = 6;
   string log_level = 7;
-}
+}

manager/service.go

@@ -69,11 +69,13 @@ func (ms *managerService) Run(ctx context.Context, c *ComputationRunReq) (string
 		Description:     c.Description,
 		ResultConsumers: c.ResultConsumers,
 		AgentConfig: agent.AgentConfig{
-			Port:     c.AgentConfig.Port,
-			Host:     c.AgentConfig.Host,
-			KeyFile:  c.AgentConfig.KeyFile,
-			CertFile: c.AgentConfig.CertFile,
-			LogLevel: c.AgentConfig.LogLevel,
+			Port:         c.AgentConfig.Port,
+			Host:         c.AgentConfig.Host,
+			KeyFile:      c.AgentConfig.KeyFile,
+			CertFile:     c.AgentConfig.CertFile,
+			ServerCAFile: c.AgentConfig.ServerCaFile,
+			ClientCAFile: c.AgentConfig.ClientCaFile,
+			LogLevel:     c.AgentConfig.LogLevel,
 		},
 	}
 	for _, algo := range c.Algorithms {

pkg/clients/grpc/connect.go

@@ -3,25 +3,38 @@
 package grpc
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"os"
 	"time"
 
 	"github.com/absmach/magistrala/pkg/errors"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
-	gogrpc "google.golang.org/grpc"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 )
 
+type security int
+
+const (
+	withoutTLS security = iota
+	withTLS
+	withmTLS
+)
+
 var (
 	errGrpcConnect = errors.New("failed to connect to grpc server")
 	errGrpcClose   = errors.New("failed to close grpc connection")
 )
 
 type Config struct {
-	ClientTLS bool          `env:"CLIENT_TLS"    envDefault:"false"`
-	CACerts   string        `env:"CA_CERTS"      envDefault:""`
-	URL       string        `env:"URL"           envDefault:"localhost:7001"`
-	Timeout   time.Duration `env:"TIMEOUT"       envDefault:"60s"`
+	ClientCert   string        `env:"CLIENT_CERT"     envDefault:""`
+	ClientKey    string        `env:"CLIENT_KEY"      envDefault:""`
+	ServerCAFile string        `env:"SERVER_CA_CERTS" envDefault:""`
+	URL          string        `env:"URL"             envDefault:"localhost:7001"`
+	Timeout      time.Duration `env:"TIMEOUT"         envDefault:"60s"`
 }
 
 type Client interface {
@@ -32,13 +45,13 @@ type Client interface {
 	Secure() string
 
 	// Connection returns the gRPC connection.
-	Connection() *gogrpc.ClientConn
+	Connection() *grpc.ClientConn
 }
 
 type client struct {
-	*gogrpc.ClientConn
+	*grpc.ClientConn
 	cfg    Config
-	secure bool
+	secure security
 }
 
 var _ Client = (*client)(nil)
@@ -65,37 +78,65 @@ func (c *client) Close() error {
 }
 
 func (c *client) Secure() string {
-	if c.secure {
+	switch c.secure {
+	case withTLS:
 		return "with TLS"
+	case withmTLS:
+		return "with mTLS"
+	case withoutTLS:
+		fallthrough
+	default:
+		return "without TLS"
 	}
-	return "without TLS"
 }
 
-func (c *client) Connection() *gogrpc.ClientConn {
+func (c *client) Connection() *grpc.ClientConn {
 	return c.ClientConn
 }
 
 // connect creates new gRPC client and connect to gRPC server.
-func connect(cfg Config) (*gogrpc.ClientConn, bool, error) {
-	var opts []gogrpc.DialOption
-	secure := false
+func connect(cfg Config) (*grpc.ClientConn, security, error) {
+	opts := []grpc.DialOption{
+		grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
+	}
+	secure := withoutTLS
 	tc := insecure.NewCredentials()
 
-	if cfg.ClientTLS && cfg.CACerts != "" {
-		var err error
-		tc, err = credentials.NewClientTLSFromFile(cfg.CACerts, "")
+	if cfg.ServerCAFile != "" {
+		tlsConfig := &tls.Config{}
+
+		// Loading root ca certificates file
+		rootCA, err := os.ReadFile(cfg.ServerCAFile)
 		if err != nil {
-			return nil, secure, err
+			return nil, secure, fmt.Errorf("failed to load root ca file: %w", err)
 		}
-		secure = true
+		if len(rootCA) > 0 {
+			capool := x509.NewCertPool()
+			if !capool.AppendCertsFromPEM(rootCA) {
+				return nil, secure, fmt.Errorf("failed to append root ca to tls.Config")
+			}
+			tlsConfig.RootCAs = capool
+			secure = withTLS
+		}
+
+		// Loading mtls certificates file
+		if cfg.ClientCert != "" || cfg.ClientKey != "" {
+			certificate, err := tls.LoadX509KeyPair(cfg.ClientCert, cfg.ClientKey)
+			if err != nil {
+				return nil, secure, fmt.Errorf("failed to client certificate and key %w", err)
+			}
+			tlsConfig.Certificates = []tls.Certificate{certificate}
+			secure = withmTLS
+		}
+
+		tc = credentials.NewTLS(tlsConfig)
 	}
 
-	opts = append(opts, gogrpc.WithTransportCredentials(tc), gogrpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()))
+	opts = append(opts, grpc.WithTransportCredentials(tc))
 
-	conn, err := gogrpc.Dial(cfg.URL, opts...)
+	conn, err := grpc.Dial(cfg.URL, opts...)
 	if err != nil {
 		return nil, secure, errors.Wrap(errGrpcConnect, err)
 	}
-
 	return conn, secure, nil
 }