COCOS - 209 - Restructure agent and manager gRPC config (#297)

* restructure grpc configs

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

enhance clients

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

restructure config

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

refactor

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

rebase

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

rebase

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

use separate configuration

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

fix tests

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

fix config

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

refactor

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

Lint

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

fix tests

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

add tests

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

add test case

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

add test case

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

refactor

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

further refactor'

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

add tests

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

rebase

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

* remove redundant code

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

* fix test

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>

---------

Signed-off-by: WashingtonKK <washingtonkigan@gmail.com>
This commit is contained in:
Washington Kigani Kamadi
2024-12-04 22:32:46 +03:00
committed by GitHub
parent 92a4f8bd32
commit ec426e58a2
15 changed files with 484 additions and 265 deletions
+91 -93
View File
@@ -60,8 +60,9 @@ type serviceRegister func(srv *grpc.Server)
var _ server.Server = (*Server)(nil)
func New(ctx context.Context, cancel context.CancelFunc, name string, config server.Config, registerService serviceRegister, logger *slog.Logger, qp client.QuoteProvider, authSvc auth.Authenticator) server.Server {
listenFullAddress := fmt.Sprintf("%s:%s", config.Host, config.Port)
func New(ctx context.Context, cancel context.CancelFunc, name string, config server.ServerConfiguration, registerService serviceRegister, logger *slog.Logger, qp client.QuoteProvider, authSvc auth.Authenticator) server.Server {
base := config.GetBaseConfig()
listenFullAddress := fmt.Sprintf("%s:%s", base.Host, base.Port)
return &Server{
BaseServer: server.BaseServer{
Ctx: ctx,
@@ -91,101 +92,98 @@ func (s *Server) Start() error {
creds := grpc.Creds(insecure.NewCredentials())
var listener net.Listener = nil
switch {
case s.Config.AttestedTLS:
certificateBytes, privateKeyBytes, err := generateCertificatesForATLS()
if err != nil {
return fmt.Errorf("failed to create certificate: %w", err)
}
certificate, err := tls.X509KeyPair(certificateBytes, privateKeyBytes)
if err != nil {
return fmt.Errorf("falied due to invalid key pair: %w", err)
}
tlsConfig := &tls.Config{
ClientAuth: tls.NoClientCert,
Certificates: []tls.Certificate{certificate},
}
creds = grpc.Creds(credentials.NewTLS(tlsConfig))
listener, err = atls.Listen(
s.Address,
certificateBytes,
privateKeyBytes,
)
if err != nil {
return fmt.Errorf("failed to create Listener for aTLS: %w", err)
}
s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with Attested TLS", s.Name, s.Address))
case s.Config.CertFile != "" || s.Config.KeyFile != "":
certificate, err := loadX509KeyPair(s.Config.CertFile, s.Config.KeyFile)
if err != nil {
return fmt.Errorf("failed to load auth certificates: %w", err)
}
tlsConfig := &tls.Config{
ClientAuth: tls.NoClientCert,
Certificates: []tls.Certificate{certificate},
}
var mtlsCA string
// Loading Server CA file
rootCA, err := loadCertFile(s.Config.ServerCAFile)
if err != nil {
return fmt.Errorf("failed to load root ca file: %w", err)
}
if len(rootCA) > 0 {
if tlsConfig.RootCAs == nil {
tlsConfig.RootCAs = x509.NewCertPool()
}
if !tlsConfig.RootCAs.AppendCertsFromPEM(rootCA) {
return fmt.Errorf("failed to append root ca to tls.Config")
}
mtlsCA = fmt.Sprintf("root ca %s", s.Config.ServerCAFile)
}
// Loading Client CA File
clientCA, err := loadCertFile(s.Config.ClientCAFile)
if err != nil {
return fmt.Errorf("failed to load client ca file: %w", err)
}
if len(clientCA) > 0 {
if tlsConfig.ClientCAs == nil {
tlsConfig.ClientCAs = x509.NewCertPool()
}
if !tlsConfig.ClientCAs.AppendCertsFromPEM(clientCA) {
return fmt.Errorf("failed to append client ca to tls.Config")
}
mtlsCA = fmt.Sprintf("%s client ca %s", mtlsCA, s.Config.ClientCAFile)
}
if mtlsCA != "" {
tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
}
creds = grpc.Creds(credentials.NewTLS(tlsConfig))
switch c := s.Config.(type) {
case server.AgentConfig:
switch {
case mtlsCA != "":
s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS/mTLS", s.Name, s.Address))
case c.AttestedTLS:
certificateBytes, privateKeyBytes, err := generateCertificatesForATLS()
if err != nil {
return fmt.Errorf("failed to create certificate: %w", err)
}
certificate, err := tls.X509KeyPair(certificateBytes, privateKeyBytes)
if err != nil {
return fmt.Errorf("falied due to invalid key pair: %w", err)
}
tlsConfig := &tls.Config{
ClientAuth: tls.NoClientCert,
Certificates: []tls.Certificate{certificate},
}
creds = grpc.Creds(credentials.NewTLS(tlsConfig))
listener, err = atls.Listen(
s.Address,
certificateBytes,
privateKeyBytes,
)
if err != nil {
return fmt.Errorf("failed to create Listener for aTLS: %w", err)
}
s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with Attested TLS", s.Name, s.Address))
case c.CertFile != "" || c.KeyFile != "":
certificate, err := loadX509KeyPair(c.CertFile, c.KeyFile)
if err != nil {
return fmt.Errorf("failed to load auth certificates: %w", err)
}
tlsConfig := &tls.Config{
ClientAuth: tls.RequireAndVerifyClientCert,
Certificates: []tls.Certificate{certificate},
}
var mtlsCA string
// Loading Server CA file
rootCA, err := loadCertFile(c.ServerCAFile)
if err != nil {
return fmt.Errorf("failed to load root ca file: %w", err)
}
if len(rootCA) > 0 {
if tlsConfig.RootCAs == nil {
tlsConfig.RootCAs = x509.NewCertPool()
}
if !tlsConfig.RootCAs.AppendCertsFromPEM(rootCA) {
return fmt.Errorf("failed to append root ca to tls.Config")
}
mtlsCA = fmt.Sprintf("root ca %s", c.ServerCAFile)
}
// Loading Client CA File
clientCA, err := loadCertFile(c.ClientCAFile)
if err != nil {
return fmt.Errorf("failed to load client ca file: %w", err)
}
if len(clientCA) > 0 {
if tlsConfig.ClientCAs == nil {
tlsConfig.ClientCAs = x509.NewCertPool()
}
if !tlsConfig.ClientCAs.AppendCertsFromPEM(clientCA) {
return fmt.Errorf("failed to append client ca to tls.Config")
}
mtlsCA = fmt.Sprintf("%s client ca %s", mtlsCA, c.ClientCAFile)
}
creds = grpc.Creds(credentials.NewTLS(tlsConfig))
switch {
case mtlsCA != "":
s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS/mTLS cert %s , key %s and %s", s.Name, s.Address, c.CertFile, c.KeyFile, mtlsCA))
default:
s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS cert %s and key %s", s.Name, s.Address, c.CertFile, c.KeyFile))
}
listener, err = net.Listen("tcp", s.Address)
if err != nil {
return fmt.Errorf("failed to listen on port %s: %w", s.Address, err)
}
default:
s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s with TLS", s.Name, s.Address))
}
var err error
listener, err = net.Listen("tcp", s.Address)
if err != nil {
return fmt.Errorf("failed to listen on port %s: %w", s.Address, err)
listener, err = net.Listen("tcp", s.Address)
if err != nil {
return fmt.Errorf("failed to listen on port %s: %w", s.Address, err)
}
s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s without TLS", s.Name, s.Address))
}
default:
var err error
listener, err = net.Listen("tcp", s.Address)
if err != nil {
return fmt.Errorf("failed to listen on port %s: %w", s.Address, err)
}
s.Logger.Info(fmt.Sprintf("%s service gRPC server listening at %s without TLS", s.Name, s.Address))
}
grpcServerOptions = append(grpcServerOptions, creds)
+179 -74
View File
@@ -38,9 +38,13 @@ func TestNew(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
config := server.Config{
Host: "localhost",
Port: "50051",
config := server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "50051",
},
},
}
logger := slog.Default()
qp := new(mocks.QuoteProvider)
@@ -80,11 +84,15 @@ func TestServerStartWithTLSFile(t *testing.T) {
err = keyFile.Close()
assert.NoError(t, err)
config := server.Config{
Host: "localhost",
Port: "0",
CertFile: certFile.Name(),
KeyFile: keyFile.Name(),
config := server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "0",
CertFile: certFile.Name(),
KeyFile: keyFile.Name(),
},
},
}
logBuffer := &ThreadSafeBuffer{}
@@ -119,38 +127,19 @@ func TestServerStartWithTLSFile(t *testing.T) {
func TestServerStartWithmTLSFile(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cert, key, err := generateSelfSignedCert()
caCertFile, clientCertFile, clientKeyFile, err := createCertificatesFiles()
assert.NoError(t, err)
certFile, err := os.CreateTemp("", "cert*.pem")
assert.NoError(t, err)
keyFile, err := os.CreateTemp("", "key*.pem")
assert.NoError(t, err)
t.Cleanup(func() {
os.Remove(certFile.Name())
os.Remove(keyFile.Name())
})
_, err = certFile.Write(cert)
assert.NoError(t, err)
_, err = keyFile.Write(key)
assert.NoError(t, err)
err = certFile.Close()
assert.NoError(t, err)
err = keyFile.Close()
assert.NoError(t, err)
config := server.Config{
Host: "localhost",
Port: "0",
CertFile: certFile.Name(),
KeyFile: keyFile.Name(),
ServerCAFile: certFile.Name(),
ClientCAFile: certFile.Name(),
config := server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "0",
CertFile: string(clientCertFile),
KeyFile: string(clientKeyFile),
ServerCAFile: caCertFile,
},
},
}
logBuffer := &ThreadSafeBuffer{}
@@ -185,9 +174,13 @@ func TestServerStartWithmTLSFile(t *testing.T) {
func TestServerStop(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
config := server.Config{
Host: "localhost",
Port: "0",
config := server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "0",
},
},
}
buf := &ThreadSafeBuffer{}
logger := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
@@ -268,54 +261,74 @@ func (b *ThreadSafeBuffer) String() string {
func TestServerInitializationAndStartup(t *testing.T) {
testCases := []struct {
name string
config server.Config
config server.AgentConfig
expectedLog string
expectError bool
setupCallback func(*testing.T, *server.Config, *ThreadSafeBuffer)
setupCallback func(*testing.T, *server.AgentConfig, *ThreadSafeBuffer)
}{
{
name: "Non-TLS Server Startup",
config: server.Config{
Host: "localhost",
Port: "0",
config: server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "0",
},
},
},
expectedLog: "TestServer service gRPC server listening at localhost:0 without TLS",
},
{
name: "TLS Server Startup with Self-Signed Certificate",
config: server.Config{
Host: "localhost",
Port: "0",
config: server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "0",
},
},
},
setupCallback: setupTLSConfig,
expectedLog: "TestServer service gRPC server listening at localhost:0 with TLS",
},
{
name: "TLS Server Startup with Invalid Certificates",
config: server.Config{
Host: "localhost",
Port: "0",
CertFile: "invalid",
KeyFile: "invalid",
config: server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "0",
CertFile: "invalid",
KeyFile: "invalid",
},
},
},
expectError: true,
expectedLog: "failed to load auth certificates",
},
{
name: "mTLS Server Startup",
config: server.Config{
Host: "localhost",
Port: "0",
config: server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "0",
},
},
},
setupCallback: setupMTLSConfig,
expectedLog: "TestServer service gRPC server listening at localhost:0 with TLS",
},
{
name: "mTLS Server Startup with Invalid Root CA",
config: server.Config{
Host: "localhost",
Port: "0",
ServerCAFile: "invalid",
config: server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "0",
ServerCAFile: "invalid",
},
},
},
setupCallback: setupInvalidRootCAConfig,
expectError: true,
@@ -323,10 +336,14 @@ func TestServerInitializationAndStartup(t *testing.T) {
},
{
name: "mTLS Server Startup with Invalid Client CA",
config: server.Config{
Host: "localhost",
Port: "0",
ServerCAFile: "invalid",
config: server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "0",
ServerCAFile: "invalid",
},
},
},
setupCallback: setupInvalidClientCAConfig,
expectError: true,
@@ -334,9 +351,13 @@ func TestServerInitializationAndStartup(t *testing.T) {
},
{
name: "Attested TLS Server Startup",
config: server.Config{
Host: "localhost",
Port: "0",
config: server.AgentConfig{
ServerConfig: server.ServerConfig{
BaseConfig: server.BaseConfig{
Host: "localhost",
Port: "0",
},
},
AttestedTLS: true,
},
expectedLog: "TestServer service gRPC server listening at localhost:0 with Attested TLS",
@@ -347,7 +368,6 @@ func TestServerInitializationAndStartup(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if tc.setupCallback != nil {
tc.setupCallback(t, &tc.config, nil)
}
@@ -358,7 +378,6 @@ func TestServerInitializationAndStartup(t *testing.T) {
authSvc := new(authmocks.Authenticator)
srv := New(ctx, cancel, "TestServer", tc.config, func(srv *grpc.Server) {}, logger, qp, authSvc)
var wg sync.WaitGroup
wg.Add(1)
@@ -390,7 +409,7 @@ func TestServerInitializationAndStartup(t *testing.T) {
}
}
func setupTLSConfig(t *testing.T, config *server.Config, _ *ThreadSafeBuffer) {
func setupTLSConfig(t *testing.T, config *server.AgentConfig, _ *ThreadSafeBuffer) {
cert, key, err := generateSelfSignedCert()
assert.NoError(t, err)
@@ -398,7 +417,7 @@ func setupTLSConfig(t *testing.T, config *server.Config, _ *ThreadSafeBuffer) {
config.KeyFile = string(key)
}
func setupMTLSConfig(t *testing.T, config *server.Config, _ *ThreadSafeBuffer) {
func setupMTLSConfig(t *testing.T, config *server.AgentConfig, _ *ThreadSafeBuffer) {
cert, key, err := generateSelfSignedCert()
assert.NoError(t, err)
@@ -408,7 +427,7 @@ func setupMTLSConfig(t *testing.T, config *server.Config, _ *ThreadSafeBuffer) {
config.ClientCAFile = string(cert)
}
func setupInvalidRootCAConfig(t *testing.T, config *server.Config, _ *ThreadSafeBuffer) {
func setupInvalidRootCAConfig(t *testing.T, config *server.AgentConfig, _ *ThreadSafeBuffer) {
cert, key, err := generateSelfSignedCert()
assert.NoError(t, err)
@@ -418,7 +437,7 @@ func setupInvalidRootCAConfig(t *testing.T, config *server.Config, _ *ThreadSafe
config.ClientCAFile = string(cert)
}
func setupInvalidClientCAConfig(t *testing.T, config *server.Config, _ *ThreadSafeBuffer) {
func setupInvalidClientCAConfig(t *testing.T, config *server.AgentConfig, _ *ThreadSafeBuffer) {
cert, key, err := generateSelfSignedCert()
assert.NoError(t, err)
@@ -427,3 +446,89 @@ func setupInvalidClientCAConfig(t *testing.T, config *server.Config, _ *ThreadSa
config.ClientCAFile = "invalid"
config.ServerCAFile = string(cert)
}
func createCertificatesFiles() (string, string, string, error) {
caKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return "", "", "", err
}
caTemplate := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
Organization: []string{"Test Org"},
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour * 24),
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
BasicConstraintsValid: true,
IsCA: true,
}
caCertDER, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey)
if err != nil {
return "", "", "", err
}
caCertFile, err := createTempFile(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caCertDER}))
if err != nil {
return "", "", "", err
}
clientKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return "", "", "", err
}
clientTemplate := x509.Certificate{
SerialNumber: big.NewInt(2),
Subject: pkix.Name{
Organization: []string{"Test Org"},
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour * 24),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
BasicConstraintsValid: true,
}
clientCertDER, err := x509.CreateCertificate(rand.Reader, &clientTemplate, &caTemplate, &clientKey.PublicKey, caKey)
if err != nil {
return "", "", "", err
}
clientCertFile, err := createTempFile(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: clientCertDER}))
if err != nil {
return "", "", "", err
}
clientKeyFile, err := createTempFile(pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(clientKey)}))
if err != nil {
return "", "", "", err
}
return caCertFile, clientCertFile, clientKeyFile, nil
}
func createTempFile(data []byte) (string, error) {
file, err := createTempFileHandle()
if err != nil {
return "", err
}
_, err = file.Write(data)
if err != nil {
return "", err
}
err = file.Close()
if err != nil {
return "", err
}
return file.Name(), nil
}
func createTempFileHandle() (*os.File, error) {
return os.CreateTemp("", "test")
}
+25 -6
View File
@@ -16,14 +16,25 @@ type Server interface {
Stop() error
}
type Config struct {
Host string `env:"HOST" envDefault:""`
Port string `env:"PORT" envDefault:""`
type ServerConfiguration interface {
GetBaseConfig() ServerConfig
}
type BaseConfig struct {
Host string `env:"HOST" envDefault:"localhost"`
Port string `env:"PORT" envDefault:"7001"`
ServerCAFile string `env:"SERVER_CA_CERTS" envDefault:""`
CertFile string `env:"SERVER_CERT" envDefault:""`
KeyFile string `env:"SERVER_KEY" envDefault:""`
ServerCAFile string `env:"SERVER_CA_CERTS" envDefault:""`
ClientCAFile string `env:"CLIENT_CA_CERTS" envDefault:""`
AttestedTLS bool `env:"ATTESTED_TLS" envDefault:"false"`
}
type ServerConfig struct {
BaseConfig
}
type AgentConfig struct {
ServerConfig
AttestedTLS bool `env:"ATTESTED_TLS" envDefault:"false"`
}
type BaseServer struct {
@@ -31,11 +42,19 @@ type BaseServer struct {
Cancel context.CancelFunc
Name string
Address string
Config Config
Config ServerConfiguration
Logger *slog.Logger
Protocol string
}
func (s ServerConfig) GetBaseConfig() ServerConfig {
return s
}
func (a AgentConfig) GetBaseConfig() ServerConfig {
return a.ServerConfig
}
func stopAllServer(servers ...Server) error {
var errs []error
for _, server := range servers {