NOISSUE - Add TDX support to Manager (#446)

* Add TDX support on manager

* Add functions to check platform

* Search for tdx in kernel parameters

* Modify based on comments

cmd/manager/main.go

@@ -84,8 +84,24 @@ func main() {
 	}()
 	tracer := tp.Tracer(svcName)
 
-	qemuCfg := qemu.Config{}
-	if err := env.ParseWithOptions(&qemuCfg, env.Options{Prefix: envPrefixQemu}); err != nil {
+	qemuCfg, err := qemu.NewConfig()
+	if err != nil {
+		logger.Error(fmt.Sprintf("failed to create config: %v", err))
+		exitCode = 1
+		return
+	}
+
+	if qemuCfg.EnableTDX {
+		logger.Info("Manager started with TDX enabled")
+	} else if qemuCfg.EnableSEVSNP {
+		logger.Info("Manager started with SEV-SNP enabled")
+	} else if qemuCfg.EnableSEV {
+		logger.Info("Manager started with SEV enabled")
+	} else {
+		logger.Info("Manager started without confidential computing support")
+	}
+
+	if err := env.ParseWithOptions(qemuCfg, env.Options{Prefix: envPrefixQemu}); err != nil {
 		logger.Error(fmt.Sprintf("failed to load QEMU configuration: %s", err))
 		exitCode = 1
 		return
@@ -100,7 +116,7 @@ func main() {
 		return
 	}
 
-	svc, err := newService(ctx, logger, tracer, qemuCfg, cfg.AttestationPolicyBinary, cfg.IgvmMeasureBinary, cfg.PcrValues, cfg.EosVersion)
+	svc, err := newService(ctx, logger, tracer, *qemuCfg, cfg.AttestationPolicyBinary, cfg.IgvmMeasureBinary, cfg.PcrValues, cfg.EosVersion)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

cocos-manager.env

@@ -44,6 +44,9 @@ MANAGER_QEMU_SEV_ID=sev0
 MANAGER_QEMU_SEV_CBITPOS=51
 MANAGER_QEMU_SEV_REDUCED_PHYS_BITS=1
 MANAGER_QEMU_HOST_DATA=
+MANAGER_QEMU_TDX_ID=tdx0
+MANAGER_QEMU_QUOTE_GENERATION_PORT=4050
+MANAGER_QEMU_OVMF_FILE=/usr/share/ovmf/OVMF.fd
 MANAGER_QEMU_VSOCK_ID=vhost-vsock-pci0
 MANAGER_QEMU_VSOCK_GUEST_CID=3
 MANAGER_QEMU_VSOCK_VNC=0
@@ -51,6 +54,7 @@ MANAGER_QEMU_BIN_PATH=qemu-system-x86_64
 MANAGER_QEMU_USE_SUDO=true
 MANAGER_QEMU_ENABLE_SEV=false
 MANAGER_QEMU_ENABLE_SEV_SNP=false
+MANAGER_QEMU_ENABLE_TDX=false
 MANAGER_QEMU_IGVM_FILE=/etc/cocos/coconut-qemu.igvm
 MANAGER_QEMU_ENABLE_KVM=true
 MANAGER_QEMU_MACHINE=q35

go.mod

@@ -27,6 +27,7 @@ require (
 
 require (
 	cloud.google.com/go/storage v1.51.0
+	github.com/caarlos0/env/v10 v10.0.0
 	github.com/golang-jwt/jwt/v5 v5.2.2
 	github.com/google/gce-tcb-verifier v0.3.1
 )

go.sum

@@ -40,6 +40,8 @@ github.com/absmach/magistrala v0.15.1 h1:3Bk2hlyWcV591LxPYwlvRcyCXTfuZ1g/EkNmU+o
 github.com/absmach/magistrala v0.15.1/go.mod h1:9pto6xuBt/IuCtZRdEha0iDQKNQ5tyNOjLXJgUiikYk=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA=
+github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18=
 github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA=
 github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=

manager/README.md

@@ -48,6 +48,9 @@ The service is configured using the environment variables from the following tab
 | MANAGER_QEMU_SEV_REDUCED_PHYS_BITS         | The number of reduced physical address bits for SEV.                                                             | 1                              |
 | MANAGER_QEMU_ENABLE_HOST_DATA              | Enable additional data for the SEV host.                                                                         | false                          |
 | MANAGER_QEMU_HOST_DATA                     | Additional data for the SEV host.                                                                                |                                |
+| MANAGER_QEMU_TDX_ID                        | The ID for the Trust Domain Extensions (TDX) device.                                                             | tdx0                           |
+| MANAGER_QEMU_QUOTE_GENERATION_PORT         | The port number for virtual socket used to communicate with the Quote Generation Service (QGS).                  | 4050                           |
+| MANAGER_QEMU_OVMF_FILE                     | The file path for the OVMF file (combined OVMF_CODE and OVMF_VARS file).                                         | /usr/share/ovmf/OVMF.fd        |
 | MANAGER_QEMU_IGVM_ID                       | The ID of the IGVM file.                                                                                         | igvm0                          |
 | MANAGER_QEMU_IGVM_FILE                     | The file path to the IGVM file.                                                                                  | /root/coconut-qemu.igvm        |
 | MANAGER_QEMU_VSOCK_ID                      | The ID for the virtual socket device.                                                                            | vhost-vsock-pci0               |
@@ -57,6 +60,7 @@ The service is configured using the environment variables from the following tab
 | MANAGER_QEMU_USE_SUDO                      | Whether to use sudo to run QEMU.                                                                                 | false                          |
 | MANAGER_QEMU_ENABLE_SEV                    | Whether to enable Secure Encrypted Virtualization (SEV).                                                         | false                          |
 | MANAGER_QEMU_ENABLE_SEV_SNP                | Whether to enable Secure Nested Paging (SEV-SNP).                                                                | true                           |
+| MANAGER_QEMU_ENABLE_TDX                    | Whether to enable Trust Domain Extensions (TDX).                                                                 | false                          |
 | MANAGER_QEMU_ENABLE_KVM                    | Whether to enable the Kernel-based Virtual Machine (KVM) acceleration.                                           | true                           |
 | MANAGER_QEMU_MACHINE                       | The machine type for QEMU.                                                                                       | q35                            |
 | MANAGER_QEMU_CPU                           | The CPU model for QEMU.                                                                                          | EPYC                           |
@@ -268,6 +272,20 @@ MANAGER_QEMU_IGVM_FILE=<path to IGVM file> \
 ./build/cocos-manager
 ```
 
+To enable [TDX](https://www.intel.com/content/www/us/en/developer/tools/trust-domain-extensions/overview.html) support, start manager like this
+
+```sh
+MANAGER_GRPC_URL=localhost:7001 \
+MANAGER_LOG_LEVEL=debug \
+MANAGER_QEMU_ENABLE_SEV=false \
+MANAGER_QEMU_ENABLE_SEV_SNP=false \
+MANAGER_QEMU_ENABLE_TDX=true \
+MANAGER_QEMU_CPU=host \
+MANAGER_QEMU_BIN_PATH=<path to QEMU binary> \
+MANAGER_QEMU_OVMF_FILE=<path to OVMF file> \
+./build/cocos-manager
+```
+
 ### Troubleshooting
 
 If the `ps aux | grep qemu-system-x86_64` give you something like this

manager/attestation_policy.go

@@ -110,8 +110,8 @@ func (ms *managerService) FetchAttestationPolicy(_ context.Context, computationI
 		attestationPolicy.Config.Policy.Measurement = measurement
 	}
 
-	if vmi.Config.SevConfig.EnableHostData {
-		hostData, err := base64.StdEncoding.DecodeString(vmi.Config.SevConfig.HostData)
+	if vmi.Config.SEVConfig.EnableHostData {
+		hostData, err := base64.StdEncoding.DecodeString(vmi.Config.SEVConfig.HostData)
 		if err != nil {
 			return nil, err
 		}

manager/qemu/config.go

@@ -5,11 +5,14 @@ package qemu
 import (
 	"fmt"
 	"strconv"
+
+	"github.com/caarlos0/env/v10"
 )
 
 const (
 	BaseGuestCID      = 3
 	KernelCommandLine = "quiet console=null"
+	TDXObject         = "'{\"qom-type\":\"tdx-guest\",\"id\":\"%s\",\"quote-generation-socket\":{\"type\": \"vsock\", \"cid\":\"2\",\"port\":\"%d\"}}'"
 )
 
 type MemoryConfig struct {
@@ -52,7 +55,7 @@ type DiskImgConfig struct {
 	RootFsFile string `env:"DISK_IMG_ROOTFS_FILE" envDefault:"img/rootfs.cpio.gz"`
 }
 
-type SevConfig struct {
+type SEVConfig struct {
 	ID              string `env:"SEV_ID"                envDefault:"sev0"`
 	CBitPos         int    `env:"SEV_CBITPOS"           envDefault:"51"`
 	ReducedPhysBits int    `env:"SEV_REDUCED_PHYS_BITS" envDefault:"1"`
@@ -60,6 +63,12 @@ type SevConfig struct {
 	HostData        string `env:"HOST_DATA"             envDefault:""`
 }
 
+type TDXConfig struct {
+	ID                  string `env:"TDX_ID"                  envDefault:"tdx0"`
+	QuoteGenerationPort int    `env:"QUOTE_GENERATION_PORT"   envDefault:"4050"`
+	OVMF                string `env:"OVMF_FILE"               envDefault:"/usr/share/ovmf/OVMF.fd"`
+}
+
 type IGVMConfig struct {
 	ID   string `env:"IGVM_ID"        envDefault:"igvm0"`
 	File string `env:"IGVM_FILE"      envDefault:"/root/coconut-qemu.igvm"`
@@ -71,10 +80,11 @@ type VSockConfig struct {
 }
 
 type Config struct {
+	EnableSEV    bool
+	EnableSEVSNP bool
+	EnableTDX    bool
 	QemuBinPath  string `env:"BIN_PATH"       envDefault:"qemu-system-x86_64"`
 	UseSudo      bool   `env:"USE_SUDO"       envDefault:"false"`
-	EnableSEV    bool   `env:"ENABLE_SEV"     envDefault:"false"`
-	EnableSEVSNP bool   `env:"ENABLE_SEV_SNP" envDefault:"true"`
 
 	EnableKVM bool `env:"ENABLE_KVM" envDefault:"true"`
 
@@ -101,7 +111,10 @@ type Config struct {
 	DiskImgConfig
 
 	// SEV
-	SevConfig
+	SEVConfig
+
+	// TDX
+	TDXConfig
 
 	// vTPM
 	IGVMConfig
@@ -142,7 +155,7 @@ func (config Config) ConstructQemuArgs() []string {
 		config.MemoryConfig.Slots,
 		config.MemoryConfig.Max))
 
-	if !config.EnableSEVSNP {
+	if !config.EnableSEVSNP && !config.EnableTDX {
 		// OVMF
 		args = append(args, "-drive",
 			fmt.Sprintf("if=%s,format=%s,unit=%d,file=%s,readonly=%s",
@@ -183,15 +196,15 @@ func (config Config) ConstructQemuArgs() []string {
 
 		args = append(args, "-machine",
 			fmt.Sprintf("confidential-guest-support=%s,memory-backend=%s,igvm-cfg=%s",
-				config.SevConfig.ID,
+				config.SEVConfig.ID,
 				config.MemID,
 				config.IGVMConfig.ID))
 
 		if config.EnableSEVSNP {
 			sevType = "sev-snp-guest"
 
-			if config.SevConfig.EnableHostData {
-				hostData = fmt.Sprintf(",host-data=%s", config.SevConfig.HostData)
+			if config.SEVConfig.EnableHostData {
+				hostData = fmt.Sprintf(",host-data=%s", config.SEVConfig.HostData)
 			}
 		}
 
@@ -203,9 +216,9 @@ func (config Config) ConstructQemuArgs() []string {
 		args = append(args, "-object",
 			fmt.Sprintf("%s,id=%s,cbitpos=%d,reduced-phys-bits=%d%s",
 				sevType,
-				config.SevConfig.ID,
-				config.SevConfig.CBitPos,
-				config.SevConfig.ReducedPhysBits,
+				config.SEVConfig.ID,
+				config.SEVConfig.CBitPos,
+				config.SEVConfig.ReducedPhysBits,
 				hostData))
 
 		args = append(args, "-object",
@@ -214,6 +227,26 @@ func (config Config) ConstructQemuArgs() []string {
 				config.IGVMConfig.File))
 	}
 
+	if config.EnableTDX {
+		args = append(args, "-object",
+			fmt.Sprintf(TDXObject,
+				config.TDXConfig.ID,
+				config.TDXConfig.QuoteGenerationPort))
+
+		args = append(args, "-machine",
+			fmt.Sprintf("confidential-guest-support=%s,memory-backend=%s,hpet=off",
+				config.TDXConfig.ID,
+				config.MemID))
+
+		args = append(args, "-object",
+			fmt.Sprintf("memory-backend-memfd,id=%s,size=%s,share=true,prealloc=false",
+				config.MemID,
+				config.MemoryConfig.Size))
+
+		args = append(args, "-bios", config.TDXConfig.OVMF)
+		args = append(args, "-nodefaults")
+	}
+
 	args = append(args, "-kernel", config.DiskImgConfig.KernelFile)
 	args = append(args, "-append", strconv.Quote(KernelCommandLine))
 	args = append(args, "-initrd", config.DiskImgConfig.RootFsFile)
@@ -237,3 +270,17 @@ func (config Config) ConstructQemuArgs() []string {
 
 	return args
 }
+
+func NewConfig() (*Config, error) {
+	cfg := Config{}
+
+	if err := env.Parse(&cfg); err != nil {
+		return nil, err
+	}
+
+	cfg.EnableSEV = SEVEnabledOnHost()
+	cfg.EnableSEVSNP = SEVSNPEnabledOnHost()
+	cfg.EnableTDX = TDXEnabledOnHost()
+
+	return &cfg, nil
+}

manager/qemu/config_test.go

@@ -127,7 +127,7 @@ func TestConstructQemuArgs(t *testing.T) {
 					KernelFile: "img/bzImage",
 					RootFsFile: "img/rootfs.cpio.gz",
 				},
-				SevConfig: SevConfig{
+				SEVConfig: SEVConfig{
 					ID:              "sev0",
 					CBitPos:         51,
 					ReducedPhysBits: 1,
@@ -174,7 +174,7 @@ func TestConstructQemuArgs(t *testing.T) {
 func TestConstructQemuArgs_HostData(t *testing.T) {
 	config := Config{
 		EnableSEVSNP: true,
-		SevConfig: SevConfig{
+		SEVConfig: SEVConfig{
 			ID:              "sev0",
 			CBitPos:         51,
 			ReducedPhysBits: 1,

manager/qemu/vm.go

@@ -7,6 +7,7 @@ import (
 	"log/slog"
 	"os"
 	"os/exec"
+	"strings"
 	"syscall"
 	"time"
 
@@ -60,7 +61,8 @@ func (v *qemuVM) Start() (err error) {
 	}
 
 	v.vmi.Config.NetDevConfig.ID = fmt.Sprintf("%s-%s", v.vmi.Config.NetDevConfig.ID, id)
-	v.vmi.Config.SevConfig.ID = fmt.Sprintf("%s-%s", v.vmi.Config.SevConfig.ID, id)
+	v.vmi.Config.SEVConfig.ID = fmt.Sprintf("%s-%s", v.vmi.Config.SEVConfig.ID, id)
+	v.vmi.Config.TDXConfig.ID = fmt.Sprintf("%s-%s", v.vmi.Config.TDXConfig.ID, id)
 
 	if !v.vmi.Config.EnableSEVSNP {
 		// Copy firmware vars file.
@@ -197,3 +199,54 @@ func (v *qemuVM) GetCID() int {
 func (v *qemuVM) GetConfig() interface{} {
 	return v.vmi
 }
+
+func SEVEnabled(cpuinfo string, sevPresent bool) bool {
+	return strings.Contains(cpuinfo, "sev") && sevPresent
+}
+
+func SEVSNPEnabled(cpuinfo, kernelParam string) bool {
+	return strings.Contains(cpuinfo, "sev_snp") && strings.TrimSpace(kernelParam) == "1"
+}
+
+func TDXEnabled(cpuinfo, kernelParam string) bool {
+	return strings.Contains(cpuinfo, "tdx_host_platform") && strings.TrimSpace(kernelParam) == "1"
+}
+
+// Checks if SEV is supported and usable by verifying both CPU flags and the /dev/sev device.
+func SEVEnabledOnHost() bool {
+	cpuinfo, err := os.ReadFile("/proc/cpuinfo")
+	if err != nil {
+		return false
+	}
+
+	_, err = os.Stat("/dev/sev")
+	return SEVEnabled(string(cpuinfo), err == nil)
+}
+
+func SEVSNPEnabledOnHost() bool {
+	cpuinfo, err := os.ReadFile("/proc/cpuinfo")
+	if err != nil {
+		return false
+	}
+
+	kernelParam, err := os.ReadFile("/sys/module/kvm_amd/parameters/sev_snp")
+	if err != nil {
+		return false
+	}
+
+	return SEVSNPEnabled(string(cpuinfo), string(kernelParam))
+}
+
+func TDXEnabledOnHost() bool {
+	cpuinfo, err := os.ReadFile("/proc/cpuinfo")
+	if err != nil {
+		return false
+	}
+
+	kernelParam, err := os.ReadFile("/sys/module/kvm_intel/parameters/tdx")
+	if err != nil {
+		return false
+	}
+
+	return TDXEnabled(string(cpuinfo), string(kernelParam))
+}

manager/qemu/vm_test.go

@@ -161,3 +161,45 @@ func TestGetConfig(t *testing.T) {
 	config := vm.GetConfig()
 	assert.Equal(t, expectedConfig, config)
 }
+
+func TestSEVEnabled(t *testing.T) {
+	t.Run("cpuinfo contains sev and device exists", func(t *testing.T) {
+		assert.True(t, SEVEnabled("flags: xyz sev abc", true))
+	})
+
+	t.Run("cpuinfo missing sev", func(t *testing.T) {
+		assert.False(t, SEVEnabled("flags: xyz abc", true))
+	})
+
+	t.Run("device does not exist", func(t *testing.T) {
+		assert.False(t, SEVEnabled("flags: sev abc", false))
+	})
+}
+
+func TestSEVSNPEnabled(t *testing.T) {
+	t.Run("cpuinfo and kvm param correct", func(t *testing.T) {
+		assert.True(t, SEVSNPEnabled("flags: sev_snp abc", "1"))
+	})
+
+	t.Run("missing sev_snp in cpuinfo", func(t *testing.T) {
+		assert.False(t, SEVSNPEnabled("flags: abc", "1"))
+	})
+
+	t.Run("kernel param not enabled", func(t *testing.T) {
+		assert.False(t, SEVSNPEnabled("flags: sev_snp", "0"))
+	})
+}
+
+func TestTDXEnabled(t *testing.T) {
+	t.Run("cpuinfo and kvm param correct", func(t *testing.T) {
+		assert.True(t, TDXEnabled("flags: tdx_host_platform abc", "1"))
+	})
+
+	t.Run("missing tdx_host_platform in cpuinfo", func(t *testing.T) {
+		assert.False(t, TDXEnabled("flags: abc", "1"))
+	})
+
+	t.Run("kernel param not enabled", func(t *testing.T) {
+		assert.False(t, TDXEnabled("flags: tdx_host_platform", "0"))
+	})
+}

manager/service.go

@@ -218,7 +218,7 @@ func (ms *managerService) CreateVM(ctx context.Context, req *CreateReq) (string,
 	if cfg.Config.EnableSEVSNP {
 		todo := sha3.Sum256([]byte("TODO"))
 		// Define host-data value of QEMU for SEV-SNP, with a base64 encoding of the computation hash.
-		cfg.Config.SevConfig.HostData = base64.StdEncoding.EncodeToString(todo[:])
+		cfg.Config.SEVConfig.HostData = base64.StdEncoding.EncodeToString(todo[:])
 	}
 
 	cvm := ms.vmFactory(cfg, id, ms.logger)