diff --git a/agent/service.go b/agent/service.go index d1eaa5db..71872858 100644 --- a/agent/service.go +++ b/agent/service.go @@ -85,6 +85,29 @@ var ( ImaPcrIndex = 10 ) +func ensureDir(path string, mode os.FileMode) error { + info, err := os.Stat(path) + switch { + case err == nil: + if info.IsDir() { + return nil + } + if err := os.Remove(path); err != nil { + return fmt.Errorf("removing non-directory path %q: %w", path, err) + } + case os.IsNotExist(err): + // Continue and create it below. + default: + return fmt.Errorf("stating path %q: %w", path, err) + } + + if err := os.MkdirAll(path, mode); err != nil { + return fmt.Errorf("creating directory %q: %w", path, err) + } + + return nil +} + var ( // ErrMalformedEntity indicates malformed entity specification (e.g. // invalid username or password). @@ -478,8 +501,8 @@ func (as *agentService) downloadAlgorithmIfRemote(state statemachine.State) { as.algoReceived = true as.algoRequirements = res.Requirements // Store requirements for installation - // Create datasets directory - if err := os.Mkdir(algorithm.DatasetsDir, 0o755); err != nil { + // The initramfs may have already provisioned /cocos/datasets. + if err := ensureDir(algorithm.DatasetsDir, 0o755); err != nil { as.runError = fmt.Errorf("error creating datasets directory: %w", err) as.logger.Error(as.runError.Error()) as.sm.SendEvent(RunFailed) @@ -976,7 +999,7 @@ func (as *agentService) Algo(ctx context.Context, algo Algorithm) error { as.algoRequirements = algo.Requirements as.algoReceived = true - if err := os.Mkdir(algorithm.DatasetsDir, 0o755); err != nil { + if err := ensureDir(algorithm.DatasetsDir, 0o755); err != nil { return fmt.Errorf("error creating datasets directory: %v", err) } @@ -1145,7 +1168,7 @@ func (as *agentService) runComputation(state statemachine.State) { } }() - if err := os.Mkdir(algorithm.ResultsDir, 0o755); err != nil { + if err := ensureDir(algorithm.ResultsDir, 0o755); err != nil { as.mu.Lock() as.runError = fmt.Errorf("error creating results directory: %s", err.Error()) as.mu.Unlock() diff --git a/hal/disk/Config.in b/hal/disk/Config.in new file mode 100644 index 00000000..6cc33f89 --- /dev/null +++ b/hal/disk/Config.in @@ -0,0 +1,9 @@ +source "$BR2_EXTERNAL_COCOS_PATH/package/agent/Config.in" +source "$BR2_EXTERNAL_COCOS_PATH/package/attestation-service/Config.in" +source "$BR2_EXTERNAL_COCOS_PATH/package/cc-attestation-agent/Config.in" +source "$BR2_EXTERNAL_COCOS_PATH/package/coco-keyprovider/Config.in" +source "$BR2_EXTERNAL_COCOS_PATH/package/wasmedge/Config.in" +source "$BR2_EXTERNAL_COCOS_PATH/package/log-forwarder/Config.in" +source "$BR2_EXTERNAL_COCOS_PATH/package/computation-runner/Config.in" +source "$BR2_EXTERNAL_COCOS_PATH/package/egress-proxy/Config.in" +source "$BR2_EXTERNAL_COCOS_PATH/package/ingress-proxy/Config.in" diff --git a/hal/disk/README.md b/hal/disk/README.md new file mode 100644 index 00000000..14cc6e7d --- /dev/null +++ b/hal/disk/README.md @@ -0,0 +1,214 @@ +# Disk Image Workflow + +This directory is the Buildroot external tree for the current Cocos disk test +VM image and its runtime configuration. + +## Layout + +- [configs/cocos_defconfig](./configs/cocos_defconfig): + Buildroot configuration for the bootable image. +- [board/rootfs-overlay/init](./board/rootfs-overlay/init): + early initramfs script that provisions `/cocos`, mounts the real root, and + switches into the installed system. +- [board/cocos/genimage.cfg](./board/cocos/genimage.cfg): + GPT disk layout for the final `disk.img`. +- [board/cocos/post-image.sh](./board/cocos/post-image.sh): + builds the minimal initramfs, stages EFI files, signs boot artifacts, and + assembles `disk.img`. +- [external.desc](./external.desc): Buildroot external tree descriptor. +- [external.mk](./external.mk): includes package makefiles from `package/*`. + +## Current Buildroot Image + +The current Buildroot flow produces a bootable GPT disk image: + +- `efi` partition: FAT EFI system partition with GRUB, kernel, and initramfs +- `root` partition: ext4 root filesystem protected by dm-verity +- `verity` partition: dm-verity hash tree for the root filesystem +- `cocos` partition: blank partition provisioned at boot as an encrypted ext4 + filesystem mounted at `/cocos` + +The final image is written to: + +```bash +output/images/disk.img +``` + +The root filesystem image is also available separately as: + +```bash +output/images/rootfs.ext4 +``` + +## Current Boot Flow + +At boot, GRUB loads: + +- `bzImage` +- `initrd.cpio.gz` + +The initramfs script in +[board/rootfs-overlay/init](./board/rootfs-overlay/init) +then: + +1. mounts `/proc`, `/sys`, `devtmpfs`, and `devpts` +2. assumes the boot disk is `/dev/sda` +3. opens a dm-verity mapping for the root filesystem using: + - `/dev/sda2` as the data partition + - `/dev/sda3` as the verity hash partition + - `roothash=` from the kernel command line +4. mounts `/dev/mapper/root_verity` read-only at `/root` +5. generates a fresh ephemeral key +6. formats `/dev/sda4` as LUKS2 +7. opens it as `/dev/mapper/cocos_crypt` +8. formats that mapper as ext4 and mounts it at `/root/cocos` +9. creates working directories on `/cocos`, including: + - `/cocos/.cache/oci` + - `/cocos/datasets` + - `/cocos/docker` + - `/cocos/cocos_init` +10. mounts `tmpfs` on `/tmp` and `/var` because the root filesystem is + intentionally read-only +11. bind-mounts `/cocos/docker` onto `/var/lib/docker` +12. bind-mounts `/cocos/cocos_init` onto `/cocos_init` +13. rewrites `/etc/fstab` in the mounted root to describe the live runtime +14. preserves or adds 9P mounts for: + - `certs_share` -> `/etc/certs` + - `env_share` -> `/etc/cocos` +15. securely wipes the temporary LUKS key file +16. runs `switch_root /root /sbin/init` + +Important details: + +- the root filesystem is verified through dm-verity before it is mounted +- `/cocos` is encrypted with an ephemeral per-boot key +- that key is not persisted, so `/cocos` is provisioned fresh on each boot + +## Runtime Filesystem Model + +The running system is split into: + +- read-only root on `/` +- encrypted writable storage on `/cocos` +- `tmpfs` on `/tmp` +- `tmpfs` on `/var` + +Service state that must survive within a boot session is redirected away from +the read-only root: + +- Docker data lives on `/cocos/docker` +- agent setup scripts work through `/cocos_init`, which is backed by + `/cocos/cocos_init` +- algorithm datasets and results live under `/cocos` + +This means services can use `/cocos` like a regular directory tree after boot, +even though it is backed by an encrypted mapper created in early userspace. + +## systemd Runtime Expectations + +Several services depend on files mounted from 9P shares under `/etc/certs` and +`/etc/cocos`. To avoid boot-order races, the rootfs overlay includes systemd +drop-ins under: + +```bash +board/rootfs-overlay/usr/lib/systemd/system/*service.d/ +``` + +These drop-ins require the relevant mount points before starting services such +as: + +- `egress-proxy.service` +- `log-forwarder.service` +- `computation-runner.service` +- `cocos-agent.service` + +The overlay also ships tmpfiles rules in +[board/rootfs-overlay/usr/lib/tmpfiles.d/cocos.conf](./board/rootfs-overlay/usr/lib/tmpfiles.d/cocos.conf) +to create: + +- `/var/log/cocos` +- `/run/cocos` + +## Agent Packaging In Buildroot + +The Buildroot `agent` package is wired to build the binary from the local Cocos +checkout, not only from a downloaded release snapshot. The package definition is +in [package/agent/agent.mk](./package/agent/agent.mk). + +That package currently: + +- builds `cocos-agent` from the local source tree +- installs the local + [cocos-agent.service](../../init/systemd/cocos-agent.service) +- installs the local + [agent_setup.sh](../../init/systemd/agent_setup.sh) +- installs the local + [agent_start_script.sh](../../init/systemd/agent_start_script.sh) + +So changes under: + +- `cocos/agent/...` +- `cocos/init/systemd/...` + +are intended to be picked up by the next Buildroot rebuild. + +## Buildroot Packages And Tools + +The current `cocos_defconfig` includes the components needed by the boot flow +and runtime image, including: + +- systemd +- DHCP client +- `cryptsetup` +- `eudev` +- `e2fsprogs` +- Docker, containerd, and runc +- `skopeo` +- TPM2 tools +- 9P filesystem support +- GRUB2 EFI boot support +- host `genimage` + +The initramfs built in `post-image.sh` is intentionally minimal and contains +only the binaries needed for early boot, dm-verity root verification, and +`/cocos` provisioning. + +## Secure Boot Notes + +During `post-image.sh`: + +- GRUB is rebuilt with `--disable-shim-lock` +- `bootx64.efi` and `bzImage` are signed with the configured Secure Boot keys + when those keys are present + +This flow is designed for booting directly through OVMF with your own enrolled +keys. It does not currently rely on booting through `shim`. + +## Rebuilding + +This directory is meant to be used as a Buildroot external tree. From this +directory, configure a Buildroot checkout with: + +```bash +make -C /path/to/buildroot BR2_EXTERNAL=$PWD cocos_defconfig +``` + +Then build with: + +```bash +make -C /path/to/buildroot BR2_EXTERNAL=$PWD -j$(nproc) +``` + +The resulting boot image is: + +```bash +/path/to/buildroot/output/images/disk.img +``` + +Additional generated artifacts include: + +```bash +/path/to/buildroot/output/images/rootfs.ext4 +/path/to/buildroot/output/images/rootfs.verity +/path/to/buildroot/output/images/rootfs.roothash +``` diff --git a/hal/disk/board/cocos/genimage.cfg b/hal/disk/board/cocos/genimage.cfg new file mode 100644 index 00000000..bf588504 --- /dev/null +++ b/hal/disk/board/cocos/genimage.cfg @@ -0,0 +1,42 @@ +image efi-part.vfat { + vfat { + file EFI { + image = "efi-part/EFI" + } + file bzImage { + image = "efi-part/bzImage" + } + file initrd.cpio.gz { + image = "efi-part/initrd.cpio.gz" + } + } + size = 256M +} + +image disk.img { + hdimage { + partition-table-type = "gpt" + } + + partition efi { + image = "efi-part.vfat" + partition-type-uuid = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" + offset = 1M + bootable = true + } + + partition root { + image = "rootfs.ext4" + partition-type-uuid = "0FC63DAF-8483-4772-8E79-3D69D8477DE4" + } + + partition verity { + image = "rootfs.verity" + partition-type-uuid = "0FC63DAF-8483-4772-8E79-3D69D8477DE4" + } + + partition cocos { + size = "20480M" + partition-type-uuid = "0FC63DAF-8483-4772-8E79-3D69D8477DE4" + } +} diff --git a/hal/disk/board/cocos/linux.config b/hal/disk/board/cocos/linux.config new file mode 100644 index 00000000..5f1a02ec --- /dev/null +++ b/hal/disk/board/cocos/linux.config @@ -0,0 +1,279 @@ +### +# Architecture / base +### +CONFIG_SYSVIPC=y +CONFIG_SMP=y +CONFIG_EXPERT=y +CONFIG_LOCALVERSION_AUTO=n + +### +# Modules +### +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y + +### +# Virtualization +### +CONFIG_HYPERVISOR_GUEST=y +CONFIG_PARAVIRT=y +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_KVM_SW_PROTECTED_VM=y +CONFIG_KVM_INTEL=y +CONFIG_VIRT_DRIVERS=y + +### +# Cgroups — base + Docker/container subsystems +### +CONFIG_CGROUPS=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_MISC=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_BPF=y +CONFIG_CGROUP_NET_PRIO=y +CONFIG_CGROUP_NET_CLASSID=y +CONFIG_CPUSETS=y +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y + +### +# Namespaces — required by containerd / runc +### +CONFIG_NAMESPACES=y +CONFIG_UTS_NS=y +CONFIG_IPC_NS=y +CONFIG_USER_NS=y +CONFIG_PID_NS=y +CONFIG_NET_NS=y + +### +# PCI +### +CONFIG_PCI=y +CONFIG_PCI_MSI=y +CONFIG_IRQ_REMAP=y + +### +# Initramfs +### +CONFIG_BLK_DEV_INITRD=y +CONFIG_RD_GZIP=y + +### +# Block devices +### +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_BLK_DEV_SD=y +CONFIG_SCSI_VIRTIO=y +CONFIG_ATA=y +CONFIG_ATA_PIIX=y +CONFIG_VIRTIO_BLK=y + +# Loop device (used by containerd image mounts) +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_LOOP_MIN_COUNT=8 + +### +# Device mapper — FDE, dm-verity, dm-crypt, dm-integrity +# These must be built-in (y) because they are needed before the +# rootfs is mounted, during the initramfs FDE init stage. +### +CONFIG_MD=y +CONFIG_BLK_DEV_DM_BUILTIN=y +CONFIG_BLK_DEV_DM=y +CONFIG_DM_CRYPT=y +CONFIG_DM_VERITY=y +CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG=y +# CONFIG_DM_VERITY_FEC is not set +CONFIG_DM_INTEGRITY=y +CONFIG_DM_INIT=y + +### +# Networking — base +### +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_INET=y +# CONFIG_WIRELESS is not set +CONFIG_NETDEVICES=y +CONFIG_VIRTIO_NET=y +CONFIG_NE2K_PCI=y +CONFIG_8139CP=y +# CONFIG_WLAN is not set +CONFIG_VSOCKETS=y +CONFIG_VIRTIO_VSOCKETS=y + +# Virtual Ethernet pairs and bridge (Docker networking) +CONFIG_VETH=m +CONFIG_BRIDGE=m +CONFIG_BRIDGE_NETFILTER=m + +### +# Netfilter — Docker NAT, iptables, conntrack (modules, loaded on demand) +### +CONFIG_NETFILTER=y +CONFIG_NETFILTER_ADVANCED=y +CONFIG_NF_CONNTRACK=m +CONFIG_NF_CONNTRACK_MARK=y +CONFIG_NF_NAT=m +CONFIG_NF_NAT_MASQUERADE=y +CONFIG_NF_TABLES=y +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m + +### +# BPF +### +CONFIG_BPF_SYSCALL=y + +### +# Filesystems +### +CONFIG_EXT4_FS=y +CONFIG_OVERLAY_FS=y +CONFIG_AUTOFS4_FS=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_PROC_FS=y +CONFIG_SYSFS=y + +### +# 9P filesystem (virtio shares for certs and env) +### +CONFIG_NET_9P=y +CONFIG_NET_9P_VIRTIO=y +CONFIG_9P_FS=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_9P_FS_SECURITY=y + +### +# Virtio devices +### +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VIRTIO_INPUT=y +CONFIG_VIRTIO_CONSOLE=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y +CONFIG_HW_RANDOM_VIRTIO=m + +### +# Console / Input +### +CONFIG_INPUT_EVDEV=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y + +### +# Kernel features required by systemd +### +CONFIG_FHANDLE=y +CONFIG_INOTIFY_USER=y +CONFIG_SIGNALFD=y +CONFIG_TIMERFD=y +CONFIG_EPOLL=y +CONFIG_POSIX_MQUEUE=y +CONFIG_POSIX_MQUEUE_SYSCTL=y +CONFIG_UNWINDER_FRAME_POINTER=y + +### +# Security +### +CONFIG_SECCOMP=y +CONFIG_SECCOMP_FILTER=y +CONFIG_SECURITY=y +CONFIG_SECURITYFS=y + +### +# EFI +### +CONFIG_EFI=y +CONFIG_EFI_STUB=y + +### +# AMD SEV-SNP +### +CONFIG_AMD_MEM_ENCRYPT=y +CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT=n +CONFIG_SEV_GUEST=y +CONFIG_IOMMU_DEFAULT_PASSTHROUGH=n + +### +# Intel TDX +### +CONFIG_X86_X2APIC=y +CONFIG_X86_CPUID=y +CONFIG_X86_SGX=y +CONFIG_X86_SGX_KVM=y +CONFIG_INTEL_TDX_GUEST=y +CONFIG_TDX_GUEST_DRIVER=y + +### +# Preemption (disabled for VM performance) +### +CONFIG_PREEMPT_COUNT=n +CONFIG_PREEMPT=n +CONFIG_PREEMPT_DYNAMIC=n +CONFIG_DEBUG_PREEMPT=n + +### +# Key/signature management +### +CONFIG_SYSTEM_TRUSTED_KEYS=n +CONFIG_SYSTEM_REVOCATION_KEYS=n +CONFIG_MODULE_SIG_KEY=n +CONFIG_KEYS=y +CONFIG_ENCRYPTED_KEYS=y + +### +# Crypto — AES-GCM (LUKS2 cipher) + SHA-256 (dm-verity hash) +### +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_GHASH=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_CRYPTO_ECHAINIV=y +CONFIG_CRYPTO_XTS=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_ESSIV=y +CONFIG_CRYPTO_USER_API=y +CONFIG_CRYPTO_USER_API_HASH=y +CONFIG_CRYPTO_USER_API_SKCIPHER=y +CONFIG_CRYPTO_USER_API_AEAD=y +CONFIG_CRYPTO_AES_NI_INTEL=m +CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL=m + +### +# TPM +### +CONFIG_TCG_TPM=y +CONFIG_TCG_TPM2_HMAC=y +CONFIG_TCG_PLATFORM=y + +### +# IMA (Linux Integrity Measurement Architecture) +### +CONFIG_INTEGRITY=y +CONFIG_INTEGRITY_SIGNATURE=y +CONFIG_IMA=y +CONFIG_IMA_MEASURE_PCR_IDX=10 +CONFIG_IMA_LSM_RULES=y +CONFIG_IMA_APPRAISE=y +CONFIG_IMA_DEFAULT_TEMPLATE="ima-ng" +CONFIG_IMA_DEFAULT_HASH="sha256" + +### +# Disabled options +### +CONFIG_KSM=n +CONFIG_EISA=n diff --git a/hal/disk/board/cocos/post-build.sh b/hal/disk/board/cocos/post-build.sh new file mode 100755 index 00000000..1fdf70ae --- /dev/null +++ b/hal/disk/board/cocos/post-build.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -u +set -e + +# Add a console on tty1 +if [ -e ${TARGET_DIR}/etc/inittab ]; then + grep -qE '^tty1::' ${TARGET_DIR}/etc/inittab || \ + sed -i '/GENERIC_SERIAL/a\ +tty1::respawn:/sbin/getty -L tty1 0 vt100 # QEMU graphical window' ${TARGET_DIR}/etc/inittab +fi diff --git a/hal/disk/board/cocos/post-image.sh b/hal/disk/board/cocos/post-image.sh new file mode 100755 index 00000000..1cd6b1f2 --- /dev/null +++ b/hal/disk/board/cocos/post-image.sh @@ -0,0 +1,273 @@ +#!/bin/bash + +COCOS_BOARD_DIR="$(dirname "$0")" +DEFCONFIG_NAME="$(basename "$2")" +README_FILES="${COCOS_BOARD_DIR}/readme.txt" +START_QEMU_SCRIPT="${BINARIES_DIR}/start-qemu.sh" + +# --------------------------------------------------------------------------- +# Build a minimal FDE initramfs (rootfs.cpio.gz) containing only the tools +# needed to mount the root partition read-only, provision LUKS2, and switch_root. +# All other packages live +# on the ext4 disk image and are available after switch_root. +# --------------------------------------------------------------------------- +echo "[post-image] Building minimal FDE initramfs..." + +INITRAMFS_STAGE="${BUILD_DIR}/initramfs-staging" +rm -rf "${INITRAMFS_STAGE}" + +# Merged-usr layout: bin/sbin/lib/lib64 are symlinks into usr/, matching the +# Buildroot target layout so that hardcoded ELF interpreter paths (ld-linux) +# and the #!/bin/sh shebang both resolve correctly inside the initramfs. +mkdir -p "${INITRAMFS_STAGE}/usr/bin" \ + "${INITRAMFS_STAGE}/usr/sbin" \ + "${INITRAMFS_STAGE}/usr/lib" \ + "${INITRAMFS_STAGE}/dev" \ + "${INITRAMFS_STAGE}/proc" \ + "${INITRAMFS_STAGE}/sys" \ + "${INITRAMFS_STAGE}/tmp" \ + "${INITRAMFS_STAGE}/run" \ + "${INITRAMFS_STAGE}/root" \ + "${INITRAMFS_STAGE}/etc/udev/rules.d" +ln -s usr/bin "${INITRAMFS_STAGE}/bin" +ln -s usr/sbin "${INITRAMFS_STAGE}/sbin" +ln -s usr/lib "${INITRAMFS_STAGE}/lib" +ln -s usr/lib "${INITRAMFS_STAGE}/lib64" + +# init script (PID 1) +install -m 0755 "${BR2_EXTERNAL_COCOS_PATH}/board/rootfs-overlay/init" \ + "${INITRAMFS_STAGE}/init" + +# Binaries required by the init script +FDE_BINS=" + bash + cryptsetup + veritysetup + mkfs.ext4 + mount + umount + losetup + switch_root + dd + shred + tr + cut + grep + awk + cat + ls + cp + mkdir + readlink + dirname + lsblk + udevadm + blkid + rm +" + +for BIN in ${FDE_BINS}; do + SRC="$(find "${TARGET_DIR}/usr/bin" "${TARGET_DIR}/usr/sbin" \ + "${TARGET_DIR}/bin" "${TARGET_DIR}/sbin" \ + -name "${BIN}" \( -type f -o -type l \) 2>/dev/null | head -1)" + if [ -n "${SRC}" ]; then + cp -P "${SRC}" "${INITRAMFS_STAGE}/usr/bin/${BIN}" + chmod 0755 "${INITRAMFS_STAGE}/usr/bin/${BIN}" 2>/dev/null || true + # If this is a symlink, also copy the resolved target binary (e.g. busybox, coreutils, mke2fs) + # so that other applet symlinks pointing to the same target also work at runtime. + if [ -L "${SRC}" ]; then + REAL_SRC="$(readlink -f "${SRC}")" + REAL_NAME="$(basename "${REAL_SRC}")" + if [ -f "${REAL_SRC}" ] && [ ! -e "${INITRAMFS_STAGE}/usr/bin/${REAL_NAME}" ]; then + cp "${REAL_SRC}" "${INITRAMFS_STAGE}/usr/bin/${REAL_NAME}" + chmod 0755 "${INITRAMFS_STAGE}/usr/bin/${REAL_NAME}" 2>/dev/null || true + fi + fi + else + echo "[post-image] WARNING: ${BIN} not found in target, skipping" + fi +done + +# sh symlink so #!/bin/sh in the init script resolves correctly +ln -sf bash "${INITRAMFS_STAGE}/usr/bin/sh" + +# Shared libraries from usr/lib (TARGET_DIR uses merged-usr so lib → usr/lib) +# Skip large runtimes that are only needed on the real root. +find "${TARGET_DIR}/usr/lib" \( \ + -path "*/python3*" -o \ + -path "*/gcc*" -o \ + -path "*/wasmedge*" \ + \) -prune -o \ + \( -name "*.so" -o -name "*.so.*" \) -print | while read -r LIB; do + REL="${LIB#${TARGET_DIR}/usr/lib/}" + DEST="${INITRAMFS_STAGE}/usr/lib/${REL}" + mkdir -p "$(dirname "${DEST}")" + cp -P "${LIB}" "${DEST}" +done + +# udev rules (needed for udevadm settle) +if [ -d "${TARGET_DIR}/etc/udev" ]; then + cp -a "${TARGET_DIR}/etc/udev/." "${INITRAMFS_STAGE}/etc/udev/" +fi + +# /dev seed nodes +mknod -m 0600 "${INITRAMFS_STAGE}/dev/console" c 5 1 2>/dev/null || true +mknod -m 0666 "${INITRAMFS_STAGE}/dev/null" c 1 3 2>/dev/null || true + +echo "[post-image] Packing initramfs..." +( cd "${INITRAMFS_STAGE}" && \ + find . | cpio --quiet -o -H newc -R 0:0 | gzip -9 \ + > "${BINARIES_DIR}/rootfs.cpio.gz" ) +echo "[post-image] rootfs.cpio.gz: $(du -sh "${BINARIES_DIR}/rootfs.cpio.gz" | cut -f1)" + +ROOTFS_IMAGE="${BINARIES_DIR}/rootfs.ext4" +VERITY_IMAGE="${BINARIES_DIR}/rootfs.verity" +ROOT_HASH_FILE="${BINARIES_DIR}/rootfs.roothash" +VERITYSETUP_BIN="${HOST_DIR}/bin/veritysetup" + +if [ ! -x "${VERITYSETUP_BIN}" ]; then + VERITYSETUP_BIN="${HOST_DIR}/sbin/veritysetup" +fi + +if [ ! -x "${VERITYSETUP_BIN}" ]; then + echo "[post-image] FATAL: host veritysetup not found at ${VERITYSETUP_BIN}" + exit 1 +fi + +echo "[post-image] Building dm-verity hash image..." +rm -f "${VERITY_IMAGE}" "${ROOT_HASH_FILE}" +truncate -s 256M "${VERITY_IMAGE}" +VERITY_FORMAT_OUTPUT="$("${VERITYSETUP_BIN}" format "${ROOTFS_IMAGE}" "${VERITY_IMAGE}")" || { + echo "[post-image] FATAL: veritysetup format failed" + exit 1 +} + +ROOT_HASH="$(printf '%s\n' "${VERITY_FORMAT_OUTPUT}" | awk -F': ' '/^Root hash:/ {print $2}' | tr -d '[:space:]')" +if [ -z "${ROOT_HASH}" ]; then + echo "[post-image] FATAL: failed to parse dm-verity root hash" + printf '%s\n' "${VERITY_FORMAT_OUTPUT}" + exit 1 +fi +printf '%s\n' "${ROOT_HASH}" > "${ROOT_HASH_FILE}" +echo "[post-image] dm-verity root hash: ${ROOT_HASH}" + +# Stage kernel and initramfs for the EFI partition. +# Buildroot's GRUB2 package has already placed bootx64.efi at +# ${BINARIES_DIR}/efi-part/EFI/BOOT/bootx64.efi; we add the kernel, +# initramfs, and overwrite the default grub.cfg with our boot entry. +echo "[post-image] Staging EFI partition files..." +mkdir -p "${BINARIES_DIR}/efi-part/EFI/BOOT" +cp "${BINARIES_DIR}/bzImage" "${BINARIES_DIR}/efi-part/bzImage" +cp "${BINARIES_DIR}/rootfs.cpio.gz" "${BINARIES_DIR}/efi-part/initrd.cpio.gz" + +cat > "${BINARIES_DIR}/efi-part/EFI/BOOT/grub.cfg" << GRUBCFG +set default=0 +set timeout=0 + +menuentry "Cocos" { + linux /bzImage console=ttyS0 roothash=${ROOT_HASH} systemd.verity=0 systemd.gpt_auto=0 + initrd /initrd.cpio.gz +} +GRUBCFG + +# Regenerate bootx64.efi with --disable-shim-lock so GRUB can load the kernel +# directly without requiring the shim bootloader (OVMF still verifies GRUB via +# Secure Boot; shim is not needed when booting from a custom OVMF with own DB key). +GRUB_CORE="$(ls -d "${BUILD_DIR}"/grub2-*/build-x86_64-efi/grub-core 2>/dev/null | head -1)" +if [ -n "${GRUB_CORE}" ]; then + echo "[post-image] Regenerating bootx64.efi with --disable-shim-lock..." + "${HOST_DIR}/bin/grub-mkimage" \ + -d "${GRUB_CORE}" \ + -O x86_64-efi \ + -o "${BINARIES_DIR}/efi-part/EFI/BOOT/bootx64.efi" \ + -p "/EFI/BOOT" \ + --disable-shim-lock \ + boot linux echo normal part_gpt fat ls search || { + echo "[post-image] FATAL: grub-mkimage failed" + exit 1 + } +else + echo "[post-image] WARNING: GRUB core dir not found, skipping --disable-shim-lock rebuild" +fi + +# Sign GRUB and kernel for UEFI Secure Boot. +# Keys are resolved in order: env var → board/secure-boot/ defaults. +SB_KEY="${SB_KEY:-${COCOS_BOARD_DIR}/secure-boot/db.key}" +SB_CERT="${SB_CERT:-${COCOS_BOARD_DIR}/secure-boot/db.crt}" +if [ -f "${SB_KEY}" ] && [ -f "${SB_CERT}" ]; then + echo "[post-image] Signing EFI binaries for Secure Boot..." + sbsign --key "${SB_KEY}" --cert "${SB_CERT}" \ + --output "${BINARIES_DIR}/efi-part/EFI/BOOT/bootx64.efi" \ + "${BINARIES_DIR}/efi-part/EFI/BOOT/bootx64.efi" || { + echo "[post-image] FATAL: Failed to sign bootx64.efi" + exit 1 + } + sbsign --key "${SB_KEY}" --cert "${SB_CERT}" \ + --output "${BINARIES_DIR}/efi-part/bzImage" \ + "${BINARIES_DIR}/efi-part/bzImage" || { + echo "[post-image] FATAL: Failed to sign bzImage" + exit 1 + } + echo "[post-image] Secure Boot signing complete" +else + echo "[post-image] WARNING: Secure Boot keys not found — EFI binaries are unsigned" + echo "[post-image] Default location: ${COCOS_BOARD_DIR}/secure-boot/db.key + db.crt" + echo "[post-image] Override: SB_KEY=/path/to/db.key SB_CERT=/path/to/db.crt make" +fi + +GENIMAGE_CFG="${COCOS_BOARD_DIR}/genimage.cfg" +if [ -f "${GENIMAGE_CFG}" ]; then + GENIMAGE_TMP="${BUILD_DIR}/genimage.tmp" + rm -rf "${GENIMAGE_TMP}" + genimage \ + --rootpath "${TARGET_DIR}" \ + --tmppath "${GENIMAGE_TMP}" \ + --inputpath "${BINARIES_DIR}" \ + --outputpath "${BINARIES_DIR}" \ + --config "${GENIMAGE_CFG}" +fi + +if [[ "${DEFCONFIG_NAME}" =~ ^"cocos_*" ]]; then + # Not a Qemu defconfig, can't test. + exit 0 +fi + +# Search for "# qemu_*_defconfig" tag in all readme.txt files. +# Qemu command line on multilines using back slash are accepted. +# shellcheck disable=SC2086 # glob over each readme file +QEMU_CMD_LINE="$(sed -r ':a; /\\$/N; s/\\\n//; s/\t/ /; ta; /# '"${DEFCONFIG_NAME}"'$/!d; s/#.*//' ${README_FILES})" + +if [ -z "${QEMU_CMD_LINE}" ]; then + # No Qemu cmd line found, can't test. + exit 0 +fi + +# Remove output/images path since the script will be in +# the same directory as the kernel and the rootfs images. +QEMU_CMD_LINE="${QEMU_CMD_LINE//output\/images\//}" + +# Remove -serial stdio if present, keep it as default args +DEFAULT_ARGS="$(sed -r -e '/-serial stdio/!d; s/.*(-serial stdio).*/\1/' <<<"${QEMU_CMD_LINE}")" +QEMU_CMD_LINE="${QEMU_CMD_LINE//-serial stdio/}" + +# Remove any string before qemu-system-* +QEMU_CMD_LINE="$(sed -r -e 's/^.*(qemu-system-)/\1/' <<<"${QEMU_CMD_LINE}")" + +# Disable graphical output and redirect serial I/Os to console +case ${DEFCONFIG_NAME} in + (qemu_sh4eb_r2d_defconfig|qemu_sh4_r2d_defconfig) + # Special case for SH4 + SERIAL_ARGS="-serial stdio -display none" + ;; + (*) + SERIAL_ARGS="-nographic" + ;; +esac + +sed -e "s|@SERIAL_ARGS@|${SERIAL_ARGS}|g" \ + -e "s|@DEFAULT_ARGS@|${DEFAULT_ARGS}|g" \ + -e "s|@QEMU_CMD_LINE@|${QEMU_CMD_LINE}|g" \ + -e "s|@HOST_DIR@|${HOST_DIR}|g" \ + <"${COCOS_BOARD_DIR}/start-qemu.sh.in" \ + >"${START_QEMU_SCRIPT}" +chmod +x "${START_QEMU_SCRIPT}" diff --git a/hal/disk/board/cocos/readme.txt b/hal/disk/board/cocos/readme.txt new file mode 100644 index 00000000..26637f86 --- /dev/null +++ b/hal/disk/board/cocos/readme.txt @@ -0,0 +1,7 @@ +Run the emulation with: + + qemu-system-x86_64 -M pc -kernel output/images/bzImage -drive file=output/images/rootfs.ext2,if=virtio,format=raw -append "rootwait root=/dev/vda console=tty1 console=ttyS0" -serial stdio -net nic,model=virtio -net user # cocos_defconfig + +Optionally add -smp N to emulate a SMP system with N CPUs. + +The login prompt will appear in the graphical window. diff --git a/hal/disk/board/cocos/secure-boot/.gitignore b/hal/disk/board/cocos/secure-boot/.gitignore new file mode 100644 index 00000000..78b7e5a6 --- /dev/null +++ b/hal/disk/board/cocos/secure-boot/.gitignore @@ -0,0 +1,3 @@ +# Private key must not be committed +db.key +db.crt \ No newline at end of file diff --git a/hal/disk/board/cocos/start-qemu.sh.in b/hal/disk/board/cocos/start-qemu.sh.in new file mode 100644 index 00000000..c7a9ac28 --- /dev/null +++ b/hal/disk/board/cocos/start-qemu.sh.in @@ -0,0 +1,28 @@ +#!/bin/sh + +BINARIES_DIR="${0%/*}/" +# shellcheck disable=SC2164 +cd "${BINARIES_DIR}" + +mode_serial=false +mode_sys_qemu=false +while [ "$1" ]; do + case "$1" in + --serial-only|serial-only) mode_serial=true; shift;; + --use-system-qemu) mode_sys_qemu=true; shift;; + --) shift; break;; + *) echo "unknown option: $1" >&2; exit 1;; + esac +done + +if ${mode_serial}; then + EXTRA_ARGS='@SERIAL_ARGS@' +else + EXTRA_ARGS='@DEFAULT_ARGS@' +fi + +if ! ${mode_sys_qemu}; then + export PATH="@HOST_DIR@/bin:${PATH}" +fi + +exec @QEMU_CMD_LINE@ ${EXTRA_ARGS} "$@" \ No newline at end of file diff --git a/hal/disk/board/rootfs-overlay/etc/fstab b/hal/disk/board/rootfs-overlay/etc/fstab new file mode 100644 index 00000000..5bf216ea --- /dev/null +++ b/hal/disk/board/rootfs-overlay/etc/fstab @@ -0,0 +1,7 @@ +# Root is mounted read-only by the initramfs through dm-verity. +# /cocos, /var, /tmp, and bind mounts are set up by the initramfs init script. +/dev/mapper/root_verity / ext4 ro,defaults 0 0 + +# 9P virtio shares — provided by the hypervisor, optional (nofail) +certs_share /etc/certs 9p trans=virtio,version=9p2000.L,cache=mmap,nofail 0 0 +env_share /etc/cocos 9p trans=virtio,version=9p2000.L,cache=mmap,nofail 0 0 diff --git a/hal/disk/board/rootfs-overlay/etc/ocicrypt_keyprovider.conf b/hal/disk/board/rootfs-overlay/etc/ocicrypt_keyprovider.conf new file mode 100644 index 00000000..6499cb27 --- /dev/null +++ b/hal/disk/board/rootfs-overlay/etc/ocicrypt_keyprovider.conf @@ -0,0 +1,7 @@ +{ + "key-providers": { + "attestation-agent": { + "grpc": "127.0.0.1:50011" + } + } +} diff --git a/hal/disk/board/rootfs-overlay/init b/hal/disk/board/rootfs-overlay/init new file mode 100755 index 00000000..6841f74b --- /dev/null +++ b/hal/disk/board/rootfs-overlay/init @@ -0,0 +1,265 @@ +#!/bin/sh + +export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +if (exec 0/dev/null; then + exec 0/dev/console + exec 2>/dev/console +fi + +echo "Welcome to the Cocos FDE test VM initramfs!" +echo "This is a minimal initramfs environment used for testing the FDE provisioning flow." +echo "If you see this message, the initramfs was loaded and executed successfully." +echo "The initramfs will now attempt to provision the disk and mount the real root filesystem." +echo "If any step fails, it will drop to a shell for debugging." + +[ -d /dev ] || mkdir -m 0755 /dev +[ -d /etc ] || mkdir -m 0755 /etc +[ -d /root ] || mkdir -m 0700 /root +[ -d /run ] || mkdir -m 0755 /run +[ -d /sys ] || mkdir /sys +[ -d /proc ] || mkdir /proc +[ -d /tmp ] || mkdir /tmp + +if [ -L /etc/resolv.conf ]; then + RESOLV_TARGET="$(readlink /etc/resolv.conf)" + case "$RESOLV_TARGET" in + /*) + RESOLV_PATH="$RESOLV_TARGET" + ;; + *) + RESOLV_PATH="/etc/$RESOLV_TARGET" + ;; + esac + + mkdir -p "$(dirname "$RESOLV_PATH")" + [ -e "$RESOLV_PATH" ] || : > "$RESOLV_PATH" +else + [ -e /etc/resolv.conf ] || : > /etc/resolv.conf +fi + +mount -t sysfs -o nodev,noexec,nosuid sysfs /sys +mount -t proc -o nodev,noexec,nosuid proc /proc + +mkdir -p /sys/kernel/config +if ! grep -q ' /sys/kernel/config ' /proc/mounts; then + mount -t configfs configfs /sys/kernel/config 2>/dev/null || true +fi + +mount -t devtmpfs -o nosuid,mode=0755 udev /dev +mkdir /dev/pts +mount -t devpts -o noexec,nosuid,gid=5,mode=0620 devpts /dev/pts || true + +MNT_DIR=/root +BASE=$(pwd) + +DST=/dev/sda +ROOTFS_TYPE="ext4" +ROOT_VERITY_MAP=root_verity +ROOT_VERITY_MAPPER="/dev/mapper/$ROOT_VERITY_MAP" +COCOS_MOUNT=/cocos +COCOS_MAP=cocos_crypt +COCOS_MAPPER="/dev/mapper/$COCOS_MAP" +LUKS_PARAMS="--cipher aes-gcm-random --integrity aead" + +settle_devices() { + echo "[init] Waiting for devices to settle..." + if command -v udevadm >/dev/null 2>&1; then + udevadm settle --timeout=10 || sleep 2 + else + sleep 2 + fi +} + +wipe_file() { + file_path="$1" + if [ -z "$file_path" ] || [ ! -e "$file_path" ]; then + return 0 + fi + + shred -vfz -n 3 "$file_path" 2>/dev/null || dd if=/dev/zero of="$file_path" bs=64 count=1 + rm -f "$file_path" +} + +partition_path() { + disk="$1" + partition="$2" + + case "$disk" in + *[0-9]) + printf '%sp%s\n' "$disk" "$partition" + ;; + *) + printf '%s%s\n' "$disk" "$partition" + ;; + esac +} + +append_9p_entry() { + pattern="$1" + default_entry="$2" + existing_entry="" + + if [ -f "$FSTAB_BAK" ]; then + existing_entry="$(grep -E "$pattern" "$FSTAB_BAK" | head -n 1 || true)" + fi + + if [ -n "$existing_entry" ]; then + printf '%s\n' "$existing_entry" >> "$FSTAB" + else + printf '%s\n' "$default_entry" >> "$FSTAB" + fi +} + +cmdline_arg() { + key="$1" + for arg in $(cat /proc/cmdline); do + case "$arg" in + "$key="*) + printf '%s\n' "${arg#*=}" + return 0 + ;; + esac + done + return 1 +} + +echo "[init] Starting disk provisioning..." +ROOT_PART="$(partition_path "$DST" 2)" +VERITY_PART="$(partition_path "$DST" 3)" +COCOS_PART="$(partition_path "$DST" 4)" +ROOT_HASH="$(cmdline_arg roothash)" + +if [ -z "$ROOT_HASH" ]; then + echo "[init] FATAL: Missing roothash= on kernel command line" + exec /bin/sh +fi + +settle_devices + +for part in "$ROOT_PART" "$VERITY_PART" "$COCOS_PART"; do + if [ ! -b "$part" ]; then + echo "[init] FATAL: Could not find partition $part" + echo "[init] Available block devices:" + lsblk || ls -la /dev/ || true + echo "[init] Dropping to shell." + exec /bin/sh + fi +done + +echo "[init] Opening dm-verity root mapping..." +veritysetup open "$ROOT_PART" "$ROOT_VERITY_MAP" "$VERITY_PART" "$ROOT_HASH" || { + echo "[init] FATAL: Failed to open dm-verity mapping for root" + exec /bin/sh +} + +echo "[init] Mounting root at $MNT_DIR (read-only)..." +mount -o ro -t "$ROOTFS_TYPE" "$ROOT_VERITY_MAPPER" "$MNT_DIR" || { + echo "[init] FATAL: Failed to mount verity root" + veritysetup close "$ROOT_VERITY_MAP" 2>/dev/null || true + exec /bin/sh +} + +echo "[init] Generating ephemeral key for $COCOS_MOUNT..." +dd if=/dev/urandom of=kk.bin bs=64 count=1 || { + echo "[init] FATAL: Failed to generate encryption key" + umount "$MNT_DIR" 2>/dev/null || true + exec /bin/sh +} +KK_BIN=$BASE/kk.bin + +cryptsetup luksFormat "$COCOS_PART" --type luks2 $LUKS_PARAMS --key-file="$KK_BIN" -q || { + echo "[init] FATAL: LUKS format failed" + umount "$MNT_DIR" 2>/dev/null || true + veritysetup close "$ROOT_VERITY_MAP" 2>/dev/null || true + wipe_file "$KK_BIN" + exec /bin/sh +} + +cryptsetup open "$COCOS_PART" "$COCOS_MAP" --key-file="$KK_BIN" || { + echo "[init] FATAL: Failed to open LUKS container for $COCOS_MOUNT" + umount "$MNT_DIR" 2>/dev/null || true + veritysetup close "$ROOT_VERITY_MAP" 2>/dev/null || true + wipe_file "$KK_BIN" + exec /bin/sh +} + +mkfs.ext4 -F -m 0 "$COCOS_MAPPER" >/dev/null || { + echo "[init] FATAL: Failed to create ext4 filesystem for $COCOS_MOUNT" + cryptsetup close "$COCOS_MAP" 2>/dev/null || true + umount "$MNT_DIR" 2>/dev/null || true + veritysetup close "$ROOT_VERITY_MAP" 2>/dev/null || true + wipe_file "$KK_BIN" + exec /bin/sh +} + +echo "[init] Mounting encrypted $COCOS_MOUNT..." +mkdir -p "$MNT_DIR$COCOS_MOUNT" +mount -t ext4 "$COCOS_MAPPER" "$MNT_DIR$COCOS_MOUNT" || { + echo "[init] FATAL: Failed to mount encrypted $COCOS_MOUNT filesystem" + cryptsetup close "$COCOS_MAP" 2>/dev/null || true + umount "$MNT_DIR" 2>/dev/null || true + veritysetup close "$ROOT_VERITY_MAP" 2>/dev/null || true + wipe_file "$KK_BIN" + exec /bin/sh +} + +mkdir -p \ + "$MNT_DIR$COCOS_MOUNT/.cache/oci" \ + "$MNT_DIR$COCOS_MOUNT/datasets" \ + "$MNT_DIR$COCOS_MOUNT/docker" \ + "$MNT_DIR$COCOS_MOUNT/cocos_init" + +# The root is read-only; provide tmpfs for writable system directories. +mount -t tmpfs tmpfs "$MNT_DIR/tmp" +mount -t tmpfs -o mode=0755 tmpfs "$MNT_DIR/var" + +# Bind Docker's data root onto /cocos so large images don't exhaust RAM. +mkdir -p "$MNT_DIR/var/lib/docker" +mount --bind "$MNT_DIR$COCOS_MOUNT/docker" "$MNT_DIR/var/lib/docker" + +# /cocos_init is on the read-only root; shadow it with a writable +# copy on /cocos so agent setup scripts can write state alongside the scripts. +if [ -d "$MNT_DIR/cocos_init" ]; then + cp -a "$MNT_DIR/cocos_init/." "$MNT_DIR$COCOS_MOUNT/cocos_init/" 2>/dev/null || true + mount --bind "$MNT_DIR$COCOS_MOUNT/cocos_init" "$MNT_DIR/cocos_init" || true +fi + +mount --move /proc $MNT_DIR/proc +mount --move /sys $MNT_DIR/sys + +FSTAB="$MNT_DIR/etc/fstab" +FSTAB_BAK="$MNT_DIR/etc/fstab.bak" + +mkdir -p "$MNT_DIR/etc/certs" "$MNT_DIR/etc/cocos" 2>/dev/null || true + +if [ -f "$FSTAB" ]; then + mv "$FSTAB" "$FSTAB_BAK" +fi + +cat > "$FSTAB" << EOF +# Generated by init script +$ROOT_VERITY_MAPPER / $ROOTFS_TYPE ro,defaults 0 0 +EOF + +append_9p_entry \ + '^certs_share[[:space:]]+/etc/certs[[:space:]]+9p([[:space:]]|$)' \ + 'certs_share /etc/certs 9p trans=virtio,version=9p2000.L,cache=mmap,nofail 0 0' + +append_9p_entry \ + '^env_share[[:space:]]+/etc/cocos[[:space:]]+9p([[:space:]]|$)' \ + 'env_share /etc/cocos 9p trans=virtio,version=9p2000.L,cache=mmap,nofail 0 0' + +printf '%s\n' '# /cocos is mounted by the FDE initramfs using an ephemeral LUKS key.' >> "$FSTAB" + +# Securely wipe the encryption key before switching root. +echo "[init] Securely wiping the $COCOS_MOUNT encryption key..." +wipe_file "$KK_BIN" + +echo "[init] Switching to real root..." +exec switch_root $MNT_DIR/ /sbin/init + +# If switch_root somehow returns: +echo "[init] switch_root failed, dropping to shell" +exec /bin/sh diff --git a/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/cocos-agent.service.d/mounts.conf b/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/cocos-agent.service.d/mounts.conf new file mode 100644 index 00000000..20061279 --- /dev/null +++ b/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/cocos-agent.service.d/mounts.conf @@ -0,0 +1,3 @@ +[Unit] +RequiresMountsFor=/etc/cocos /etc/certs + diff --git a/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/computation-runner.service.d/mounts.conf b/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/computation-runner.service.d/mounts.conf new file mode 100644 index 00000000..c60c301d --- /dev/null +++ b/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/computation-runner.service.d/mounts.conf @@ -0,0 +1,3 @@ +[Unit] +RequiresMountsFor=/etc/cocos + diff --git a/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/egress-proxy.service.d/mounts.conf b/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/egress-proxy.service.d/mounts.conf new file mode 100644 index 00000000..c60c301d --- /dev/null +++ b/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/egress-proxy.service.d/mounts.conf @@ -0,0 +1,3 @@ +[Unit] +RequiresMountsFor=/etc/cocos + diff --git a/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/log-forwarder.service.d/mounts.conf b/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/log-forwarder.service.d/mounts.conf new file mode 100644 index 00000000..c60c301d --- /dev/null +++ b/hal/disk/board/rootfs-overlay/usr/lib/systemd/system/log-forwarder.service.d/mounts.conf @@ -0,0 +1,3 @@ +[Unit] +RequiresMountsFor=/etc/cocos + diff --git a/hal/disk/board/rootfs-overlay/usr/lib/tmpfiles.d/cocos.conf b/hal/disk/board/rootfs-overlay/usr/lib/tmpfiles.d/cocos.conf new file mode 100644 index 00000000..e466be4f --- /dev/null +++ b/hal/disk/board/rootfs-overlay/usr/lib/tmpfiles.d/cocos.conf @@ -0,0 +1,2 @@ +d /var/log/cocos 0755 root root - +d /run/cocos 0755 root root - diff --git a/hal/disk/configs/cocos_defconfig b/hal/disk/configs/cocos_defconfig new file mode 100644 index 00000000..80e505ed --- /dev/null +++ b/hal/disk/configs/cocos_defconfig @@ -0,0 +1,117 @@ +# Architecture +BR2_x86_64=y + +# System +BR2_TARGET_GENERIC_HOSTNAME="cocos" +BR2_TARGET_GENERIC_ISSUE="Welcome to Cocos" +BR2_PACKAGE_DHCP=y +BR2_PACKAGE_DHCP_CLIENT=y +BR2_INIT_SYSTEMD=y +BR2_SYSTEM_BIN_SH_BASH=y + +# Filesystem +# BR2_TARGET_ROOTFS_TAR is not set +# Initramfs (rootfs.cpio.gz) is built by post-image.sh from only the FDE tools, +# not from the full target rootfs. The full rootfs goes to rootfs.ext4 (disk image). +BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_COCOS_PATH)/board/rootfs-overlay" + +# Patches for existing Buildroot packages +BR2_GLOBAL_PATCH_DIR="$(BR2_EXTERNAL_COCOS_PATH)/patches" + +# Bootloader +BR2_TARGET_GRUB2=y +BR2_TARGET_GRUB2_X86_64_EFI=y +BR2_TARGET_GRUB2_BUILTIN_MODULES_EFI="boot linux echo normal part_gpt fat ls search" + +# Disk image +BR2_TARGET_ROOTFS_EXT2=y +BR2_TARGET_ROOTFS_EXT2_4=y +BR2_TARGET_ROOTFS_EXT2_SIZE="10G" +BR2_PACKAGE_HOST_GENIMAGE=y +BR2_PACKAGE_HOST_CRYPTSETUP=y + +# Image +BR2_ROOTFS_POST_BUILD_SCRIPT="$(BR2_EXTERNAL_COCOS_PATH)/board/cocos/post-build.sh" + +# Image +BR2_ROOTFS_POST_IMAGE_SCRIPT="$(BR2_EXTERNAL_COCOS_PATH)/board/cocos/post-image.sh" +BR2_ROOTFS_POST_SCRIPT_ARGS="$(BR2_DEFCONFIG)" + +# Linux headers same as kernel +BR2_PACKAGE_HOST_LINUX_HEADERS_CUSTOM_6_11=y +BR2_TOOLCHAIN_HEADERS_LATEST=y +BR2_TOOLCHAIN_HEADERS_AT_LEAST="6.11-rc7" + +# Kernel +BR2_LINUX_KERNEL=y +BR2_LINUX_KERNEL_CUSTOM_GIT=y +BR2_LINUX_KERNEL_CUSTOM_REPO_URL="https://github.com/coconut-svsm/linux.git" +BR2_LINUX_KERNEL_CUSTOM_REPO_VERSION="svsm" +BR2_LINUX_KERNEL_VERSION="svsm" +BR2_LINUX_KERNEL_PATCH="" +BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y +BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="$(BR2_EXTERNAL_COCOS_PATH)/board/cocos/linux.config" +BR2_LINUX_KERNEL_NEEDS_HOST_LIBELF=y + +# host-qemu for gitlab testing +BR2_PACKAGE_HOST_QEMU=y +BR2_PACKAGE_HOST_QEMU_SYSTEM_MODE=y + +# Python +BR2_PACKAGE_PYTHON3=y +BR2_PACKAGE_PYTHON_PIP=y +BR2_PACKAGE_BZIP2=y +BR2_PACKAGE_XZ=y +BR2_PACKAGE_ZIP=y +BR2_PACKAGE_PYTHON3_ZLIB=y +BR2_PACKAGE_PYTHON3_XZ=y +BR2_PACKAGE_PYTHON3_BZIP2=y +BR2_INSTALL_LIBSTDCPP=y +BR2_TOOLCHAIN_BUILDROOT_CXX=y +BR2_PACKAGE_HOST_GCC_TARGET=y +BR2_TOOLCHAIN_BUILDROOT_LIBSTDCPP=y +BR2_PACKAGE_GCC=y +BR2_PACKAGE_GCC_TARGET=y +BR2_PACKAGE_LIBSTDCPP=y + +# FDE +BR2_PACKAGE_NBD=y +BR2_PACKAGE_NBD_CLIENT=y +BR2_PACKAGE_CRYPTSETUP=y +BR2_ROOTFS_DEVICE_CREATION_DYNAMIC_EUDEV=y +BR2_PACKAGE_EUDEV=y +BR2_PACKAGE_HAS_UDEV=y +BR2_PACKAGE_MULTIPATH_TOOLS=y +BR2_PACKAGE_UTIL_LINUX_BINARIES=y +BR2_PACKAGE_E2FSPROGS=y +BR2_LINUX_KERNEL_NEEDS_HOST_PAHOLE=y + +# TPM2 +BR2_PACKAGE_TPM2_TOOLS=y +BR2_PACKAGE_COREUTILS=y + +# Docker +BR2_PACKAGE_LIBSECCOMP_ARCH_SUPPORTS=y +BR2_PACKAGE_LIBSECCOMP=y +BR2_PACKAGE_CA_CERTIFICATES=y +BR2_PACKAGE_DOCKER_CLI=y +BR2_PACKAGE_DOCKER_COMPOSE=y +BR2_PACKAGE_DOCKER_ENGINE=y +BR2_PACKAGE_CONTAINERD=y +BR2_PACKAGE_RUNC=y +BR2_PACKAGE_IPTABLES=y + +# Skopeo for OCI image handling with CoCo Keyprovider +BR2_PACKAGE_SKOPEO=y +BR2_PACKAGE_GPGME=y +BR2_PACKAGE_LVM2=y +BR2_PACKAGE_LVM2_STANDARD_INSTALL=y +BR2_PACKAGE_9PFS=y + +# Host tools +BR2_PACKAGE_HOST_RUSTC=y +BR2_PACKAGE_HOST_RUST_BIN=y + +# Cocos AI Packages +BR2_PACKAGE_AGENT=y +# BR2_PACKAGE_CC_ATTESTATION_AGENT is not set diff --git a/hal/disk/external.desc b/hal/disk/external.desc new file mode 100644 index 00000000..de4e25fd --- /dev/null +++ b/hal/disk/external.desc @@ -0,0 +1,2 @@ +name: COCOS +desc: External buildroot tree for Cocos AI diff --git a/hal/disk/external.mk b/hal/disk/external.mk new file mode 100644 index 00000000..0bfc533a --- /dev/null +++ b/hal/disk/external.mk @@ -0,0 +1 @@ +include $(sort $(wildcard $(BR2_EXTERNAL_COCOS_PATH)/package/*/*.mk)) diff --git a/hal/disk/package/agent/Config.in b/hal/disk/package/agent/Config.in new file mode 100644 index 00000000..0e9093be --- /dev/null +++ b/hal/disk/package/agent/Config.in @@ -0,0 +1,13 @@ +config BR2_PACKAGE_AGENT + bool "agent" + default y + select BR2_PACKAGE_ATTESTATION_SERVICE + select BR2_PACKAGE_LOG_FORWARDER + select BR2_PACKAGE_COMPUTATION_RUNNER + select BR2_PACKAGE_INGRESS_PROXY + select BR2_PACKAGE_EGRESS_PROXY + help + Confidential Computing Agent is a state machine capable of + receiving datasets and algorithm, running computations, and + fetching the attestation report from within the + Confidential VM. diff --git a/hal/disk/package/agent/agent.mk b/hal/disk/package/agent/agent.mk new file mode 100644 index 00000000..66a9eb87 --- /dev/null +++ b/hal/disk/package/agent/agent.mk @@ -0,0 +1,27 @@ +################################################################################ +# +# Cocos AI Agent +# +################################################################################ + +AGENT_VERSION = main +AGENT_SITE = $(call github,ultravioletrs,cocos,$(AGENT_VERSION)) + +define AGENT_BUILD_CMDS + $(MAKE) -C $(@D) agent EMBED_ENABLED=$(AGENT_EMBED_ENABLED) +endef + +define AGENT_INSTALL_TARGET_CMDS + mkdir -p $(TARGET_DIR)/cocos/ + mkdir -p $(TARGET_DIR)/var/log/cocos + mkdir -p $(TARGET_DIR)/cocos_init/ + $(INSTALL) -D -m 0750 $(@D)/build/cocos-agent $(TARGET_DIR)/bin +endef + +define AGENT_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 0640 $(@D)/init/systemd/cocos-agent.service $(TARGET_DIR)/usr/lib/systemd/system/cocos-agent.service + $(INSTALL) -D -m 0750 $(@D)/init/systemd/agent_setup.sh $(TARGET_DIR)/cocos_init/agent_setup.sh + $(INSTALL) -D -m 0750 $(@D)/init/systemd/agent_start_script.sh $(TARGET_DIR)/cocos_init/agent_start_script.sh +endef + +$(eval $(generic-package)) diff --git a/hal/disk/package/attestation-service/Config.in b/hal/disk/package/attestation-service/Config.in new file mode 100644 index 00000000..90287be1 --- /dev/null +++ b/hal/disk/package/attestation-service/Config.in @@ -0,0 +1,11 @@ +config BR2_PACKAGE_ATTESTATION_SERVICE + bool + default y + help + Cocos AI attestation service that generates EAT tokens + for TEE attestation (SNP, TDX, vTPM, Azure). + + This service can optionally use the Confidential Containers + attestation-agent as a backend provider via gRPC. + + https://github.com/ultravioletrs/cocos diff --git a/hal/disk/package/attestation-service/attestation-service.mk b/hal/disk/package/attestation-service/attestation-service.mk new file mode 100644 index 00000000..aae86836 --- /dev/null +++ b/hal/disk/package/attestation-service/attestation-service.mk @@ -0,0 +1,34 @@ +################################################################################ +# +# attestation-service +# +################################################################################ + +ATTESTATION_SERVICE_VERSION = main +ATTESTATION_SERVICE_SITE = $(call github,ultravioletrs,cocos,$(ATTESTATION_SERVICE_VERSION)) + +define ATTESTATION_SERVICE_BUILD_CMDS + $(MAKE) -C $(@D) attestation-service +endef + +define ATTESTATION_SERVICE_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0755 $(@D)/build/cocos-attestation-service $(TARGET_DIR)/usr/bin/attestation-service +endef + +ifeq ($(BR2_PACKAGE_CC_ATTESTATION_AGENT),y) +define ATTESTATION_SERVICE_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 0640 $(@D)/init/systemd/attestation-service.service $(TARGET_DIR)/usr/lib/systemd/system/attestation-service.service + $(INSTALL) -D -m 0750 $(@D)/init/systemd/attestation_setup.sh $(TARGET_DIR)/cocos_init/attestation_setup.sh + # CC attestation agent is already enabled by default +endef +else +define ATTESTATION_SERVICE_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 0640 $(@D)/init/systemd/attestation-service.service $(TARGET_DIR)/usr/lib/systemd/system/attestation-service.service + $(INSTALL) -D -m 0750 $(@D)/init/systemd/attestation_setup.sh $(TARGET_DIR)/cocos_init/attestation_setup.sh + # Disable CC attestation agent backend if not selected + sed -i 's/USE_CC_ATTESTATION_AGENT=true/USE_CC_ATTESTATION_AGENT=false/' $(TARGET_DIR)/usr/lib/systemd/system/attestation-service.service + sed -i '/Wants=attestation-agent.service/d' $(TARGET_DIR)/usr/lib/systemd/system/attestation-service.service +endef +endif + +$(eval $(generic-package)) diff --git a/hal/disk/package/cc-attestation-agent/Config.in b/hal/disk/package/cc-attestation-agent/Config.in new file mode 100644 index 00000000..c9fbfd44 --- /dev/null +++ b/hal/disk/package/cc-attestation-agent/Config.in @@ -0,0 +1,28 @@ +config BR2_PACKAGE_CC_ATTESTATION_AGENT + bool "cc-attestation-agent" + select BR2_PACKAGE_PROTOBUF + select BR2_PACKAGE_OPENSSL + select BR2_PACKAGE_TPM2_TSS + help + Confidential Containers attestation-agent for TEE attestation. + + Optional backend for the Cocos AI attestation service that + provides KBS protocol support for remote attestation and + encrypted secret provisioning. + + https://github.com/confidential-containers/guest-components + +if BR2_PACKAGE_CC_ATTESTATION_AGENT + +config BR2_PACKAGE_CC_ATTESTATION_AGENT_KBS_URL + string "Default KBS URL (optional)" + default "" + help + Optional default KBS (Key Broker Service) URL for remote + attestation and secret provisioning. + + Leave empty to operate in local attestation mode only. + + Example: https://kbs.example.com:8080 + +endif diff --git a/hal/disk/package/cc-attestation-agent/cc-attestation-agent-setup.sh b/hal/disk/package/cc-attestation-agent/cc-attestation-agent-setup.sh new file mode 100644 index 00000000..680ceb29 --- /dev/null +++ b/hal/disk/package/cc-attestation-agent/cc-attestation-agent-setup.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Setup permissions for attestation socket directory + +mkdir -p /run/cocos +chmod 755 /run/cocos diff --git a/hal/disk/package/cc-attestation-agent/cc-attestation-agent.mk b/hal/disk/package/cc-attestation-agent/cc-attestation-agent.mk new file mode 100644 index 00000000..bebd5081 --- /dev/null +++ b/hal/disk/package/cc-attestation-agent/cc-attestation-agent.mk @@ -0,0 +1,37 @@ +################################################################################ +# +# cc-attestation-agent +# +################################################################################ + +CC_ATTESTATION_AGENT_VERSION = mvp-runner +CC_ATTESTATION_AGENT_SITE = $(call github,rodneyosodo,guest-components,$(CC_ATTESTATION_AGENT_VERSION)) +CC_ATTESTATION_AGENT_LICENSE = Apache-2.0 +CC_ATTESTATION_AGENT_LICENSE_FILES = LICENSE + +CC_ATTESTATION_AGENT_DEPENDENCIES = host-rustc openssl protobuf tpm2-tss + +# Build the attestation-agent from the guest-components repository with gRPC support +define CC_ATTESTATION_AGENT_BUILD_CMDS + cd $(@D)/attestation-agent && \ + $(TARGET_MAKE_ENV) \ + CARGO_HOME=$(@D)/.cargo \ + make ATTESTER=all-attesters ttrpc=false +endef + +define CC_ATTESTATION_AGENT_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0755 \ + $(@D)/target/$(RUSTC_TARGET_NAME)/release/attestation-agent \ + $(TARGET_DIR)/usr/bin/attestation-agent +endef + +define CC_ATTESTATION_AGENT_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 0644 \ + $(BR2_EXTERNAL_COCOS_PATH)/package/cc-attestation-agent/cc-attestation-agent.service \ + $(TARGET_DIR)/usr/lib/systemd/system/attestation-agent.service + $(INSTALL) -D -m 0750 \ + $(BR2_EXTERNAL_COCOS_PATH)/package/cc-attestation-agent/cc-attestation-agent-setup.sh \ + $(TARGET_DIR)/cocos_init/attestation_setup.sh +endef + +$(eval $(generic-package)) diff --git a/hal/disk/package/cc-attestation-agent/cc-attestation-agent.service b/hal/disk/package/cc-attestation-agent/cc-attestation-agent.service new file mode 100644 index 00000000..d3aed913 --- /dev/null +++ b/hal/disk/package/cc-attestation-agent/cc-attestation-agent.service @@ -0,0 +1,13 @@ +[Unit] +Description=Confidential Containers Attestation Agent (gRPC) +After=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/attestation-agent --attestation_sock 127.0.0.1:50002 +Restart=always +RestartSec=5 +Environment=RUST_LOG=info + +[Install] +WantedBy=multi-user.target diff --git a/hal/disk/package/coco-keyprovider/Config.in b/hal/disk/package/coco-keyprovider/Config.in new file mode 100644 index 00000000..5925aad4 --- /dev/null +++ b/hal/disk/package/coco-keyprovider/Config.in @@ -0,0 +1,11 @@ +config BR2_PACKAGE_COCO_KEYPROVIDER + bool "coco-keyprovider" + depends on BR2_PACKAGE_HOST_RUSTC_ARCH_SUPPORTS + select BR2_PACKAGE_HOST_RUSTC + help + CoCo Keyprovider is a keyprovider tool for generating and + decrypting CoCo-compatible encrypted images. It implements + the ocicrypt keyprovider protocol to decrypt OCI image layers + using the Key Broker Service (KBS). + + https://github.com/confidential-containers/guest-components diff --git a/hal/disk/package/coco-keyprovider/coco-keyprovider-setup.sh b/hal/disk/package/coco-keyprovider/coco-keyprovider-setup.sh new file mode 100644 index 00000000..2572c40c --- /dev/null +++ b/hal/disk/package/coco-keyprovider/coco-keyprovider-setup.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -e + +# Read kernel command line +CMDLINE=$(cat /proc/cmdline) + +# Extract agent.aa_kbc_params value +# Format: agent.aa_kbc_params=cc_kbc::URL +PARAMS=$(echo "$CMDLINE" | tr ' ' '\n' | grep '^agent.aa_kbc_params=' | cut -d= -f2-) + +if [ -n "$PARAMS" ]; then + # Extract URL part (after ::) + KBS_URL="${PARAMS#*::}" + if [ -n "$KBS_URL" ]; then + echo "[coco-keyprovider-setup] Detected KBS URL from kernel cmdline: $KBS_URL" + KBS_ARG="--kbs $KBS_URL" + fi +else + echo "[coco-keyprovider-setup] No agent.aa_kbc_params found in kernel cmdline. Starting without --kbs." +fi + +# COCO_KP_SOCKET is set by EnvironmentFile in .service +if [ -z "$COCO_KP_SOCKET" ]; then + COCO_KP_SOCKET="127.0.0.1:50011" +fi + +echo "[coco-keyprovider-setup] Starting coco_keyprovider listening on $COCO_KP_SOCKET $KBS_ARG" +exec /usr/local/bin/coco_keyprovider --socket "$COCO_KP_SOCKET" $KBS_ARG diff --git a/hal/disk/package/coco-keyprovider/coco-keyprovider.default b/hal/disk/package/coco-keyprovider/coco-keyprovider.default new file mode 100644 index 00000000..4aa0d31c --- /dev/null +++ b/hal/disk/package/coco-keyprovider/coco-keyprovider.default @@ -0,0 +1,3 @@ +# CoCo Keyprovider Environment Variables +COCO_KP_SOCKET=127.0.0.1:50011 +RUST_LOG=info diff --git a/hal/disk/package/coco-keyprovider/coco-keyprovider.mk b/hal/disk/package/coco-keyprovider/coco-keyprovider.mk new file mode 100644 index 00000000..a706c05d --- /dev/null +++ b/hal/disk/package/coco-keyprovider/coco-keyprovider.mk @@ -0,0 +1,34 @@ +################################################################################ +# +# coco-keyprovider +# +################################################################################ + +COCO_KEYPROVIDER_VERSION = mvp-runner +COCO_KEYPROVIDER_SITE = $(call github,rodneyosodo,guest-components,$(COCO_KEYPROVIDER_VERSION)) +COCO_KEYPROVIDER_LICENSE = Apache-2.0 +COCO_KEYPROVIDER_LICENSE_FILES = LICENSE + +COCO_KEYPROVIDER_DEPENDENCIES = host-rustc + +define COCO_KEYPROVIDER_BUILD_CMDS + cd $(@D)/attestation-agent/coco_keyprovider && \ + $(TARGET_MAKE_ENV) $(TARGET_CONFIGURE_OPTS) \ + CARGO_HOME=$(HOST_DIR)/share/cargo \ + cargo build --release --target=$(RUSTC_TARGET_NAME) +endef + +define COCO_KEYPROVIDER_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0755 $(@D)/target/$(RUSTC_TARGET_NAME)/release/coco_keyprovider \ + $(TARGET_DIR)/usr/local/bin/coco_keyprovider + $(INSTALL) -D -m 0755 $(BR2_EXTERNAL_COCOS_PATH)/package/coco-keyprovider/coco-keyprovider-setup.sh \ + $(TARGET_DIR)/usr/local/bin/coco-keyprovider-setup.sh + $(INSTALL) -D -m 0644 $(BR2_EXTERNAL_COCOS_PATH)/package/coco-keyprovider/coco-keyprovider.service \ + $(TARGET_DIR)/etc/systemd/system/coco-keyprovider.service + $(INSTALL) -D -m 0644 $(BR2_EXTERNAL_COCOS_PATH)/package/coco-keyprovider/coco-keyprovider.default \ + $(TARGET_DIR)/etc/default/coco-keyprovider + mkdir -p $(TARGET_DIR)/etc + echo '{"key-providers": {"attestation-agent": {"grpc": "127.0.0.1:50011"}}}' > $(TARGET_DIR)/etc/ocicrypt_keyprovider.conf +endef + +$(eval $(generic-package)) diff --git a/hal/disk/package/coco-keyprovider/coco-keyprovider.service b/hal/disk/package/coco-keyprovider/coco-keyprovider.service new file mode 100644 index 00000000..9e90ebe7 --- /dev/null +++ b/hal/disk/package/coco-keyprovider/coco-keyprovider.service @@ -0,0 +1,25 @@ +[Unit] +Description=CoCo Keyprovider for Confidential Containers +Documentation=https://github.com/confidential-containers/guest-components +After=network-online.target attestation-agent.service +Wants=network-online.target +Requires=attestation-agent.service + +[Service] +Type=simple +EnvironmentFile=/etc/default/coco-keyprovider +RuntimeDirectory=coco-keyprovider +ExecStart=/usr/local/bin/coco-keyprovider-setup.sh +Restart=on-failure +RestartSec=5s +StandardOutput=journal +StandardError=journal + +# Security hardening +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true + +[Install] +WantedBy=multi-user.target diff --git a/hal/disk/package/computation-runner/Config.in b/hal/disk/package/computation-runner/Config.in new file mode 100644 index 00000000..18718d79 --- /dev/null +++ b/hal/disk/package/computation-runner/Config.in @@ -0,0 +1,5 @@ +config BR2_PACKAGE_COMPUTATION_RUNNER + bool "computation-runner" + select BR2_PACKAGE_LOG_FORWARDER + help + Cocos AI Computation Runner service. diff --git a/hal/disk/package/computation-runner/computation-runner.mk b/hal/disk/package/computation-runner/computation-runner.mk new file mode 100644 index 00000000..1850695a --- /dev/null +++ b/hal/disk/package/computation-runner/computation-runner.mk @@ -0,0 +1,22 @@ +################################################################################ +# +# computation-runner +# +################################################################################ + +COMPUTATION_RUNNER_VERSION = main +COMPUTATION_RUNNER_SITE = $(call github,ultravioletrs,cocos,$(COMPUTATION_RUNNER_VERSION)) + +define COMPUTATION_RUNNER_BUILD_CMDS + $(MAKE) -C $(@D) computation-runner +endef + +define COMPUTATION_RUNNER_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0750 $(@D)/build/cocos-computation-runner $(TARGET_DIR)/usr/bin/computation-runner +endef + +define COMPUTATION_RUNNER_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 0640 $(@D)/init/systemd/computation-runner.service $(TARGET_DIR)/usr/lib/systemd/system/computation-runner.service +endef + +$(eval $(generic-package)) diff --git a/hal/disk/package/egress-proxy/Config.in b/hal/disk/package/egress-proxy/Config.in new file mode 100644 index 00000000..945d2689 --- /dev/null +++ b/hal/disk/package/egress-proxy/Config.in @@ -0,0 +1,6 @@ +config BR2_PACKAGE_EGRESS_PROXY + bool "egress-proxy" + help + Cocos AI Egress Proxy Service. + + https://github.com/ultravioletrs/cocos diff --git a/hal/disk/package/egress-proxy/egress-proxy.mk b/hal/disk/package/egress-proxy/egress-proxy.mk new file mode 100644 index 00000000..eb5e5866 --- /dev/null +++ b/hal/disk/package/egress-proxy/egress-proxy.mk @@ -0,0 +1,22 @@ +################################################################################ +# +# Cocos AI Egress Proxy +# +################################################################################ + +EGRESS_PROXY_VERSION = main +EGRESS_PROXY_SITE = $(call github,ultravioletrs,cocos,$(EGRESS_PROXY_VERSION)) + +define EGRESS_PROXY_BUILD_CMDS + $(MAKE) -C $(@D) egress-proxy +endef + +define EGRESS_PROXY_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0755 $(@D)/build/cocos-egress-proxy $(TARGET_DIR)/usr/bin/egress-proxy +endef + +define EGRESS_PROXY_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 0644 $(@D)/init/systemd/egress-proxy.service $(TARGET_DIR)/usr/lib/systemd/system/egress-proxy.service +endef + +$(eval $(generic-package)) diff --git a/hal/disk/package/ingress-proxy/Config.in b/hal/disk/package/ingress-proxy/Config.in new file mode 100644 index 00000000..a0f6b36b --- /dev/null +++ b/hal/disk/package/ingress-proxy/Config.in @@ -0,0 +1,4 @@ +config BR2_PACKAGE_INGRESS_PROXY + bool "ingress-proxy" + help + Cocos Ingress Proxy service. diff --git a/hal/disk/package/ingress-proxy/ingress-proxy.mk b/hal/disk/package/ingress-proxy/ingress-proxy.mk new file mode 100644 index 00000000..e407a572 --- /dev/null +++ b/hal/disk/package/ingress-proxy/ingress-proxy.mk @@ -0,0 +1,22 @@ +################################################################################ +# +# ingress-proxy +# +################################################################################ + +INGRESS_PROXY_VERSION = main +INGRESS_PROXY_SITE = $(call github,ultravioletrs,cocos,$(INGRESS_PROXY_VERSION)) + +define INGRESS_PROXY_BUILD_CMDS + $(MAKE) -C $(@D) ingress-proxy +endef + +define INGRESS_PROXY_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0750 $(@D)/build/cocos-ingress-proxy $(TARGET_DIR)/usr/bin/ingress-proxy +endef + +# NOTE: The ingress-proxy is managed per-computation by the agent, not as a standalone +# systemd service. The binary is installed for use by the agent, but no systemd service +# is created. + +$(eval $(generic-package)) diff --git a/hal/disk/package/log-forwarder/Config.in b/hal/disk/package/log-forwarder/Config.in new file mode 100644 index 00000000..634daeb8 --- /dev/null +++ b/hal/disk/package/log-forwarder/Config.in @@ -0,0 +1,4 @@ +config BR2_PACKAGE_LOG_FORWARDER + bool "log-forwarder" + help + Cocos AI Log Forwarder service. diff --git a/hal/disk/package/log-forwarder/log-forwarder.mk b/hal/disk/package/log-forwarder/log-forwarder.mk new file mode 100644 index 00000000..47545880 --- /dev/null +++ b/hal/disk/package/log-forwarder/log-forwarder.mk @@ -0,0 +1,22 @@ +################################################################################ +# +# log-forwarder +# +################################################################################ + +LOG_FORWARDER_VERSION = main +LOG_FORWARDER_SITE = $(call github,ultravioletrs,cocos,$(LOG_FORWARDER_VERSION)) + +define LOG_FORWARDER_BUILD_CMDS + $(MAKE) -C $(@D) log-forwarder +endef + +define LOG_FORWARDER_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0750 $(@D)/build/cocos-log-forwarder $(TARGET_DIR)/usr/bin/log-forwarder +endef + +define LOG_FORWARDER_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 0640 $(@D)/init/systemd/log-forwarder.service $(TARGET_DIR)/usr/lib/systemd/system/log-forwarder.service +endef + +$(eval $(generic-package)) diff --git a/hal/disk/package/wasmedge/Config.in b/hal/disk/package/wasmedge/Config.in new file mode 100644 index 00000000..c0a7be4e --- /dev/null +++ b/hal/disk/package/wasmedge/Config.in @@ -0,0 +1,6 @@ +config BR2_PACKAGE_WASMEDGE + bool "wasmedge" + default y + help + Wasmedge is a standalone runtime for WebAssembly. + https://wasmedge.org/docs/ diff --git a/hal/disk/package/wasmedge/wasmedge.mk b/hal/disk/package/wasmedge/wasmedge.mk new file mode 100644 index 00000000..11e6881d --- /dev/null +++ b/hal/disk/package/wasmedge/wasmedge.mk @@ -0,0 +1,8 @@ +WASMEDGE_DOWNLOAD_URL = https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh + +define WASMEDGE_INSTALL_TARGET_CMDS + curl -sSf $(WASMEDGE_DOWNLOAD_URL) | bash -s -- -p $(TARGET_DIR)/usr -v 0.14.1 + echo "source /usr/env" >> $(TARGET_DIR)/etc/profile +endef + +$(eval $(generic-package)) diff --git a/hal/disk/patches/grub2/0001-efi-skip-lockdown-when-shim-lock-disabled.patch b/hal/disk/patches/grub2/0001-efi-skip-lockdown-when-shim-lock-disabled.patch new file mode 100644 index 00000000..794151dc --- /dev/null +++ b/hal/disk/patches/grub2/0001-efi-skip-lockdown-when-shim-lock-disabled.patch @@ -0,0 +1,34 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +Subject: [PATCH] efi: skip lockdown when built with --disable-shim-lock + +When GRUB is built with --disable-shim-lock, grub_shim_lock_verifier_setup() +returns early without registering the shim_lock verifier. However +grub_lockdown() was called unconditionally before that, registering the +lockdown_verifier which marks kernel files as DEFER_AUTH. With no verifier +present to approve them, every kernel load fails with "verification requested +but nobody cares". + +Fix by calling grub_shim_lock_verifier_setup() first and only calling +grub_lockdown() if shim_lock is actually active. This preserves full +lockdown behaviour in shim-based chains while allowing direct +OVMF->GRUB->kernel boot with a custom DB key and no shim. + +Signed-off-by: Cocos AI +--- + grub-core/kern/efi/init.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/grub-core/kern/efi/init.c b/grub-core/kern/efi/init.c +--- a/grub-core/kern/efi/init.c ++++ b/grub-core/kern/efi/init.c +@@ -122,8 +122,9 @@ + */ + if (grub_efi_get_secureboot () == GRUB_EFI_SECUREBOOT_MODE_ENABLED) + { +- grub_lockdown (); + grub_shim_lock_verifier_setup (); ++ if (grub_is_shim_lock_enabled ()) ++ grub_lockdown (); + } + + grub_efi_system_table->boot_services->set_watchdog_timer (0, 0, 0, NULL); diff --git a/hal/linux/README.md b/hal/linux/README.md index 43310d60..6045cd0c 100644 --- a/hal/linux/README.md +++ b/hal/linux/README.md @@ -10,7 +10,7 @@ HAL uses [Buildroot](https://buildroot.org/)'s [_External Tree_ mechanism](https git clone git@github.com:ultravioletrs/cocos.git git clone git@github.com:buildroot/buildroot.git cd buildroot -git checkout 2025.08-rc3 +git checkout 2025.11 make BR2_EXTERNAL=../cocos/hal/linux cocos_defconfig # Execute 'make menuconfig' only if you want to make additional configuration changes to Buildroot. make menuconfig diff --git a/init/systemd/agent_setup.sh b/init/systemd/agent_setup.sh index 8a9c0602..f8404a7c 100644 --- a/init/systemd/agent_setup.sh +++ b/init/systemd/agent_setup.sh @@ -22,5 +22,8 @@ if [ ! -d "$WORK_DIR" ]; then mkdir -p $WORK_DIR fi -# Resize the root file system to 100% -mount -o remount,size=100% / +# RAM-only agent images use tmpfs as the root filesystem +ROOT_FSTYPE=$(awk '$2 == "/" { print $3; exit }' /proc/mounts) +if [ "$ROOT_FSTYPE" = "tmpfs" ]; then + mount -o remount,size=100% / +fi diff --git a/init/systemd/cocos-agent.service b/init/systemd/cocos-agent.service index ef74eaf3..384a972d 100644 --- a/init/systemd/cocos-agent.service +++ b/init/systemd/cocos-agent.service @@ -1,7 +1,7 @@ [Unit] Description=Cocos AI agent -After=network.target attestation-service.service log-forwarder.service computation-runner.service egress-proxy.service coco-keyprovider.service -Requires=log-forwarder.service computation-runner.service egress-proxy.service coco-keyprovider.service +After=network.target attestation-service.service log-forwarder.service computation-runner.service egress-proxy.service +Requires=log-forwarder.service computation-runner.service egress-proxy.service Before=docker.service [Service] diff --git a/internal/proto/attestation/v1/attestation_grpc.pb.go b/internal/proto/attestation/v1/attestation_grpc.pb.go index a5f26324..3c7d673c 100644 --- a/internal/proto/attestation/v1/attestation_grpc.pb.go +++ b/internal/proto/attestation/v1/attestation_grpc.pb.go @@ -8,6 +8,7 @@ package attestation import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/manager/README.md b/manager/README.md index 4f36264f..8ab76af3 100644 --- a/manager/README.md +++ b/manager/README.md @@ -49,6 +49,12 @@ The service is configured using the environment variables from the following tab | MANAGER_QEMU_VIRTIO_NET_PCI_ROMFILE | The file path for the ROM image for the virtio-net PCI device. | | | MANAGER_QEMU_DISK_IMG_KERNEL_FILE | The file path for the kernel image. | img/bzImage | | MANAGER_QEMU_DISK_IMG_ROOTFS_FILE | The file path for the root filesystem image. | img/rootfs.cpio.gz | +| MANAGER_QEMU_ENABLE_DISK | Whether to attach a writable qcow2 disk to the CVM. | false | +| MANAGER_QEMU_SRC_DISK_FILE | Path to a qcow2 image whose virtual size is used to size the per-VM writable disk. | img/enc_os.qcow2 | +| MANAGER_QEMU_DST_DISK_FILE | Runtime path of the per-VM writable disk created by the manager. | | +| MANAGER_QEMU_DISK_ID | The QEMU drive identifier for the attached disk. | disk0 | +| MANAGER_QEMU_DISK_FORMAT | The format of the attached disk image. | qcow2 | +| MANAGER_QEMU_DISK_SCSI_ID | The SCSI controller identifier used for the attached disk. | scsi0 | | MANAGER_QEMU_SEV_SNP_ID | The ID for the Secure Encrypted Virtualization (SEV-SNP) device. | sev0 | | MANAGER_QEMU_SEV_SNP_CBITPOS | The position of the C-bit in the physical address. | 51 | | MANAGER_QEMU_SEV_SNP_REDUCED_PHYS_BITS | The number of reduced physical address bits for SEV-SNP. | 1 | @@ -112,6 +118,20 @@ Once the image is built copy the kernel and rootfs image to `cmd/manager/img` fr Another option is to use release versions of EOS that can be downloaded from the [Cocos GitHub repository](https://github.com/ultravioletrs/cocos/releases). +#### Optional writable disk + +If you want the manager to attach a writable disk to each CVM, place a qcow2 reference image at `cmd/manager/img/enc_os.qcow2`, or point `MANAGER_QEMU_SRC_DISK_FILE` to another qcow2 file. + +When `MANAGER_QEMU_ENABLE_DISK=true`, the manager: + +- reads the virtual size of `MANAGER_QEMU_SRC_DISK_FILE` with `qemu-img info` +- creates a per-VM qcow2 disk under `/tmp/cvmDisk-.qcow2` +- sizes the disk to the source image size plus 1 GiB, leaving room for the LUKS header +- attaches the disk through a virtio-scsi controller +- removes the temporary disk again when the VM stops + +`MANAGER_QEMU_DST_DISK_FILE` is primarily a runtime value. In the normal manager flow it is populated automatically and usually does not need to be set manually. + #### Test VM creation ```sh @@ -207,7 +227,7 @@ nc -zv localhost 7020 #### Conclusion -Now you are able to use `Manager` with `Agent`. Namely, `Manager` will create a VM with a separate OVMF variables file on manager `/run` request. +Now you are able to use `Manager` with `Agent`. On each manager `/run` request, the manager creates a VM with a separate OVMF variables file and, when enabled, a per-VM writable qcow2 disk. ### OVMF @@ -284,6 +304,18 @@ MANAGER_QEMU_OVMF_FILE= \ ./build/cocos-manager ``` +To enable writable disk support, start manager like this + +```sh +MANAGER_GRPC_URL=localhost:7001 \ +MANAGER_LOG_LEVEL=debug \ +MANAGER_QEMU_ENABLE_DISK=true \ +MANAGER_QEMU_SRC_DISK_FILE= \ +./build/cocos-manager +``` + +The reference qcow2 image is used to determine the disk size. The manager creates a fresh writable qcow2 disk for each VM under `/tmp` and deletes it on shutdown. + ### Troubleshooting If the `ps aux | grep qemu-system-x86_64` give you something like this @@ -294,16 +326,16 @@ darko 13913 0.0 0.0 0 0 pts/2 Z+ 20:17 0:00 [qemu-system- means that the a QEMU virtual machine that is currently defunct, meaning that it is no longer running. More precisely, the defunct process in the output is also known as a ["zombie" process](https://en.wikipedia.org/wiki/Zombie_process). -You can troubleshoot the VM launch procedure by running directly `qemu-system-x86_64` command. When you run `manager` with `MANAGER_LOG_LEVEL=info` env var set, it prints out the entire command used to launch a VM. The relevant part of the log might look like this +You can troubleshoot the VM launch procedure by running directly `qemu-system-x86_64` command. When you run `manager` with `MANAGER_LOG_LEVEL=info` env var set, it prints out the entire command used to launch a VM. When writable disk support is enabled, the relevant part of the log might look like this ``` -{"level":"info","message":"/usr/bin/qemu-system-x86_64 -enable-kvm -machine q35 -cpu EPYC -smp 4,maxcpus=64 -m 4096M,slots=5,maxmem=30G -drive if=pflash,format=raw,unit=0,file=/usr/share/OVMF/OVMF_CODE.fd,readonly=on -drive if=pflash,format=raw,unit=1,file=img/OVMF_VARS.fd -device virtio-scsi-pci,id=scsi,disable-legacy=on,iommu_platform=true -drive file=img/focal-server-cloudimg-amd64.img,if=none,id=disk0,format=qcow2 -device scsi-hd,drive=disk0 -netdev user,id=vmnic,hostfwd=tcp::2222-:22,hostfwd=tcp::9301-:9031,hostfwd=tcp::7020-:7002 -device virtio-net-pci,disable-legacy=on,iommu_platform=true,netdev=vmnic,romfile= -nographic -monitor pty","ts":"2023-08-14T18:29:19.2653908Z"} +{"level":"info","message":"/usr/bin/qemu-system-x86_64 -enable-kvm -machine q35 -cpu EPYC -smp 4,maxcpus=64 -m 4096M,slots=5,maxmem=30G -drive if=pflash,format=raw,unit=0,file=/usr/share/OVMF/OVMF_CODE.fd,readonly=on -drive if=pflash,format=raw,unit=1,file=/tmp/OVMF_VARS-.fd -netdev user,id=vmnic-,hostfwd=tcp::7020-:7002 -device virtio-net-pci,disable-legacy=on,iommu_platform=true,netdev=vmnic-,addr=0x2,romfile= -drive file=/tmp/cvmDisk-.qcow2,if=none,id=disk0,format=qcow2 -device virtio-scsi-pci,id=scsi0,disable-legacy=on,iommu_platform=true -device scsi-hd,drive=disk0,bus=scsi0.0 -kernel img/bzImage -append quiet console=null -initrd img/rootfs.cpio.gz -nographic -monitor pty","ts":"2026-04-27T00:00:00Z"} ``` You can run the command - the value of the `"message"` key - directly in the terminal: ```sh -/usr/bin/qemu-system-x86_64 -enable-kvm -machine q35 -cpu EPYC -smp 4,maxcpus=64 -m 4096M,slots=5,maxmem=30G -drive if=pflash,format=raw,unit=0,file=/usr/share/OVMF/OVMF_CODE.fd,readonly=on -drive if=pflash,format=raw,unit=1,file=img/OVMF_VARS.fd -device virtio-scsi-pci,id=scsi,disable-legacy=on,iommu_platform=true -drive file=img/focal-server-cloudimg-amd64.img,if=none,id=disk0,format=qcow2 -device scsi-hd,drive=disk0 -netdev user,id=vmnic,hostfwd=tcp::2222-:22,hostfwd=tcp::9301-:9031,hostfwd=tcp::7020-:7002 -device virtio-net-pci,disable-legacy=on,iommu_platform=true,netdev=vmnic,romfile= -nographic -monitor pty +/usr/bin/qemu-system-x86_64 -enable-kvm -machine q35 -cpu EPYC -smp 4,maxcpus=64 -m 4096M,slots=5,maxmem=30G -drive if=pflash,format=raw,unit=0,file=/usr/share/OVMF/OVMF_CODE.fd,readonly=on -drive if=pflash,format=raw,unit=1,file=/tmp/OVMF_VARS-.fd -netdev user,id=vmnic-,hostfwd=tcp::7020-:7002 -device virtio-net-pci,disable-legacy=on,iommu_platform=true,netdev=vmnic-,addr=0x2,romfile= -drive file=/tmp/cvmDisk-.qcow2,if=none,id=disk0,format=qcow2 -device virtio-scsi-pci,id=scsi0,disable-legacy=on,iommu_platform=true -device scsi-hd,drive=disk0,bus=scsi0.0 -kernel img/bzImage -append "quiet console=null" -initrd img/rootfs.cpio.gz -nographic -monitor pty ``` and look for the possible problems. This problems can usually be solved by using the adequate env var assignments. Look in the `manager/qemu/config.go` file to see the recognized env vars. Don't forget to prepend `MANAGER_QEMU_` to the name of the env vars. diff --git a/manager/qemu/config.go b/manager/qemu/config.go index 8ce9aca2..081174d2 100644 --- a/manager/qemu/config.go +++ b/manager/qemu/config.go @@ -4,6 +4,7 @@ package qemu import ( "fmt" + "strings" "github.com/caarlos0/env/v10" ) @@ -48,7 +49,7 @@ type VirtioNetPciConfig struct { ROMFile string `env:"VIRTIO_NET_PCI_ROMFILE"` } -type DiskImgConfig struct { +type KernelConfig struct { KernelFile string `env:"DISK_IMG_KERNEL_FILE" envDefault:"img/bzImage"` RootFsFile string `env:"DISK_IMG_ROOTFS_FILE" envDefault:"img/rootfs.cpio.gz"` } @@ -72,9 +73,18 @@ type IGVMConfig struct { File string `env:"IGVM_FILE" envDefault:"/root/coconut-qemu.igvm"` } +type DiskConfig struct { + SrcFile string `env:"SRC_DISK_FILE" envDefault:"img/enc_os.qcow2"` + DstFile string `env:"DST_DISK_FILE" envDefault:""` + ID string `env:"DISK_ID" envDefault:"disk0"` + Format string `env:"DISK_FORMAT" envDefault:"qcow2"` + SCSIID string `env:"DISK_SCSI_ID" envDefault:"scsi0"` +} + type Config struct { EnableSEVSNP bool EnableTDX bool + EnableDisk bool `env:"ENABLE_DISK" envDefault:"false"` QemuBinPath string `env:"BIN_PATH" envDefault:"qemu-system-x86_64"` UseSudo bool `env:"USE_SUDO" envDefault:"false"` @@ -96,8 +106,11 @@ type Config struct { NetDevConfig VirtioNetPciConfig - // disk - DiskImgConfig + // disk config + DiskConfig + + // kernel and initramfs + KernelConfig // SEV-SNP SEVSNPConfig @@ -123,6 +136,25 @@ type Config struct { EnvMount string `env:"ENV_MOUNT" envDefault:""` } +func (config Config) ValidateBootConfig() error { + if config.EnableDisk { + if strings.TrimSpace(config.DiskConfig.DstFile) == "" { + return fmt.Errorf("disk boot enabled but destination disk image is not set") + } + return nil + } + + if strings.TrimSpace(config.KernelConfig.KernelFile) == "" { + return fmt.Errorf("kernel boot enabled but kernel image is not set") + } + + if strings.TrimSpace(config.KernelConfig.RootFsFile) == "" { + return fmt.Errorf("kernel boot enabled but initramfs image is not set") + } + + return nil +} + func (config Config) ConstructQemuArgs() []string { args := []string{} @@ -179,6 +211,22 @@ func (config Config) ConstructQemuArgs() []string { config.VirtioNetPciConfig.Addr, config.VirtioNetPciConfig.ROMFile)) + if config.EnableDisk { + // disk image + args = append(args, "-drive", + fmt.Sprintf("file=%s,if=none,id=%s,format=%s", + config.DiskConfig.DstFile, + config.DiskConfig.ID, + config.DiskConfig.Format)) + args = append(args, "-device", + fmt.Sprintf("virtio-scsi-pci,id=%s,disable-legacy=on,iommu_platform=true", + config.DiskConfig.SCSIID)) + args = append(args, "-device", + fmt.Sprintf("scsi-hd,drive=%s,bus=%s.0", + config.DiskConfig.ID, + config.DiskConfig.SCSIID)) + } + // SEV-SNP if config.EnableSEVSNP { sevSnpType := "sev-snp-guest" @@ -233,9 +281,11 @@ func (config Config) ConstructQemuArgs() []string { args = append(args, "-nodefaults") } - args = append(args, "-kernel", config.DiskImgConfig.KernelFile) - args = append(args, "-append", config.KernelCommandLine) - args = append(args, "-initrd", config.DiskImgConfig.RootFsFile) + if !config.EnableDisk { + args = append(args, "-kernel", config.KernelConfig.KernelFile) + args = append(args, "-append", config.KernelCommandLine) + args = append(args, "-initrd", config.KernelConfig.RootFsFile) + } // display if config.NoGraphic { diff --git a/manager/qemu/config_test.go b/manager/qemu/config_test.go index d49d96c8..1b92a87e 100644 --- a/manager/qemu/config_test.go +++ b/manager/qemu/config_test.go @@ -51,7 +51,7 @@ func TestConstructQemuArgs(t *testing.T) { IOMMUPlatform: true, Addr: "0x2", }, - DiskImgConfig: DiskImgConfig{ + KernelConfig: KernelConfig{ KernelFile: "img/bzImage", RootFsFile: "img/rootfs.cpio.gz", }, @@ -115,7 +115,7 @@ func TestConstructQemuArgs(t *testing.T) { IOMMUPlatform: true, Addr: "0x2", }, - DiskImgConfig: DiskImgConfig{ + KernelConfig: KernelConfig{ KernelFile: "img/bzImage", RootFsFile: "img/rootfs.cpio.gz", }, @@ -194,3 +194,154 @@ func TestConstructQemuArgs_HostData(t *testing.T) { t.Errorf("ConstructQemuArgs() did not contain expected SEV-SNP configuration with host data") } } + +func TestConstructQemuArgs_TDX(t *testing.T) { + config := Config{ + EnableKVM: true, + EnableTDX: true, + Machine: "q35", + CPU: "EPYC", + SMPCount: 4, + MaxCPUs: 64, + MemID: "ram1", + MemoryConfig: MemoryConfig{ + Size: "4096M", + Slots: 8, + Max: "64G", + }, + NetDevConfig: NetDevConfig{ + ID: "vmnic", + HostFwdAgent: 7020, + GuestFwdAgent: 7002, + }, + VirtioNetPciConfig: VirtioNetPciConfig{ + DisableLegacy: "on", + IOMMUPlatform: true, + Addr: "0x2", + }, + TDXConfig: TDXConfig{ + ID: "tdx0", + QuoteGenerationPort: 4050, + OVMF: "/usr/share/ovmf/OVMF.fd", + }, + KernelConfig: KernelConfig{ + KernelFile: "img/bzImage", + RootFsFile: "img/rootfs.cpio.gz", + }, + KernelCommandLine: "quiet console=null", + NoGraphic: true, + Monitor: "pty", + } + + expected := []string{ + "-enable-kvm", + "-machine", "q35", + "-cpu", "EPYC", + "-smp", "4,maxcpus=64", + "-m", "4096M,slots=8,maxmem=64G", + "-netdev", "user,id=vmnic,hostfwd=tcp::7020-:7002", + "-device", "virtio-net-pci,disable-legacy=on,iommu_platform=true,netdev=vmnic,addr=0x2,romfile=", + "-object", "{\"qom-type\":\"tdx-guest\",\"id\":\"tdx0\",\"quote-generation-socket\":{\"type\": \"vsock\", \"cid\":\"2\",\"port\":\"4050\"}}", + "-machine", "confidential-guest-support=tdx0,memory-backend=ram1,hpet=off", + "-object", "memory-backend-memfd,id=ram1,size=4096M,share=true,prealloc=false", + "-bios", "/usr/share/ovmf/OVMF.fd", + "-nodefaults", + "-kernel", "img/bzImage", + "-append", "quiet console=null", + "-initrd", "img/rootfs.cpio.gz", + "-nographic", + "-monitor", "pty", + } + + result := config.ConstructQemuArgs() + if !reflect.DeepEqual(result, expected) { + t.Errorf("ConstructQemuArgs() = %v, want %v", result, expected) + } +} + +func TestConstructQemuArgs_DiskBootSkipsKernelAndInitrd(t *testing.T) { + config := Config{ + EnableKVM: true, + EnableDisk: true, + Machine: "q35", + CPU: "EPYC", + SMPCount: 4, + MaxCPUs: 64, + MemID: "ram1", + MemoryConfig: MemoryConfig{ + Size: "2048M", + Slots: 5, + Max: "30G", + }, + NetDevConfig: NetDevConfig{ + ID: "vmnic", + HostFwdAgent: 7020, + GuestFwdAgent: 7002, + }, + VirtioNetPciConfig: VirtioNetPciConfig{ + DisableLegacy: "on", + IOMMUPlatform: true, + Addr: "0x2", + }, + DiskConfig: DiskConfig{ + DstFile: "img/disk.img", + ID: "disk0", + Format: "qcow2", + SCSIID: "scsi0", + }, + KernelConfig: KernelConfig{ + KernelFile: "img/bzImage", + RootFsFile: "img/rootfs.cpio.gz", + }, + NoGraphic: true, + Monitor: "pty", + } + + result := config.ConstructQemuArgs() + + for _, forbidden := range []string{"-kernel", "-append", "-initrd"} { + for _, arg := range result { + if arg == forbidden { + t.Fatalf("ConstructQemuArgs() unexpectedly contained %s during disk boot: %v", forbidden, result) + } + } + } +} + +func TestConstructQemuArgs_EnableDisk(t *testing.T) { + config := Config{ + EnableDisk: true, + DiskConfig: DiskConfig{ + SrcFile: "img/enc_os.qcow2", + DstFile: "img/enc_os_dst.qcow2", + ID: "disk0", + Format: "qcow2", + SCSIID: "scsi0", + }, + } + + result := config.ConstructQemuArgs() + + expected := []string{ + "-drive", "file=img/enc_os_dst.qcow2,if=none,id=disk0,format=qcow2", + "-device", "virtio-scsi-pci,id=scsi0,disable-legacy=on,iommu_platform=true", + "-device", "scsi-hd,drive=disk0,bus=scsi0.0", + } + + var found []bool = make([]bool, len(expected)) + for i, arg := range result { + for j := 0; j < len(expected); j += 2 { + if arg == expected[j] && i+1 < len(result) && result[i+1] == expected[j+1] { + found[j] = true + found[j+1] = true + break + } + } + } + + for j, f := range found { + if !f { + t.Errorf("ConstructQemuArgs() did not contain expected disk configuration: %s", expected[j]) + } + } +} diff --git a/manager/qemu/vm.go b/manager/qemu/vm.go index bf603a54..8e8ebc14 100644 --- a/manager/qemu/vm.go +++ b/manager/qemu/vm.go @@ -3,10 +3,12 @@ package qemu import ( + "encoding/json" "fmt" "log/slog" "os" "os/exec" + "path/filepath" "strings" "syscall" "time" @@ -18,12 +20,15 @@ import ( ) const ( - firmwareVars = "OVMF_VARS" - KernelFile = "bzImage" - rootfsFile = "rootfs.cpio" - tmpDir = "/tmp" - interval = 5 * time.Second - shutdownTimeout = 30 * time.Second + firmwareVars = "OVMF_VARS" + KernelFile = "bzImage" + rootfsFile = "rootfs.cpio" + tmpDir = "/tmp" + diskDstName = "cvmDisk" + interval = 5 * time.Second + shutdownTimeout = 30 * time.Second + encryptedPartitionSizeDeltaGB = 1 + sourceDiskFormat = "qcow2" ) type VMInfo struct { @@ -39,6 +44,10 @@ type qemuVM struct { vm.StateMachine } +type qemuInfo struct { + VirtualSize int64 `json:"virtual-size"` +} + func NewVM(config any, cvmId string, logger *slog.Logger) vm.VM { return &qemuVM{ vmi: config.(VMInfo), @@ -75,6 +84,44 @@ func (v *qemuVM) Start() (err error) { v.vmi.Config.OVMFVarsConfig.File = dstFile } + if v.vmi.Config.EnableDisk { + srcDiskFile, err := filepath.Abs(v.vmi.Config.SrcFile) + if err != nil { + return err + } + + sizeGB, err := GetVirtualSizeGB(srcDiskFile) + if err != nil { + return err + } + + dstDiskFile := fmt.Sprintf("%s/%s-%s.%s", tmpDir, diskDstName, id, v.vmi.Config.DiskConfig.Format) + sizeArg := fmt.Sprintf("%dG", sizeGB+encryptedPartitionSizeDeltaGB) + + cmd := exec.Command( + "qemu-img", + "convert", + "-f", sourceDiskFormat, + "-O", v.vmi.Config.DiskConfig.Format, + srcDiskFile, + dstDiskFile, + ) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("qemu-img convert failed: %w: %s", err, string(out)) + } + + cmd = exec.Command( + "qemu-img", + "resize", + dstDiskFile, + sizeArg, + ) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("qemu-img resize failed: %w: %s", err, string(out)) + } + v.vmi.Config.DstFile = dstDiskFile + } + exe, args, err := v.executableAndArgs() if err != nil { return err @@ -111,6 +158,14 @@ func (v *qemuVM) Stop() error { } } + if v.vmi.Config.EnableDisk { + if v.vmi.Config.DstFile != "" { + if err := os.RemoveAll(v.vmi.Config.DstFile); err != nil { + return fmt.Errorf("failed to remove disk file: %v", err) + } + } + } + done := make(chan error, 1) go func() { _, err := v.cmd.Process.Wait() @@ -156,6 +211,10 @@ func (v *qemuVM) executableAndArgs() (string, []string, error) { return "", nil, err } + if err := v.vmi.Config.ValidateBootConfig(); err != nil { + return "", nil, err + } + args := v.vmi.Config.ConstructQemuArgs() if v.vmi.Config.UseSudo { @@ -231,3 +290,32 @@ func TDXEnabledOnHost() bool { return TDXEnabled(string(cpuinfo), string(kernelParam)) } + +func GetVirtualSizeBytes(path string) (int64, error) { + cmd := exec.Command("qemu-img", "info", "--output=json", path) + out, err := cmd.Output() + if err != nil { + return 0, fmt.Errorf("qemu-img info failed: %w", err) + } + + var info qemuInfo + if err := json.Unmarshal(out, &info); err != nil { + return 0, fmt.Errorf("failed to parse qemu-img JSON: %w", err) + } + + if info.VirtualSize <= 0 { + return 0, fmt.Errorf("invalid virtual size: %d", info.VirtualSize) + } + + return info.VirtualSize, nil +} + +func GetVirtualSizeGB(path string) (int, error) { + bytes, err := GetVirtualSizeBytes(path) + if err != nil { + return 0, err + } + + gb := (bytes + (1<<30 - 1)) >> 30 + return int(gb), nil +} diff --git a/manager/qemu/vm_test.go b/manager/qemu/vm_test.go index 698b513d..d4db470a 100644 --- a/manager/qemu/vm_test.go +++ b/manager/qemu/vm_test.go @@ -3,9 +3,12 @@ package qemu import ( + "fmt" "log/slog" "os" "os/exec" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -15,6 +18,31 @@ import ( const testComputationID = "test-computation" +func cleanupStrayQcow2(t *testing.T) { + t.Helper() + + wd, err := os.Getwd() + if err != nil { + t.Fatalf("failed to get working directory: %v", err) + } + + t.Cleanup(func() { + _ = os.Remove(filepath.Join(wd, "qcow2")) + }) +} + +func requireTempFile(t *testing.T, path string) { + t.Helper() + + f, err := os.Create(path) + if err != nil { + t.Fatalf("failed to create temp file %s: %v", path, err) + } + if err := f.Close(); err != nil { + t.Fatalf("failed to close temp file %s: %v", path, err) + } +} + func TestNewVM(t *testing.T) { config := VMInfo{Config: Config{}} @@ -35,6 +63,10 @@ func TestStart(t *testing.T) { File: tmpFile.Name(), }, QemuBinPath: "echo", + KernelConfig: KernelConfig{ + KernelFile: "img/bzImage", + RootFsFile: "img/rootfs.cpio.gz", + }, }} vm := NewVM(config, testComputationID, slog.Default()).(*qemuVM) @@ -58,6 +90,10 @@ func TestStartSudo(t *testing.T) { }, QemuBinPath: "echo", UseSudo: true, + KernelConfig: KernelConfig{ + KernelFile: "img/bzImage", + RootFsFile: "img/rootfs.cpio.gz", + }, }} vm := NewVM(config, testComputationID, slog.Default()).(*qemuVM) @@ -69,6 +105,136 @@ func TestStartSudo(t *testing.T) { _ = vm.Stop() } +func TestStart_EnableDisk(t *testing.T) { + cleanupStrayQcow2(t) + + toolsDir := t.TempDir() + convertLogFile := filepath.Join(toolsDir, "qemu-img-convert.log") + resizeLogFile := filepath.Join(toolsDir, "qemu-img-resize.log") + srcDiskFile := filepath.Join(toolsDir, "enc_os.qcow2") + requireTempFile(t, srcDiskFile) + + writeFakeExecutable(t, toolsDir, "qemu-img", fmt.Sprintf(`#!/bin/sh +case "$1" in +info) + printf '%%s' '{"virtual-size":2147483648}' + ;; +convert) + printf '%%s\n' "$@" > %q + dst="$7" + : > "$dst" + ;; +resize) + printf '%%s\n' "$@" > %q + ;; +*) + echo "unexpected subcommand: $1" >&2 + exit 2 + ;; +esac +`, convertLogFile, resizeLogFile)) + + writeFakeExecutable(t, toolsDir, "fake-qemu", `#!/bin/sh +trap 'exit 0' TERM INT +while :; do + sleep 1 +done +`) + prependPath(t, toolsDir) + + config := VMInfo{Config: Config{ + EnableTDX: true, + EnableDisk: true, + QemuBinPath: "fake-qemu", + DiskConfig: DiskConfig{ + SrcFile: srcDiskFile, + ID: "disk0", + Format: "qcow2", + SCSIID: "scsi0", + }, + }} + + vm := NewVM(config, testComputationID, slog.Default()).(*qemuVM) + + err := vm.Start() + assert.NoError(t, err) + assert.NotNil(t, vm.cmd) + assert.Contains(t, vm.vmi.Config.DstFile, filepath.Join(tmpDir, diskDstName)) + _, err = os.Stat(vm.vmi.Config.DstFile) + assert.NoError(t, err) + + loggedArgs, err := os.ReadFile(convertLogFile) + assert.NoError(t, err) + assert.Equal(t, []string{ + "convert", + "-f", + "qcow2", + "-O", + "qcow2", + srcDiskFile, + vm.vmi.Config.DstFile, + }, strings.Fields(string(loggedArgs))) + + loggedArgs, err = os.ReadFile(resizeLogFile) + assert.NoError(t, err) + assert.Equal(t, []string{ + "resize", + vm.vmi.Config.DstFile, + "3G", + }, strings.Fields(string(loggedArgs))) + + err = vm.Stop() + assert.NoError(t, err) + _, err = os.Stat(vm.vmi.Config.DstFile) + assert.Error(t, err) + assert.True(t, os.IsNotExist(err)) +} + +func TestStart_EnableDiskCreateError(t *testing.T) { + cleanupStrayQcow2(t) + + toolsDir := t.TempDir() + srcDiskFile := filepath.Join(toolsDir, "enc_os.qcow2") + requireTempFile(t, srcDiskFile) + + writeFakeExecutable(t, toolsDir, "qemu-img", `#!/bin/sh +case "$1" in +info) + printf '%s' '{"virtual-size":2147483648}' + ;; +convert) + echo 'disk create failed' >&2 + exit 1 + ;; +resize) + exit 0 + ;; +*) + echo "unexpected subcommand: $1" >&2 + exit 2 + ;; +esac +`) + prependPath(t, toolsDir) + + config := VMInfo{Config: Config{ + EnableTDX: true, + EnableDisk: true, + QemuBinPath: "fake-qemu", + DiskConfig: DiskConfig{ + SrcFile: srcDiskFile, + }, + }} + + vm := NewVM(config, testComputationID, slog.Default()).(*qemuVM) + + err := vm.Start() + assert.Error(t, err) + assert.ErrorContains(t, err, "qemu-img convert failed") + assert.ErrorContains(t, err, "disk create failed") + assert.Nil(t, vm.cmd) +} + func TestStop(t *testing.T) { t.Run("success", func(t *testing.T) { cmd := exec.Command("echo", "test") @@ -102,6 +268,42 @@ func TestStop(t *testing.T) { StateMachine: sm, } + err = vm.Stop() + assert.NoError(t, err) + }) + t.Run("disk enable", func(t *testing.T) { + dir := t.TempDir() + dst := filepath.Join(dir, "disk.qcow2") + + f, err := os.Create(dst) + if err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + + cmd := exec.Command("echo", "test") + err = cmd.Start() + assert.NoError(t, err) + sm := new(mocks.StateMachine) + sm.On("Transition", pkgmanager.StopComputationRun).Return(nil) + + vm := &qemuVM{ + vmi: VMInfo{ + Config: Config{ + EnableDisk: true, + DiskConfig: DiskConfig{ + DstFile: dst, + }, + }, + }, + cmd: &exec.Cmd{ + Process: cmd.Process, + }, + StateMachine: sm, + } + err = vm.Stop() assert.NoError(t, err) }) @@ -110,7 +312,13 @@ func TestStop(t *testing.T) { func TestSetProcess(t *testing.T) { vm := &qemuVM{ vmi: VMInfo{ - Config: Config{QemuBinPath: "echo"}, // Use 'echo' as a dummy QEMU binary + Config: Config{ + QemuBinPath: "echo", // Use 'echo' as a dummy QEMU binary + KernelConfig: KernelConfig{ + KernelFile: "img/bzImage", + RootFsFile: "img/rootfs.cpio.gz", + }, + }, }, } @@ -175,9 +383,156 @@ func TestTDXEnabled(t *testing.T) { } func TestSEVSNPEnabledOnHost(t *testing.T) { - assert.False(t, SEVSNPEnabledOnHost()) + cpuinfo, cpuErr := os.ReadFile("/proc/cpuinfo") + kernelParam, kernelErr := os.ReadFile("/sys/module/kvm_amd/parameters/sev_snp") + + expected := false + if cpuErr == nil && kernelErr == nil { + expected = SEVSNPEnabled(string(cpuinfo), string(kernelParam)) + } + + assert.Equal(t, expected, SEVSNPEnabledOnHost()) } func TestTDXEnabledOnHost(t *testing.T) { - assert.False(t, TDXEnabledOnHost()) + cpuinfo, cpuErr := os.ReadFile("/proc/cpuinfo") + kernelParam, kernelErr := os.ReadFile("/sys/module/kvm_intel/parameters/tdx") + + expected := false + if cpuErr == nil && kernelErr == nil { + expected = TDXEnabled(string(cpuinfo), string(kernelParam)) + } + + assert.Equal(t, expected, TDXEnabledOnHost()) +} + +func TestGetVirtualSizeBytes_Success(t *testing.T) { + cleanup := writeFakeQemuImg(t, `{"virtual-size":2147483648}`, 0) // 2 GiB + defer cleanup() + + got, err := GetVirtualSizeBytes("whatever.qcow2") + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if got != 2147483648 { + t.Fatalf("expected 2147483648, got %d", got) + } +} + +func TestGetVirtualSizeBytes_CommandFailure(t *testing.T) { + cleanup := writeFakeQemuImg(t, `{"virtual-size":2147483648}`, 1) // non-zero exit + defer cleanup() + + _, err := GetVirtualSizeBytes("whatever.qcow2") + if err == nil { + t.Fatalf("expected error, got nil") + } + if !strings.Contains(err.Error(), "qemu-img info failed") { + t.Fatalf("expected wrapped error to contain %q, got %q", "qemu-img info failed", err.Error()) + } +} + +func TestGetVirtualSizeBytes_InvalidJSON(t *testing.T) { + cleanup := writeFakeQemuImg(t, `not-json`, 0) + defer cleanup() + + _, err := GetVirtualSizeBytes("whatever.qcow2") + if err == nil { + t.Fatalf("expected error, got nil") + } + if !strings.Contains(err.Error(), "failed to parse qemu-img JSON") { + t.Fatalf("expected error to contain %q, got %q", "failed to parse qemu-img JSON", err.Error()) + } +} + +func TestGetVirtualSizeBytes_InvalidVirtualSize(t *testing.T) { + cleanup := writeFakeQemuImg(t, `{"virtual-size":0}`, 0) + defer cleanup() + + _, err := GetVirtualSizeBytes("whatever.qcow2") + if err == nil { + t.Fatalf("expected error, got nil") + } + if !strings.Contains(err.Error(), "invalid virtual size") { + t.Fatalf("expected error to contain %q, got %q", "invalid virtual size", err.Error()) + } +} + +func TestGetVirtualSizeGB_RoundsUp(t *testing.T) { + tests := []struct { + name string + virtualSz int64 + wantGB int + }{ + {"exact_1GiB", 1 << 30, 1}, + {"one_byte_over", (1 << 30) + 1, 2}, + {"just_under_2GiB", (2 << 30) - 1, 2}, + {"exact_2GiB", 2 << 30, 2}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cleanup := writeFakeQemuImg(t, fmt.Sprintf(`{"virtual-size":%d}`, tc.virtualSz), 0) + defer cleanup() + + got, err := GetVirtualSizeGB("whatever.qcow2") + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if got != tc.wantGB { + t.Fatalf("expected %d, got %d", tc.wantGB, got) + } + }) + } +} + +func writeFakeQemuImg(t *testing.T, stdout string, exitCode int) func() { + dir := t.TempDir() + fake := filepath.Join(dir, "qemu-img") + + script := fmt.Sprintf(`#!/bin/sh +# Minimal fake for: qemu-img info --output=json +if [ "$1" != "info" ]; then + echo "unexpected subcommand: $1" >&2 + exit 2 +fi + +# always print provided stdout, even if empty +printf '%s' %q +exit %d +`, stdout, stdout, exitCode) + + if err := os.WriteFile(fake, []byte(script), 0o755); err != nil { + t.Fatalf("failed to write fake qemu-img: %v", err) + } + + oldPath := os.Getenv("PATH") + if err := os.Setenv("PATH", dir+string(os.PathListSeparator)+oldPath); err != nil { + t.Fatalf("failed to set PATH: %v", err) + } + + return func() { + _ = os.Setenv("PATH", oldPath) + } +} + +func writeFakeExecutable(t *testing.T, dir, name, script string) { + t.Helper() + + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("failed to write fake executable %q: %v", name, err) + } +} + +func prependPath(t *testing.T, dir string) { + t.Helper() + + oldPath := os.Getenv("PATH") + if err := os.Setenv("PATH", dir+string(os.PathListSeparator)+oldPath); err != nil { + t.Fatalf("failed to set PATH: %v", err) + } + t.Cleanup(func() { + _ = os.Setenv("PATH", oldPath) + }) }