From 811a6062b225063646373efe7ed479ae2ee284cc Mon Sep 17 00:00:00 2001 From: Anamika AggarwaL Date: Wed, 8 Jul 2026 15:36:01 +0530 Subject: [PATCH 1/8] feat(firecracker): launch with control socket instead of config file Firecracker currently boots via --no-api --config-file, which disables its control socket entirely. Switch to --api-sock so the socket stays open and reachable after the guest starts, instead of driving everything through a static config file. Signed-off-by: Anamika AggarwaL --- pkg/unikontainers/hypervisors/firecracker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 9588a45bf..128012ab2 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -108,9 +108,9 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel // options in FC, since the string return value of the Monitor related // functions in the unikernel interface do not integrate well with FC's // json configuration. - cmdString := fc.Path() + " --no-api --config-file " + apiSockPath := filepath.Join("/tmp/", args.ContainerID+".sock") + cmdString := fc.Path() + " --api-sock " + apiSockPath JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) - cmdString += JSONConfigFile if !args.Seccomp { cmdString += " --no-seccomp" } From ae3bf54a287bc021fc42be2803ca633d03c159d7 Mon Sep 17 00:00:00 2001 From: Anamika AggarwaL Date: Sat, 11 Jul 2026 13:04:41 +0530 Subject: [PATCH 2/8] feat(firecracker): allow configuring the control socket path Add an optional socket_path field under a monitor's configuration section so users can override where the control socket is created. Falls back to today's default path when unset. The field is added to the shared MonitorConfig type, so it is available to any monitor, but only Firecracker's BuildExecCmd reads it in this change, since it is the only monitor with a control socket right now. Signed-off-by: Anamika AggarwaL --- pkg/unikontainers/hypervisors/firecracker.go | 5 ++++- pkg/unikontainers/types/types.go | 8 +++++--- pkg/unikontainers/unikontainers.go | 2 ++ pkg/unikontainers/urunc_config.go | 3 +++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 128012ab2..638fd6c64 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -108,7 +108,10 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel // options in FC, since the string return value of the Monitor related // functions in the unikernel interface do not integrate well with FC's // json configuration. - apiSockPath := filepath.Join("/tmp/", args.ContainerID+".sock") + apiSockPath := args.SocketPath + if apiSockPath == "" { + apiSockPath = filepath.Join("/tmp/", args.ContainerID+".sock") + } cmdString := fc.Path() + " --api-sock " + apiSockPath JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) if !args.Seccomp { diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2cc..e496b6493 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -96,6 +96,7 @@ type UnikernelParams struct { // FIXME: add extra fields if required by additional VMM's type ExecArgs struct { ContainerID string // The container ID + SocketPath string // Optional user-configured path for the monitor's control socket. Empty means the monitor uses its default. Environment []string // The environment variables of the monitor Command string // The unikernel's command line Seccomp bool // Enable or disable seccomp filters for the VMM @@ -133,7 +134,8 @@ type ExtraBinConfig struct { type MonitorConfig struct { DefaultMemoryMB uint `toml:"default_memory_mb"` DefaultVCPUs uint `toml:"default_vcpus"` - BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary - DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) - Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary + DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) + Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket. If not specified, the monitor uses its default. } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index fc297f937..cc347ebb2 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -354,10 +354,12 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath // ExecArgs vmmArgs := types.ExecArgs{ ContainerID: u.State.ID, + SocketPath: socketPath, UnikernelPath: unikernelPath, InitrdPath: initrdPath, Seccomp: true, // Enable Seccomp by default diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 5f21d106e..17724c030 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -127,6 +127,7 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"socket_path"] = hvCfg.SocketPath } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -180,6 +181,8 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } else { hvCfg.Vhost = boolVal } + case "socket_path": + hvCfg.SocketPath = val } cfg.Monitors[hv] = hvCfg } From 9b4f4ffe5fc953d6aaf768fe7e45ec9d8722a2be Mon Sep 17 00:00:00 2001 From: Anamika AggarwaL Date: Sat, 11 Jul 2026 13:11:37 +0530 Subject: [PATCH 3/8] docs(configuration): document the socket_path monitor option Add the new socket_path field to the Monitor Options table and the Firecracker example, matching the existing style for path/data_path. Signed-off-by: Anamika AggarwaL --- docs/configuration.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..670fc536e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,6 +112,7 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | +| `socket_path` | string | (empty) | Optional custom path for the monitor's control socket. If not specified, the monitor uses its default. Currently only used by Firecracker. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for @@ -130,6 +131,7 @@ data_path = "/usr/local/share/" default_memory_mb = 512 default_vcpus = 2 path = "/opt/firecracker/firecracker" +socket_path = "/run/urunc/fc.sock" ``` ### Extra binaries Configuration From 7efd0be897ec5165b57f22cf2dc7547885b81708 Mon Sep 17 00:00:00 2001 From: Anamika AggarwaL Date: Mon, 13 Jul 2026 12:20:40 +0530 Subject: [PATCH 4/8] feat(firecracker): add boot_mode to choose config-file-based or API-based boot Add an optional boot_mode field under a monitor's configuration. "api" (default) keeps today's behavior: drive the monitor's boot over its control socket. "config-file" lets Firecracker boot itself from the JSON config file, using the socket only for communication after the guest is running. Currently only used by Firecracker. Signed-off-by: Anamika AggarwaL --- docs/configuration.md | 2 ++ pkg/unikontainers/hypervisors/firecracker.go | 5 +++++ pkg/unikontainers/types/types.go | 2 ++ pkg/unikontainers/unikontainers.go | 2 ++ pkg/unikontainers/urunc_config.go | 3 +++ 5 files changed, 14 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 670fc536e..689a6bdc1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,6 +113,7 @@ Each monitor subsection supports the following options: | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | | `socket_path` | string | (empty) | Optional custom path for the monitor's control socket. If not specified, the monitor uses its default. Currently only used by Firecracker. | +| `boot_mode` | string | `api` | Optional: `api` drives the monitor's boot over its control socket, `config-file` lets the monitor boot itself from a config file (socket only used afterward). Currently only used by Firecracker. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for @@ -132,6 +133,7 @@ default_memory_mb = 512 default_vcpus = 2 path = "/opt/firecracker/firecracker" socket_path = "/run/urunc/fc.sock" +boot_mode = "config-file" ``` ### Extra binaries Configuration diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 638fd6c64..cd7f0b8a5 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -114,6 +114,11 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel } cmdString := fc.Path() + " --api-sock " + apiSockPath JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) + if args.BootMode == "config-file" { + // config-file-based: Firecracker boots itself from the JSON config file + // below; the socket stays open only for use after the guest is running. + cmdString += " --config-file " + JSONConfigFile + } if !args.Seccomp { cmdString += " --no-seccomp" } diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index e496b6493..5ef6faffb 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -97,6 +97,7 @@ type UnikernelParams struct { type ExecArgs struct { ContainerID string // The container ID SocketPath string // Optional user-configured path for the monitor's control socket. Empty means the monitor uses its default. + BootMode string // "api" (default) drives the monitor over its control socket. "config-file" lets the monitor boot itself from a config file, socket only for later use. Environment []string // The environment variables of the monitor Command string // The unikernel's command line Seccomp bool // Enable or disable seccomp filters for the VMM @@ -138,4 +139,5 @@ type MonitorConfig struct { DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket. If not specified, the monitor uses its default. + BootMode string `toml:"boot_mode,omitempty"` // Optional: "api" (default) or "config-file". Currently only used by Firecracker. } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index cc347ebb2..542764525 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -355,11 +355,13 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB socketPath := u.UruncCfg.Monitors[vmmType].SocketPath + bootMode := u.UruncCfg.Monitors[vmmType].BootMode // ExecArgs vmmArgs := types.ExecArgs{ ContainerID: u.State.ID, SocketPath: socketPath, + BootMode: bootMode, UnikernelPath: unikernelPath, InitrdPath: initrdPath, Seccomp: true, // Enable Seccomp by default diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 17724c030..52a028464 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -128,6 +128,7 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) cfgMap[prefix+"socket_path"] = hvCfg.SocketPath + cfgMap[prefix+"boot_mode"] = hvCfg.BootMode } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -183,6 +184,8 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } case "socket_path": hvCfg.SocketPath = val + case "boot_mode": + hvCfg.BootMode = val } cfg.Monitors[hv] = hvCfg } From a361b4ca4cc6efcc302ad921bf60680b750c8cb5 Mon Sep 17 00:00:00 2001 From: Anamika AggarwaL Date: Mon, 13 Jul 2026 13:33:22 +0530 Subject: [PATCH 5/8] feat(firecracker): drive API-based boot as a supervised child process Stop syscall.Exec-ing into Firecracker for boot_mode=api. Instead, spawn it as a child, configure it over its control socket using the existing config-building logic (now shared with the config-file path), start the guest, then supervise the child: forward SIGTERM/SIGINT and exit with its status once it exits. The child inherits whatever confinement changeRoot already set up on this process (pivot_root or chroot), so no separate confinement step is needed here. Add HasNetwork() as a cheap, early check for whether a container has a network interface, separate from the full (expensive) NetworkSetup. Rootfs prep only needs this cheap fact (see prepareMonRootfs's needsTAP argument), not the finished network, so for boot_mode=api, rootfs prep now runs concurrently with the real network setup, joining just before unikernel.Init (every supported unikernel type needs the resolved network result to build the guest's boot command line). Also fixes a real bug this surfaced: createTapDevice's file descriptor was opened O_CLOEXEC and relied on syscall.Exec to close it as a side effect of replacing the process. Since urunc no longer execs away for boot_mode=api, that fd was staying open indefinitely, blocking Firecracker's own attempt to attach to the same single-queue tap device ("Resource busy"). The device is created with TUNSETPERSIST, so it's safe to close the fd explicitly once setup is done. Signed-off-by: Anamika AggarwaL --- pkg/network/network.go | 17 +++ pkg/network/network_dynamic.go | 11 ++ pkg/network/network_static.go | 11 ++ pkg/unikontainers/hypervisors/firecracker.go | 126 +++++++++++++++-- .../hypervisors/firecracker_client.go | 132 ++++++++++++++++++ pkg/unikontainers/unikontainers.go | 84 +++++++++-- 6 files changed, 357 insertions(+), 24 deletions(-) create mode 100644 pkg/unikontainers/hypervisors/firecracker_client.go diff --git a/pkg/network/network.go b/pkg/network/network.go index 375af431b..67fad11ba 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -38,6 +38,11 @@ type UnikernelNetworkInfo struct { EthDevice Interface } type Manager interface { + // HasNetwork does a cheap check for whether this container has a network + // interface to configure, without doing the more expensive tap device + // creation and configuration. It lets callers learn this fact early, + // before the full NetworkSetup (which does the expensive work) finishes. + HasNetwork() (bool, error) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) } @@ -113,6 +118,18 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L } } + // LinkAdd opens the tap device's file descriptor with O_CLOEXEC and sets + // TUNSETPERSIST, so the interface survives independent of any open fd. + // We don't need to keep it open past this point (the rest of setup, and + // whatever process later attaches to this tap by name, uses netlink/its + // own independent open, not this fd) - closing it now, rather than + // relying on a future exec to do it, matters because a process that + // keeps this fd open (instead of exec-ing away) would otherwise block + // anything else from opening the same single-queue tap device. + for _, tapFd := range tapLink.Fds { + _ = tapFd.Close() + } + err = netlink.LinkSetMTU(tapLink, mtu) if err != nil { return nil, fmt.Errorf("failed to set tap device MTU to %d: %w", mtu, err) diff --git a/pkg/network/network_dynamic.go b/pkg/network/network_dynamic.go index 1d6a2237b..f1f5c991b 100644 --- a/pkg/network/network_dynamic.go +++ b/pkg/network/network_dynamic.go @@ -32,6 +32,17 @@ type DynamicNetwork struct { // FIXME: CUrrently only one tap device per netns can provide functional networking. We need to find a proper way to handle networking // for multiple unikernels in the same pod/network namespace. // See: https://github.com/urunc-dev/urunc/issues/13 +// HasNetwork checks, cheaply, whether a container network interface exists +// in the current netns, without creating the tap device or doing any of the +// other, more expensive setup NetworkSetup does. +func (n DynamicNetwork) HasNetwork() (bool, error) { + _, err := discoverContainerIface() + if err != nil { + return false, nil + } + return true, nil +} + func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { tapIndex, err := getTapIndex() if err != nil { diff --git a/pkg/network/network_static.go b/pkg/network/network_static.go index fa46608cc..dee813680 100644 --- a/pkg/network/network_static.go +++ b/pkg/network/network_static.go @@ -88,6 +88,17 @@ func setNATRule(iface string, sourceIP string) error { return nil } +// HasNetwork checks, cheaply, whether a container network interface exists +// in the current netns, without creating the tap device or doing any of the +// other, more expensive setup NetworkSetup does. +func (n StaticNetwork) HasNetwork() (bool, error) { + _, err := discoverContainerIface() + if err != nil { + return false, nil + } + return true, nil +} + func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { newTapName := strings.ReplaceAll(DefaultTap, "X", "0") addTCRules := false diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index cd7f0b8a5..d775fc3ee 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -15,11 +15,17 @@ package hypervisors import ( + "context" "encoding/json" + "errors" "fmt" "os" + "os/exec" + "os/signal" "path/filepath" "strings" + "syscall" + "time" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" @@ -123,6 +129,31 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel cmdString += " --no-seccomp" } + FCConfig := buildFirecrackerConfig(args, ukernel) + FCConfigJSON, err := json.Marshal(FCConfig) + if err != nil { + return nil, fmt.Errorf("failed to marshal Firecracker config: %w", err) + } + if err = os.WriteFile(JSONConfigFile, FCConfigJSON, 0o644); err != nil { //nolint: gosec + return nil, fmt.Errorf("failed to save Firecracker json config: %w", err) + } + vmmLog.WithField("Json", string(FCConfigJSON)).Debug("Firecracker json config") + + exArgs := strings.Split(cmdString, " ") + return exArgs, nil +} + +// PreExec performs pre-execution setup. Firecracker has no special pre-exec requirements. +func (fc *Firecracker) PreExec(_ types.ExecArgs) error { + return nil +} + +// buildFirecrackerConfig builds the microVM configuration from args and +// ukernel. Used both to write the JSON config file (config-file-based boot) +// and to drive the same configuration over the API socket (API-based boot), +// so both paths always agree on what the guest actually gets configured +// with. +func buildFirecrackerConfig(args types.ExecArgs, ukernel types.Unikernel) *FirecrackerConfig { // VM config for Firecracker fcMem := DefaultMemory if args.MemSizeB != 0 { @@ -192,27 +223,96 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel } } - FCConfig := &FirecrackerConfig{ + return &FirecrackerConfig{ Source: FCSource, Machine: FCMachine, Drives: FCDrives, NetIfs: FCNet, VSock: FCVSockDev, } - FCConfigJSON, err := json.Marshal(FCConfig) - if err != nil { - return nil, fmt.Errorf("failed to marshal Firecracker config: %w", err) +} + +// RunSocketBoot starts Firecracker as a child process instead of replacing +// the current process via exec. Since nothing here changes the process's +// root (no SysProcAttr.Chroot is set), the child simply inherits whatever +// confinement changeRoot already established on the caller earlier in Exec +// (pivot_root or chroot, whichever the container spec calls for) - so it +// ends up exactly as confined as the exec path would have made it, with no +// separate confinement step needed here. +// +// It configures the guest over the API socket using the same config +// BuildExecCmd would have written to a file, starts the guest, then +// supervises the child until it exits: forwarding SIGTERM/SIGINT, and +// exiting this process with the child's exit code once it's done, mirroring +// the semantics syscall.Exec would have had. +// +// On success this function does not return: it calls os.Exit with the +// child's exit status once the child exits. It returns an error only if +// startup or configuration fails before the guest could ever run. +func (fc *Firecracker) RunSocketBoot(args types.ExecArgs, ukernel types.Unikernel, execCmd []string) error { + cfg := buildFirecrackerConfig(args, ukernel) + + cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec + cmd.Env = args.Environment + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to start firecracker: %w", err) } - if err = os.WriteFile(JSONConfigFile, FCConfigJSON, 0o644); err != nil { //nolint: gosec - return nil, fmt.Errorf("failed to save Firecracker json config: %w", err) + + socketPath := args.SocketPath + if socketPath == "" { + socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") } - vmmLog.WithField("Json", string(FCConfigJSON)).Debug("Firecracker json config") + client := newFirecrackerClient(socketPath) + ctx := context.Background() - exArgs := strings.Split(cmdString, " ") - return exArgs, nil -} + if err := client.waitForSocket(5 * time.Second); err != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + return fmt.Errorf("firecracker socket never became ready: %w", err) + } + if err := client.configure(ctx, cfg); err != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + return fmt.Errorf("failed to configure firecracker over the socket: %w", err) + } + if err := client.startGuest(ctx); err != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + return fmt.Errorf("failed to start the guest: %w", err) + } -// PreExec performs pre-execution setup. Firecracker has no special pre-exec requirements. -func (fc *Firecracker) PreExec(_ types.ExecArgs) error { - return nil + // Forward the signals containerd would send to stop the container. + // SIGKILL cannot be caught, so it is not listed here: if it arrives, + // this process dies immediately and the child is left running, a known + // gap for this bounded experiment, not yet handled. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) + go func() { + sig, ok := <-sigCh + if !ok { + return + } + if s, ok := sig.(syscall.Signal); ok { + _ = cmd.Process.Signal(s) + } + }() + + waitErr := cmd.Wait() + signal.Stop(sigCh) + close(sigCh) + + exitCode := 0 + if waitErr != nil { + var exitErr *exec.ExitError + if errors.As(waitErr, &exitErr) { + exitCode = exitErr.ExitCode() + } else { + vmmLog.WithError(waitErr).Error("firecracker exited with an unexpected error") + exitCode = 1 + } + } + os.Exit(exitCode) + return nil // unreachable } diff --git a/pkg/unikontainers/hypervisors/firecracker_client.go b/pkg/unikontainers/hypervisors/firecracker_client.go new file mode 100644 index 000000000..097ba9297 --- /dev/null +++ b/pkg/unikontainers/hypervisors/firecracker_client.go @@ -0,0 +1,132 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "time" +) + +// firecrackerClient drives a running Firecracker process over its HTTP-over-Unix +// API socket: it waits for the socket, configures the microVM and starts the +// guest. +// +// This type is the API driver only; starting (and owning) the Firecracker +// process is handled by the caller (see RunSocketBoot). +type firecrackerClient struct { + socketPath string + httpClient *http.Client +} + +// newFirecrackerClient returns a client that talks to the Firecracker API socket +// at socketPath. The HTTP transport dials that Unix socket for every request, so +// "http://localhost/" requests actually travel over the socket. +func newFirecrackerClient(socketPath string) *firecrackerClient { + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socketPath) + }, + } + return &firecrackerClient{ + socketPath: socketPath, + httpClient: &http.Client{Transport: transport}, + } +} + +// waitForSocket blocks until the Firecracker API socket exists and accepts +// connections, or the timeout elapses. Firecracker creates the socket shortly +// after it starts, so the caller polls here before sending any configuration. +func (c *firecrackerClient) waitForSocket(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("unix", c.socketPath, 50*time.Millisecond) + if err == nil { + _ = conn.Close() + return nil + } + lastErr = err + time.Sleep(10 * time.Millisecond) + } + if lastErr == nil { + lastErr = context.DeadlineExceeded + } + return fmt.Errorf("firecracker API socket %q not ready within %s: %w", c.socketPath, timeout, lastErr) +} + +// configure sends the full microVM configuration over the API socket, in the +// order Firecracker expects, before the guest is started. +func (c *firecrackerClient) configure(ctx context.Context, cfg *FirecrackerConfig) error { + if err := c.put(ctx, "/machine-config", cfg.Machine); err != nil { + return err + } + if err := c.put(ctx, "/boot-source", cfg.Source); err != nil { + return err + } + for _, drive := range cfg.Drives { + if err := c.put(ctx, "/drives/"+drive.DriveID, drive); err != nil { + return err + } + } + for _, iface := range cfg.NetIfs { + if err := c.put(ctx, "/network-interfaces/"+iface.IfaceID, iface); err != nil { + return err + } + } + // vsock is only configured when vAccel-over-vsock is enabled (uds_path set). + if cfg.VSock.UDSPath != "" { + if err := c.put(ctx, "/vsock", cfg.VSock); err != nil { + return err + } + } + return nil +} + +// startGuest issues the InstanceStart action, which powers on the microVM and +// boots the guest. Firecracker allows this to succeed only once. +func (c *firecrackerClient) startGuest(ctx context.Context) error { + return c.put(ctx, "/actions", map[string]string{"action_type": "InstanceStart"}) +} + +// put marshals body to JSON and sends it as an HTTP PUT to the given API path +// over the Unix socket, returning an error for any non-2xx response. +func (c *firecrackerClient) put(ctx context.Context, path string, body any) error { + payload, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("marshal %s request: %w", path, err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://localhost"+path, bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("build %s request: %w", path, err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("send %s request: %w", path, err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("%s returned HTTP %d: %s", path, resp.StatusCode, string(respBody)) + } + return nil +} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 542764525..8eaae7dd3 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -231,6 +231,19 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { return netArgs, nil } +// HasNetwork does a cheap check for whether this container has a network +// interface to configure, without doing the more expensive tap device +// creation and configuration that SetupNet does. It exists so callers can +// learn this fact early, while the full SetupNet runs concurrently. +func (u *Unikontainer) HasNetwork() (bool, error) { + networkType := u.getNetworkType() + netManager, err := network.NewNetworkManager(networkType) + if err != nil { + return false, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) + } + return netManager.HasNetwork() +} + // chooseRootfs determines the best rootfs configuration based on available options // Priority order: // 1. Initrd (if specified) @@ -406,19 +419,45 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } // handle network - netArgs, err := u.SetupNet() - if err != nil { - uniklog.Errorf("failed to setup network: %v", err) - return err + // + // For Firecracker's API-based boot mode, rootfs prep (below) only needs + // the cheap "does this container have a network interface" fact, not the + // fully finished network setup (see prepareMonRootfs's needsTAP argument + // below) - the expensive part of network setup (creating the tap device, + // TC rules, fetching interface info) doesn't need to block it. So in + // that case we run the real SetupNet concurrently with rootfs prep, and + // only wait for it once we actually need the resolved IP/gateway/MAC - + // which is right before unikernel.Init, since every supported unikernel + // type bakes the resolved network info into the guest's boot command + // line. For every other case, this stays exactly as sequential as today. + isAPIBoot := vmmType == string(hypervisors.FirecrackerVmm) && bootMode != "config-file" + + var netArgs types.NetDevParams + var netSetupErr error + var netWG sync.WaitGroup + var withTUNTAP bool + + if isAPIBoot { + hasNet, err := u.HasNetwork() + if err != nil { + uniklog.Errorf("failed to check for a container network: %v", err) + return err + } + withTUNTAP = hasNet + netWG.Add(1) + go func() { + defer netWG.Done() + netArgs, netSetupErr = u.SetupNet() + }() + } else { + netArgs, err = u.SetupNet() + if err != nil { + uniklog.Errorf("failed to setup network: %v", err) + return err + } + withTUNTAP = netArgs.IP != "" } metrics.Capture(m.TS16) - withTUNTAP := netArgs.IP != "" - - // UnikernelParams - unikernelParams.Net = netArgs - - // ExecArgs - vmmArgs.Net = netArgs // virtiofsd config virtiofsdConfig := u.UruncCfg.ExtraBins["virtiofsd"] @@ -583,6 +622,20 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { vmmArgs.VSockDevID = idToGuestCID(u.State.ID) } + // If network setup is still running concurrently (isAPIBoot), this is + // where we actually need its result: every supported unikernel type + // bakes the resolved IP/gateway/MAC into the guest's boot command line + // inside Init below. + if isAPIBoot { + netWG.Wait() + if netSetupErr != nil { + uniklog.Errorf("failed to setup network: %v", netSetupErr) + return netSetupErr + } + } + unikernelParams.Net = netArgs + vmmArgs.Net = netArgs + // unikernel err = unikernel.Init(unikernelParams) if errors.Is(err, unikernels.ErrUndefinedVersion) || @@ -665,6 +718,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if isAPIBoot { + fc, ok := vmm.(*hypervisors.Firecracker) + if !ok { + return fmt.Errorf("boot_mode=api is only supported for the firecracker monitor") + } + uniklog.WithField("command", execCmd).Debug("Starting Firecracker as a supervised child, driving it over its control socket") + return fc.RunSocketBoot(vmmArgs, unikernel, execCmd) + } + // Execute the VMM using the command we built earlier. uniklog.WithField("command", execCmd).Debug("Ready to execve VMM") return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec From 095f46f24240269d18a9f3b06d01e24f221aba9c Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Mon, 20 Jul 2026 12:17:57 +0530 Subject: [PATCH 6/8] feat(firecracker): configure the VMM in stages while urunc sets up Spawn Firecracker at the start of Exec and send each piece of its configuration as soon as urunc produces it: machine-config immediately, the network interface once the tap device exists, drives and boot source right after unikernel.Init, and InstanceStart only after the start-success handshake. The API client now holds one persistent connection, established before changeRoot, since the socket path is not resolvable after the root changes. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/firecracker.go | 171 ++++++++++++++---- .../hypervisors/firecracker_client.go | 112 ++++++++---- pkg/unikontainers/unikontainers.go | 95 ++++++++-- 3 files changed, 282 insertions(+), 96 deletions(-) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index d775fc3ee..8df46d4fa 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -232,57 +232,150 @@ func buildFirecrackerConfig(args types.ExecArgs, ukernel types.Unikernel) *Firec } } -// RunSocketBoot starts Firecracker as a child process instead of replacing -// the current process via exec. Since nothing here changes the process's -// root (no SysProcAttr.Chroot is set), the child simply inherits whatever -// confinement changeRoot already established on the caller earlier in Exec -// (pivot_root or chroot, whichever the container spec calls for) - so it -// ends up exactly as confined as the exec path would have made it, with no -// separate confinement step needed here. -// -// It configures the guest over the API socket using the same config -// BuildExecCmd would have written to a file, starts the guest, then -// supervises the child until it exits: forwarding SIGTERM/SIGINT, and -// exiting this process with the child's exit code once it's done, mirroring -// the semantics syscall.Exec would have had. +// FirecrackerSession is a Firecracker child process being configured over its +// API socket in stages, while the caller performs its own setup work between +// the stages. Create it with SpawnSocketVMM, feed it configuration with the +// Configure* methods as each piece becomes available, boot the guest with +// StartGuest, then hand the calling process over with Supervise. +type FirecrackerSession struct { + cmd *exec.Cmd + client *firecrackerClient +} + +// SpawnSocketVMM starts Firecracker as a child process with only its API +// socket enabled, and establishes the single persistent connection all later +// configuration stages use (it survives a later changeRoot; see +// firecrackerClient). // -// On success this function does not return: it calls os.Exit with the -// child's exit status once the child exits. It returns an error only if -// startup or configuration fails before the guest could ever run. -func (fc *Firecracker) RunSocketBoot(args types.ExecArgs, ukernel types.Unikernel, execCmd []string) error { - cfg := buildFirecrackerConfig(args, ukernel) +// This is called early in Exec, before the monitor rootfs is prepared and +// before changeRoot, so unlike the exec path the child keeps the caller's +// current (pre-pivot) mount view for its lifetime. It does inherit the +// sandbox's network namespace, which Exec has already joined. When uid/gid +// are non-zero the child is started directly under that credential, since +// the caller only drops its own privileges (setupUser) much later. +func (fc *Firecracker) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*FirecrackerSession, error) { + socketPath := args.SocketPath + if socketPath == "" { + socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") + } + execCmd := []string{fc.Path(), "--api-sock", socketPath} + if !args.Seccomp { + execCmd = append(execCmd, "--no-seccomp") + } cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec cmd.Env = args.Environment cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr + if uid != 0 || gid != 0 { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uid, Gid: gid}, + } + } + vmmLog.WithField("command", execCmd).Debug("starting Firecracker as a supervised child") if err := cmd.Start(); err != nil { - return fmt.Errorf("failed to start firecracker: %w", err) + return nil, fmt.Errorf("failed to start firecracker: %w", err) } - socketPath := args.SocketPath - if socketPath == "" { - socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") - } client := newFirecrackerClient(socketPath) - ctx := context.Background() + if err := client.connect(5 * time.Second); err != nil { + s := &FirecrackerSession{cmd: cmd, client: client} + s.Kill() + return nil, fmt.Errorf("firecracker socket never became ready: %w", err) + } + return &FirecrackerSession{cmd: cmd, client: client}, nil +} + +// ConfigureMachine sends the vCPU/memory configuration. Nothing but the +// container spec is needed for this, so it is the first stage, sent right +// after the spawn. +func (s *FirecrackerSession) ConfigureMachine(ctx context.Context, args types.ExecArgs) error { + fcMem := DefaultMemory + if args.MemSizeB != 0 { + fcMem = bytesToMiB(args.MemSizeB) + if fcMem == 0 { + fcMem = DefaultMemory + } + } + machine := FirecrackerMachine{ + VcpuCount: args.VCPUs, + MemSizeMiB: fcMem, + Smt: false, + TrackDirtyPages: false, + } + vmmLog.Debug("staged boot: sending machine-config") + return s.client.putMachineConfig(ctx, machine) +} - if err := client.waitForSocket(5 * time.Second); err != nil { - _ = cmd.Process.Kill() - _, _ = cmd.Process.Wait() - return fmt.Errorf("firecracker socket never became ready: %w", err) +// ConfigureNetwork attaches the container's network interface. Firecracker +// opens the tap device during this call, so it must only run once the +// network setup that creates the tap has finished. No-op without a tap. +func (s *FirecrackerSession) ConfigureNetwork(ctx context.Context, net types.NetDevParams) error { + if net.TapDev == "" { + return nil } - if err := client.configure(ctx, cfg); err != nil { - _ = cmd.Process.Kill() - _, _ = cmd.Process.Wait() - return fmt.Errorf("failed to configure firecracker over the socket: %w", err) + iface := FirecrackerNet{ + IfaceID: "net1", + GuestMAC: net.MAC, + HostIF: net.TapDev, } - if err := client.startGuest(ctx); err != nil { - _ = cmd.Process.Kill() - _, _ = cmd.Process.Wait() - return fmt.Errorf("failed to start the guest: %w", err) + vmmLog.Debug("staged boot: sending network-interfaces") + return s.client.putNetworkIface(ctx, iface) +} + +// ConfigureGuest sends everything that depends on unikernel.Init having run: +// the block devices (their IDs/paths come from the unikernel's +// MonitorBlockCli), the boot source (its boot args are the unikernel command +// line, which bakes in the resolved network), and the vsock device if any. +// +// The child was spawned before changeRoot, so it resolves paths in the +// pre-pivot mount view; monRootfs (the directory the caller will pivot into) +// is therefore prefixed onto every path the child has to open. +func (s *FirecrackerSession) ConfigureGuest(ctx context.Context, args types.ExecArgs, ukernel types.Unikernel, monRootfs string) error { + cfg := buildFirecrackerConfig(args, ukernel) + + cfg.Source.ImagePath = filepath.Join(monRootfs, cfg.Source.ImagePath) + if cfg.Source.InitrdPath != "" { + cfg.Source.InitrdPath = filepath.Join(monRootfs, cfg.Source.InitrdPath) } + for i := range cfg.Drives { + cfg.Drives[i].HostPath = filepath.Join(monRootfs, cfg.Drives[i].HostPath) + } + if cfg.VSock.UDSPath != "" { + cfg.VSock.UDSPath = filepath.Join(monRootfs, cfg.VSock.UDSPath) + } + + vmmLog.Debug("staged boot: sending drives, boot-source and vsock") + if err := s.client.putDrives(ctx, cfg.Drives); err != nil { + return err + } + if err := s.client.putBootSource(ctx, cfg.Source); err != nil { + return err + } + return s.client.putVSock(ctx, cfg.VSock) +} +// StartGuest powers on the configured microVM. +func (s *FirecrackerSession) StartGuest(ctx context.Context) error { + vmmLog.Debug("staged boot: sending InstanceStart") + return s.client.startGuest(ctx) +} + +// Kill terminates the child and reaps it. For error paths before Supervise. +func (s *FirecrackerSession) Kill() { + _ = s.cmd.Process.Kill() + _, _ = s.cmd.Process.Wait() +} + +// Supervise hands the calling process over to the child for the rest of its +// life: it forwards SIGTERM/SIGINT and, once the child exits, exits this +// process with the child's exit code, mirroring the semantics syscall.Exec +// would have had. The caller must not exit before the child, since it is +// the container's init process. +// +// On success this function does not return: it calls os.Exit with the +// child's exit status once the child exits. +func (s *FirecrackerSession) Supervise() error { // Forward the signals containerd would send to stop the container. // SIGKILL cannot be caught, so it is not listed here: if it arrives, // this process dies immediately and the child is left running, a known @@ -294,12 +387,12 @@ func (fc *Firecracker) RunSocketBoot(args types.ExecArgs, ukernel types.Unikerne if !ok { return } - if s, ok := sig.(syscall.Signal); ok { - _ = cmd.Process.Signal(s) + if sg, ok := sig.(syscall.Signal); ok { + _ = s.cmd.Process.Signal(sg) } }() - waitErr := cmd.Wait() + waitErr := s.cmd.Wait() signal.Stop(sigCh) close(sigCh) diff --git a/pkg/unikontainers/hypervisors/firecracker_client.go b/pkg/unikontainers/hypervisors/firecracker_client.go index 097ba9297..f57c049cc 100644 --- a/pkg/unikontainers/hypervisors/firecracker_client.go +++ b/pkg/unikontainers/hypervisors/firecracker_client.go @@ -22,46 +22,77 @@ import ( "io" "net" "net/http" + "sync" "time" ) // firecrackerClient drives a running Firecracker process over its HTTP-over-Unix -// API socket: it waits for the socket, configures the microVM and starts the -// guest. +// API socket, one configuration resource at a time, so the caller can send each +// piece of configuration as soon as it becomes available. +// +// The client establishes a single connection in connect() and keeps it alive +// for all requests. This matters because the caller may change its root +// (pivot_root/chroot) between configuration stages: the socket path stops +// being resolvable from the new root, but the already-open connection keeps +// working, since open file descriptors survive a root change. // // This type is the API driver only; starting (and owning) the Firecracker -// process is handled by the caller (see RunSocketBoot). +// process is handled by the caller (see SpawnSocketVMM). type firecrackerClient struct { socketPath string httpClient *http.Client + + dialMu sync.Mutex + heldConn net.Conn } // newFirecrackerClient returns a client that talks to the Firecracker API socket -// at socketPath. The HTTP transport dials that Unix socket for every request, so -// "http://localhost/" requests actually travel over the socket. +// at socketPath. "http://localhost/" requests actually travel over the +// socket. Call connect() before issuing any request. func newFirecrackerClient(socketPath string) *firecrackerClient { + c := &firecrackerClient{socketPath: socketPath} transport := &http.Transport{ - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - var d net.Dialer - return d.DialContext(ctx, "unix", socketPath) - }, + DialContext: c.dialContext, + // Keep the single connection alive for the whole staged boot: it is + // established before the caller may change root, and the socket path + // is not resolvable afterwards, so an idle close between stages + // would break every later stage. + IdleConnTimeout: 0, + MaxIdleConns: 1, + MaxIdleConnsPerHost: 1, } - return &firecrackerClient{ - socketPath: socketPath, - httpClient: &http.Client{Transport: transport}, + c.httpClient = &http.Client{Transport: transport} + return c +} + +// dialContext hands the transport the connection pre-established by connect(), +// and falls back to dialing the socket path directly if that connection was +// already consumed (which only works while the path is still resolvable). +func (c *firecrackerClient) dialContext(ctx context.Context, _, _ string) (net.Conn, error) { + c.dialMu.Lock() + defer c.dialMu.Unlock() + if c.heldConn != nil { + conn := c.heldConn + c.heldConn = nil + return conn, nil } + var d net.Dialer + return d.DialContext(ctx, "unix", c.socketPath) } -// waitForSocket blocks until the Firecracker API socket exists and accepts +// connect blocks until the Firecracker API socket exists and accepts // connections, or the timeout elapses. Firecracker creates the socket shortly -// after it starts, so the caller polls here before sending any configuration. -func (c *firecrackerClient) waitForSocket(timeout time.Duration) error { +// after it starts. The successful connection is kept and reused for all +// requests (see the type comment for why). +func (c *firecrackerClient) connect(timeout time.Duration) error { deadline := time.Now().Add(timeout) var lastErr error for time.Now().Before(deadline) { conn, err := net.DialTimeout("unix", c.socketPath, 50*time.Millisecond) if err == nil { - _ = conn.Close() + c.dialMu.Lock() + c.heldConn = conn + c.dialMu.Unlock() return nil } lastErr = err @@ -73,34 +104,41 @@ func (c *firecrackerClient) waitForSocket(timeout time.Duration) error { return fmt.Errorf("firecracker API socket %q not ready within %s: %w", c.socketPath, timeout, lastErr) } -// configure sends the full microVM configuration over the API socket, in the -// order Firecracker expects, before the guest is started. -func (c *firecrackerClient) configure(ctx context.Context, cfg *FirecrackerConfig) error { - if err := c.put(ctx, "/machine-config", cfg.Machine); err != nil { - return err - } - if err := c.put(ctx, "/boot-source", cfg.Source); err != nil { - return err - } - for _, drive := range cfg.Drives { +// putMachineConfig configures vCPUs and memory. This needs nothing but the +// container spec, so it can be sent as the very first stage. +func (c *firecrackerClient) putMachineConfig(ctx context.Context, machine FirecrackerMachine) error { + return c.put(ctx, "/machine-config", machine) +} + +// putNetworkIface attaches one network interface. Firecracker opens the tap +// device during this call, so it must not be sent before the tap exists. +func (c *firecrackerClient) putNetworkIface(ctx context.Context, iface FirecrackerNet) error { + return c.put(ctx, "/network-interfaces/"+iface.IfaceID, iface) +} + +// putDrives attaches the guest's block devices. +func (c *firecrackerClient) putDrives(ctx context.Context, drives []FirecrackerDrive) error { + for _, drive := range drives { if err := c.put(ctx, "/drives/"+drive.DriveID, drive); err != nil { return err } } - for _, iface := range cfg.NetIfs { - if err := c.put(ctx, "/network-interfaces/"+iface.IfaceID, iface); err != nil { - return err - } - } - // vsock is only configured when vAccel-over-vsock is enabled (uds_path set). - if cfg.VSock.UDSPath != "" { - if err := c.put(ctx, "/vsock", cfg.VSock); err != nil { - return err - } - } return nil } +// putBootSource configures the kernel image, boot arguments and initrd. +func (c *firecrackerClient) putBootSource(ctx context.Context, source FirecrackerBootSource) error { + return c.put(ctx, "/boot-source", source) +} + +// putVSock configures the vsock device. No-op when unset (empty uds_path). +func (c *firecrackerClient) putVSock(ctx context.Context, vsock FirecrackerVSockDev) error { + if vsock.UDSPath == "" { + return nil + } + return c.put(ctx, "/vsock", vsock) +} + // startGuest issues the InstanceStart action, which powers on the microVM and // boots the guest. Firecracker allows this to succeed only once. func (c *firecrackerClient) startGuest(ctx context.Context) error { diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 8eaae7dd3..7650bdc85 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -16,6 +16,7 @@ package unikontainers import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -420,16 +421,21 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // handle network // - // For Firecracker's API-based boot mode, rootfs prep (below) only needs - // the cheap "does this container have a network interface" fact, not the - // fully finished network setup (see prepareMonRootfs's needsTAP argument - // below) - the expensive part of network setup (creating the tap device, - // TC rules, fetching interface info) doesn't need to block it. So in - // that case we run the real SetupNet concurrently with rootfs prep, and - // only wait for it once we actually need the resolved IP/gateway/MAC - - // which is right before unikernel.Init, since every supported unikernel - // type bakes the resolved network info into the guest's boot command - // line. For every other case, this stays exactly as sequential as today. + // For Firecracker's API-based boot mode, the VMM is spawned right here, + // before any of the expensive setup work, and configured in stages while + // that work runs: machine-config immediately (it needs nothing but the + // spec), the network interface as soon as network setup finishes, and + // everything unikernel-dependent (drives, boot source) right after + // unikernel.Init. The guest itself is only started (InstanceStart) after + // the start-success handshake, preserving OCI start ordering. + // + // Rootfs prep (below) only needs the cheap "does this container have a + // network interface" fact, not the fully finished network setup (see + // prepareMonRootfs's needsTAP argument below), so the real SetupNet runs + // concurrently with rootfs prep, joined right before unikernel.Init, + // since every supported unikernel type bakes the resolved IP/gateway/MAC + // into the guest's boot command line. For every other case, this stays + // exactly as sequential as today. isAPIBoot := vmmType == string(hypervisors.FirecrackerVmm) && bootMode != "config-file" var netArgs types.NetDevParams @@ -437,7 +443,33 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { var netWG sync.WaitGroup var withTUNTAP bool + var fcSession *hypervisors.FirecrackerSession + fcHandedOff := false + defer func() { + // Any error return after the spawn must not leave the VMM child + // behind. Supervise never returns (os.Exit), so this only fires on + // error paths. + if fcSession != nil && !fcHandedOff { + fcSession.Kill() + } + }() + ctx := context.Background() + if isAPIBoot { + fc, ok := vmm.(*hypervisors.Firecracker) + if !ok { + return fmt.Errorf("boot_mode=api is only supported for the firecracker monitor") + } + fcSession, err = fc.SpawnSocketVMM(vmmArgs, procAttrs.UID, procAttrs.GID) + if err != nil { + uniklog.Errorf("failed to spawn firecracker: %v", err) + return err + } + if err = fcSession.ConfigureMachine(ctx, vmmArgs); err != nil { + uniklog.Errorf("failed to configure the machine over the socket: %v", err) + return err + } + hasNet, err := u.HasNetwork() if err != nil { uniklog.Errorf("failed to check for a container network: %v", err) @@ -625,13 +657,18 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // If network setup is still running concurrently (isAPIBoot), this is // where we actually need its result: every supported unikernel type // bakes the resolved IP/gateway/MAC into the guest's boot command line - // inside Init below. + // inside Init below. The tap device exists from this point on, so the + // network interface is also sent to the VMM right away. if isAPIBoot { netWG.Wait() if netSetupErr != nil { uniklog.Errorf("failed to setup network: %v", netSetupErr) return netSetupErr } + if err = fcSession.ConfigureNetwork(ctx, netArgs); err != nil { + uniklog.Errorf("failed to configure the network over the socket: %v", err) + return err + } } unikernelParams.Net = netArgs vmmArgs.Net = netArgs @@ -655,6 +692,16 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // ExecArgs vmmArgs.Command = unikernelCmd + // Everything unikernel-dependent is known now, so it goes to the VMM + // before changeRoot below: the child kept the pre-pivot mount view, so + // ConfigureGuest prefixes the monitor rootfs onto the paths it sends. + if isAPIBoot { + if err = fcSession.ConfigureGuest(ctx, vmmArgs, unikernel, rootfsParams.MonRootfs); err != nil { + uniklog.Errorf("failed to configure the guest over the socket: %v", err) + return err + } + } + // pivot _, err = findNS(u.Spec.Linux.Namespaces, specs.MountNamespace) // We just want to check if a mount namespace was define din the list @@ -694,10 +741,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // Build the VMM command once and verify it can be constructed successfully. // This ensures we don't report the container as started if command building fails. - execCmd, err := vmm.BuildExecCmd(vmmArgs, unikernel) - if err != nil { - uniklog.WithError(err).Error("failed to build VMM command") - return err + // For the API-based boot the VMM is already running and fully configured + // (the equivalent validation), so there is no command to build. + var execCmd []string + if !isAPIBoot { + execCmd, err = vmm.BuildExecCmd(vmmArgs, unikernel) + if err != nil { + uniklog.WithError(err).Error("failed to build VMM command") + return err + } } // Notify urunc start that the monitor is ready to execute. @@ -719,12 +771,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } if isAPIBoot { - fc, ok := vmm.(*hypervisors.Firecracker) - if !ok { - return fmt.Errorf("boot_mode=api is only supported for the firecracker monitor") + // The VMM is configured; boot the guest and hand this process over + // to supervising the child. This process must not exit before the + // child, since it is the container's init process. + if err = fcSession.StartGuest(ctx); err != nil { + uniklog.Errorf("failed to start the guest: %v", err) + return err } - uniklog.WithField("command", execCmd).Debug("Starting Firecracker as a supervised child, driving it over its control socket") - return fc.RunSocketBoot(vmmArgs, unikernel, execCmd) + fcHandedOff = true + return fcSession.Supervise() } // Execute the VMM using the command we built earlier. From 8a32f41f3a2d8d77960315a355324c6807cfffda Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 21 Jul 2026 21:26:11 +0530 Subject: [PATCH 7/8] perf(firecracker): poll the control socket every 1ms instead of 10ms Firecracker binds its API socket within ~1ms of starting, but connect() polled every 10ms, so it usually slept several ms past the moment the socket was ready before noticing. Poll every 1ms: this recovers almost all of that wasted wait (~6ms less socket-wait measured in isolation), while a 1ms sleep stays well clear of a CPU-burning busy-loop. The end-to-end boot time is unchanged (guest boot dominates it), so this is a small efficiency fix, not a boot-time speedup. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/firecracker_client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/unikontainers/hypervisors/firecracker_client.go b/pkg/unikontainers/hypervisors/firecracker_client.go index f57c049cc..77bbd26fd 100644 --- a/pkg/unikontainers/hypervisors/firecracker_client.go +++ b/pkg/unikontainers/hypervisors/firecracker_client.go @@ -96,7 +96,10 @@ func (c *firecrackerClient) connect(timeout time.Duration) error { return nil } lastErr = err - time.Sleep(10 * time.Millisecond) + // Firecracker binds the socket within ~1ms of starting, so poll + // tightly: a 1ms interval captures nearly all of the readiness + // latency a coarser interval would waste, without busy-spinning. + time.Sleep(1 * time.Millisecond) } if lastErr == nil { lastErr = context.DeadlineExceeded From 9837cf2a0a92c0013e53cad94fbd84ec13a132c3 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 21 Jul 2026 21:26:11 +0530 Subject: [PATCH 8/8] fix(firecracker): remove a stale control socket before spawning Firecracker creates its API socket with bind(), which fails if a file already exists at that path. A crash or a reused container id can leave a stale socket behind, blocking the next start. Remove any leftover socket before spawning, ignoring the not-found case. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/firecracker.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 8df46d4fa..eec005d21 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -258,6 +258,12 @@ func (fc *Firecracker) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*Fi if socketPath == "" { socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") } + // Firecracker binds this path itself; a stale socket file left from an + // earlier run (e.g. a crash) would make its bind fail with "address + // already in use", so remove any leftover first. + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to remove stale socket %q: %w", socketPath, err) + } execCmd := []string{fc.Path(), "--api-sock", socketPath} if !args.Seccomp { execCmd = append(execCmd, "--no-seccomp")