feat(firecracker): launch with control socket instead of config file

docs/configuration.md

@@ -112,6 +112,8 @@ Each monitor subsection supports the following options:
 | `default_vcpus` | integer | `1` | Default number of virtual CPUs |
 | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH |
 | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory |
+| `socket_path` | string | (empty) | Optional custom path for the monitor's control socket. If not specified, the monitor uses its default. Currently only used by Firecracker. |
+| `boot_mode` | string | `api` | Optional: `api` drives the monitor's boot over its control socket, `config-file` lets the monitor boot itself from a config file (socket only used afterward). Currently only used by Firecracker. |
 
 Since Qemu is the only currently supported monitor which requires extra data to
 boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for
@@ -130,6 +132,8 @@ data_path = "/usr/local/share/"
 default_memory_mb = 512
 default_vcpus = 2
 path = "/opt/firecracker/firecracker"
+socket_path = "/run/urunc/fc.sock"
+boot_mode = "config-file"
 ```
 
 ### Extra binaries Configuration

pkg/network/network.go

@@ -38,6 +38,11 @@ type UnikernelNetworkInfo struct {
 	EthDevice Interface
 }
 type Manager interface {
+	// HasNetwork does a cheap check for whether this container has a network
+	// interface to configure, without doing the more expensive tap device
+	// creation and configuration. It lets callers learn this fact early,
+	// before the full NetworkSetup (which does the expensive work) finishes.
+	HasNetwork() (bool, error)
 	NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error)
 }
 
@@ -113,6 +118,18 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L
 		}
 	}
 
+	// LinkAdd opens the tap device's file descriptor with O_CLOEXEC and sets
+	// TUNSETPERSIST, so the interface survives independent of any open fd.
+	// We don't need to keep it open past this point (the rest of setup, and
+	// whatever process later attaches to this tap by name, uses netlink/its
+	// own independent open, not this fd) - closing it now, rather than
+	// relying on a future exec to do it, matters because a process that
+	// keeps this fd open (instead of exec-ing away) would otherwise block
+	// anything else from opening the same single-queue tap device.
+	for _, tapFd := range tapLink.Fds {
+		_ = tapFd.Close()
+	}
+
 	err = netlink.LinkSetMTU(tapLink, mtu)
 	if err != nil {
 		return nil, fmt.Errorf("failed to set tap device MTU to %d: %w", mtu, err)

pkg/network/network_dynamic.go

@@ -32,6 +32,17 @@ type DynamicNetwork struct {
 // FIXME: CUrrently only one tap device per netns can provide functional networking. We need to find a proper way to handle networking
 // for multiple unikernels in the same pod/network namespace.
 // See: https://github.com/urunc-dev/urunc/issues/13
+// HasNetwork checks, cheaply, whether a container network interface exists
+// in the current netns, without creating the tap device or doing any of the
+// other, more expensive setup NetworkSetup does.
+func (n DynamicNetwork) HasNetwork() (bool, error) {
+	_, err := discoverContainerIface()
+	if err != nil {
+		return false, nil
+	}
+	return true, nil
+}
+
 func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) {
 	tapIndex, err := getTapIndex()
 	if err != nil {

pkg/network/network_static.go

@@ -88,6 +88,17 @@ func setNATRule(iface string, sourceIP string) error {
 	return nil
 }
 
+// HasNetwork checks, cheaply, whether a container network interface exists
+// in the current netns, without creating the tap device or doing any of the
+// other, more expensive setup NetworkSetup does.
+func (n StaticNetwork) HasNetwork() (bool, error) {
+	_, err := discoverContainerIface()
+	if err != nil {
+		return false, nil
+	}
+	return true, nil
+}
+
 func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) {
 	newTapName := strings.ReplaceAll(DefaultTap, "X", "0")
 	addTCRules := false

pkg/unikontainers/hypervisors/firecracker.go

@@ -15,11 +15,17 @@
 package hypervisors
 
 import (
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
+	"os/exec"
+	"os/signal"
 	"path/filepath"
 	"strings"
+	"syscall"
+	"time"
 
 	"github.com/urunc-dev/urunc/pkg/unikontainers/types"
 	"golang.org/x/sys/unix"
@@ -108,13 +114,46 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel
 	// options in FC, since the string return value of the Monitor related
 	// functions in the unikernel interface do not integrate well with FC's
 	// json configuration.
-	cmdString := fc.Path() + " --no-api --config-file "
+	apiSockPath := args.SocketPath
+	if apiSockPath == "" {
+		apiSockPath = filepath.Join("/tmp/", args.ContainerID+".sock")
+	}
+	cmdString := fc.Path() + " --api-sock " + apiSockPath
 	JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename)
-	cmdString += JSONConfigFile
+	if args.BootMode == "config-file" {
+		// config-file-based: Firecracker boots itself from the JSON config file
+		// below; the socket stays open only for use after the guest is running.
+		cmdString += " --config-file " + JSONConfigFile
+	}
 	if !args.Seccomp {
 		cmdString += " --no-seccomp"
 	}
 
+	FCConfig := buildFirecrackerConfig(args, ukernel)
+	FCConfigJSON, err := json.Marshal(FCConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal Firecracker config: %w", err)
+	}
+	if err = os.WriteFile(JSONConfigFile, FCConfigJSON, 0o644); err != nil { //nolint: gosec
+		return nil, fmt.Errorf("failed to save Firecracker json config: %w", err)
+	}
+	vmmLog.WithField("Json", string(FCConfigJSON)).Debug("Firecracker json config")
+
+	exArgs := strings.Split(cmdString, " ")
+	return exArgs, nil
+}
+
+// PreExec performs pre-execution setup. Firecracker has no special pre-exec requirements.
+func (fc *Firecracker) PreExec(_ types.ExecArgs) error {
+	return nil
+}
+
+// buildFirecrackerConfig builds the microVM configuration from args and
+// ukernel. Used both to write the JSON config file (config-file-based boot)
+// and to drive the same configuration over the API socket (API-based boot),
+// so both paths always agree on what the guest actually gets configured
+// with.
+func buildFirecrackerConfig(args types.ExecArgs, ukernel types.Unikernel) *FirecrackerConfig {
 	// VM config for Firecracker
 	fcMem := DefaultMemory
 	if args.MemSizeB != 0 {
@@ -184,27 +223,195 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel
 		}
 	}
 
-	FCConfig := &FirecrackerConfig{
+	return &FirecrackerConfig{
 		Source:  FCSource,
 		Machine: FCMachine,
 		Drives:  FCDrives,
 		NetIfs:  FCNet,
 		VSock:   FCVSockDev,
 	}
-	FCConfigJSON, err := json.Marshal(FCConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to marshal Firecracker config: %w", err)
+}
+
+// FirecrackerSession is a Firecracker child process being configured over its
+// API socket in stages, while the caller performs its own setup work between
+// the stages. Create it with SpawnSocketVMM, feed it configuration with the
+// Configure* methods as each piece becomes available, boot the guest with
+// StartGuest, then hand the calling process over with Supervise.
+type FirecrackerSession struct {
+	cmd    *exec.Cmd
+	client *firecrackerClient
+}
+
+// SpawnSocketVMM starts Firecracker as a child process with only its API
+// socket enabled, and establishes the single persistent connection all later
+// configuration stages use (it survives a later changeRoot; see
+// firecrackerClient).
+//
+// This is called early in Exec, before the monitor rootfs is prepared and
+// before changeRoot, so unlike the exec path the child keeps the caller's
+// current (pre-pivot) mount view for its lifetime. It does inherit the
+// sandbox's network namespace, which Exec has already joined. When uid/gid
+// are non-zero the child is started directly under that credential, since
+// the caller only drops its own privileges (setupUser) much later.
+func (fc *Firecracker) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*FirecrackerSession, error) {
+	socketPath := args.SocketPath
+	if socketPath == "" {
+		socketPath = filepath.Join("/tmp/", args.ContainerID+".sock")
 	}
-	if err = os.WriteFile(JSONConfigFile, FCConfigJSON, 0o644); err != nil { //nolint: gosec
-		return nil, fmt.Errorf("failed to save Firecracker json config: %w", err)
+	// Firecracker binds this path itself; a stale socket file left from an
+	// earlier run (e.g. a crash) would make its bind fail with "address
+	// already in use", so remove any leftover first.
+	if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("failed to remove stale socket %q: %w", socketPath, err)
+	}
+	execCmd := []string{fc.Path(), "--api-sock", socketPath}
+	if !args.Seccomp {
+		execCmd = append(execCmd, "--no-seccomp")
 	}
-	vmmLog.WithField("Json", string(FCConfigJSON)).Debug("Firecracker json config")
 
-	exArgs := strings.Split(cmdString, " ")
-	return exArgs, nil
+	cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec
+	cmd.Env = args.Environment
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if uid != 0 || gid != 0 {
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			Credential: &syscall.Credential{Uid: uid, Gid: gid},
+		}
+	}
+	vmmLog.WithField("command", execCmd).Debug("starting Firecracker as a supervised child")
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start firecracker: %w", err)
+	}
+
+	client := newFirecrackerClient(socketPath)
+	if err := client.connect(5 * time.Second); err != nil {
+		s := &FirecrackerSession{cmd: cmd, client: client}
+		s.Kill()
+		return nil, fmt.Errorf("firecracker socket never became ready: %w", err)
+	}
+	return &FirecrackerSession{cmd: cmd, client: client}, nil
 }
 
-// PreExec performs pre-execution setup. Firecracker has no special pre-exec requirements.
-func (fc *Firecracker) PreExec(_ types.ExecArgs) error {
-	return nil
+// ConfigureMachine sends the vCPU/memory configuration. Nothing but the
+// container spec is needed for this, so it is the first stage, sent right
+// after the spawn.
+func (s *FirecrackerSession) ConfigureMachine(ctx context.Context, args types.ExecArgs) error {
+	fcMem := DefaultMemory
+	if args.MemSizeB != 0 {
+		fcMem = bytesToMiB(args.MemSizeB)
+		if fcMem == 0 {
+			fcMem = DefaultMemory
+		}
+	}
+	machine := FirecrackerMachine{
+		VcpuCount:       args.VCPUs,
+		MemSizeMiB:      fcMem,
+		Smt:             false,
+		TrackDirtyPages: false,
+	}
+	vmmLog.Debug("staged boot: sending machine-config")
+	return s.client.putMachineConfig(ctx, machine)
+}
+
+// ConfigureNetwork attaches the container's network interface. Firecracker
+// opens the tap device during this call, so it must only run once the
+// network setup that creates the tap has finished. No-op without a tap.
+func (s *FirecrackerSession) ConfigureNetwork(ctx context.Context, net types.NetDevParams) error {
+	if net.TapDev == "" {
+		return nil
+	}
+	iface := FirecrackerNet{
+		IfaceID:  "net1",
+		GuestMAC: net.MAC,
+		HostIF:   net.TapDev,
+	}
+	vmmLog.Debug("staged boot: sending network-interfaces")
+	return s.client.putNetworkIface(ctx, iface)
+}
+
+// ConfigureGuest sends everything that depends on unikernel.Init having run:
+// the block devices (their IDs/paths come from the unikernel's
+// MonitorBlockCli), the boot source (its boot args are the unikernel command
+// line, which bakes in the resolved network), and the vsock device if any.
+//
+// The child was spawned before changeRoot, so it resolves paths in the
+// pre-pivot mount view; monRootfs (the directory the caller will pivot into)
+// is therefore prefixed onto every path the child has to open.
+func (s *FirecrackerSession) ConfigureGuest(ctx context.Context, args types.ExecArgs, ukernel types.Unikernel, monRootfs string) error {
+	cfg := buildFirecrackerConfig(args, ukernel)
+
+	cfg.Source.ImagePath = filepath.Join(monRootfs, cfg.Source.ImagePath)
+	if cfg.Source.InitrdPath != "" {
+		cfg.Source.InitrdPath = filepath.Join(monRootfs, cfg.Source.InitrdPath)
+	}
+	for i := range cfg.Drives {
+		cfg.Drives[i].HostPath = filepath.Join(monRootfs, cfg.Drives[i].HostPath)
+	}
+	if cfg.VSock.UDSPath != "" {
+		cfg.VSock.UDSPath = filepath.Join(monRootfs, cfg.VSock.UDSPath)
+	}
+
+	vmmLog.Debug("staged boot: sending drives, boot-source and vsock")
+	if err := s.client.putDrives(ctx, cfg.Drives); err != nil {
+		return err
+	}
+	if err := s.client.putBootSource(ctx, cfg.Source); err != nil {
+		return err
+	}
+	return s.client.putVSock(ctx, cfg.VSock)
+}
+
+// StartGuest powers on the configured microVM.
+func (s *FirecrackerSession) StartGuest(ctx context.Context) error {
+	vmmLog.Debug("staged boot: sending InstanceStart")
+	return s.client.startGuest(ctx)
+}
+
+// Kill terminates the child and reaps it. For error paths before Supervise.
+func (s *FirecrackerSession) Kill() {
+	_ = s.cmd.Process.Kill()
+	_, _ = s.cmd.Process.Wait()
+}
+
+// Supervise hands the calling process over to the child for the rest of its
+// life: it forwards SIGTERM/SIGINT and, once the child exits, exits this
+// process with the child's exit code, mirroring the semantics syscall.Exec
+// would have had. The caller must not exit before the child, since it is
+// the container's init process.
+//
+// On success this function does not return: it calls os.Exit with the
+// child's exit status once the child exits.
+func (s *FirecrackerSession) Supervise() error {
+	// Forward the signals containerd would send to stop the container.
+	// SIGKILL cannot be caught, so it is not listed here: if it arrives,
+	// this process dies immediately and the child is left running, a known
+	// gap for this bounded experiment, not yet handled.
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
+	go func() {
+		sig, ok := <-sigCh
+		if !ok {
+			return
+		}
+		if sg, ok := sig.(syscall.Signal); ok {
+			_ = s.cmd.Process.Signal(sg)
+		}
+	}()
+
+	waitErr := s.cmd.Wait()
+	signal.Stop(sigCh)
+	close(sigCh)
+
+	exitCode := 0
+	if waitErr != nil {
+		var exitErr *exec.ExitError
+		if errors.As(waitErr, &exitErr) {
+			exitCode = exitErr.ExitCode()
+		} else {
+			vmmLog.WithError(waitErr).Error("firecracker exited with an unexpected error")
+			exitCode = 1
+		}
+	}
+	os.Exit(exitCode)
+	return nil // unreachable
 }