diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..689a6bdc1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,6 +112,8 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | +| `socket_path` | string | (empty) | Optional custom path for the monitor's control socket. If not specified, the monitor uses its default. Currently only used by Firecracker. | +| `boot_mode` | string | `api` | Optional: `api` drives the monitor's boot over its control socket, `config-file` lets the monitor boot itself from a config file (socket only used afterward). Currently only used by Firecracker. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for @@ -130,6 +132,8 @@ data_path = "/usr/local/share/" default_memory_mb = 512 default_vcpus = 2 path = "/opt/firecracker/firecracker" +socket_path = "/run/urunc/fc.sock" +boot_mode = "config-file" ``` ### Extra binaries Configuration diff --git a/pkg/network/network.go b/pkg/network/network.go index 375af431b..67fad11ba 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -38,6 +38,11 @@ type UnikernelNetworkInfo struct { EthDevice Interface } type Manager interface { + // HasNetwork does a cheap check for whether this container has a network + // interface to configure, without doing the more expensive tap device + // creation and configuration. It lets callers learn this fact early, + // before the full NetworkSetup (which does the expensive work) finishes. + HasNetwork() (bool, error) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) } @@ -113,6 +118,18 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L } } + // LinkAdd opens the tap device's file descriptor with O_CLOEXEC and sets + // TUNSETPERSIST, so the interface survives independent of any open fd. + // We don't need to keep it open past this point (the rest of setup, and + // whatever process later attaches to this tap by name, uses netlink/its + // own independent open, not this fd) - closing it now, rather than + // relying on a future exec to do it, matters because a process that + // keeps this fd open (instead of exec-ing away) would otherwise block + // anything else from opening the same single-queue tap device. + for _, tapFd := range tapLink.Fds { + _ = tapFd.Close() + } + err = netlink.LinkSetMTU(tapLink, mtu) if err != nil { return nil, fmt.Errorf("failed to set tap device MTU to %d: %w", mtu, err) diff --git a/pkg/network/network_dynamic.go b/pkg/network/network_dynamic.go index 1d6a2237b..f1f5c991b 100644 --- a/pkg/network/network_dynamic.go +++ b/pkg/network/network_dynamic.go @@ -32,6 +32,17 @@ type DynamicNetwork struct { // FIXME: CUrrently only one tap device per netns can provide functional networking. We need to find a proper way to handle networking // for multiple unikernels in the same pod/network namespace. // See: https://github.com/urunc-dev/urunc/issues/13 +// HasNetwork checks, cheaply, whether a container network interface exists +// in the current netns, without creating the tap device or doing any of the +// other, more expensive setup NetworkSetup does. +func (n DynamicNetwork) HasNetwork() (bool, error) { + _, err := discoverContainerIface() + if err != nil { + return false, nil + } + return true, nil +} + func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { tapIndex, err := getTapIndex() if err != nil { diff --git a/pkg/network/network_static.go b/pkg/network/network_static.go index fa46608cc..dee813680 100644 --- a/pkg/network/network_static.go +++ b/pkg/network/network_static.go @@ -88,6 +88,17 @@ func setNATRule(iface string, sourceIP string) error { return nil } +// HasNetwork checks, cheaply, whether a container network interface exists +// in the current netns, without creating the tap device or doing any of the +// other, more expensive setup NetworkSetup does. +func (n StaticNetwork) HasNetwork() (bool, error) { + _, err := discoverContainerIface() + if err != nil { + return false, nil + } + return true, nil +} + func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { newTapName := strings.ReplaceAll(DefaultTap, "X", "0") addTCRules := false diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 9588a45bf..eec005d21 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -15,11 +15,17 @@ package hypervisors import ( + "context" "encoding/json" + "errors" "fmt" "os" + "os/exec" + "os/signal" "path/filepath" "strings" + "syscall" + "time" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" @@ -108,13 +114,46 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel // options in FC, since the string return value of the Monitor related // functions in the unikernel interface do not integrate well with FC's // json configuration. - cmdString := fc.Path() + " --no-api --config-file " + apiSockPath := args.SocketPath + if apiSockPath == "" { + apiSockPath = filepath.Join("/tmp/", args.ContainerID+".sock") + } + cmdString := fc.Path() + " --api-sock " + apiSockPath JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) - cmdString += JSONConfigFile + if args.BootMode == "config-file" { + // config-file-based: Firecracker boots itself from the JSON config file + // below; the socket stays open only for use after the guest is running. + cmdString += " --config-file " + JSONConfigFile + } if !args.Seccomp { cmdString += " --no-seccomp" } + FCConfig := buildFirecrackerConfig(args, ukernel) + FCConfigJSON, err := json.Marshal(FCConfig) + if err != nil { + return nil, fmt.Errorf("failed to marshal Firecracker config: %w", err) + } + if err = os.WriteFile(JSONConfigFile, FCConfigJSON, 0o644); err != nil { //nolint: gosec + return nil, fmt.Errorf("failed to save Firecracker json config: %w", err) + } + vmmLog.WithField("Json", string(FCConfigJSON)).Debug("Firecracker json config") + + exArgs := strings.Split(cmdString, " ") + return exArgs, nil +} + +// PreExec performs pre-execution setup. Firecracker has no special pre-exec requirements. +func (fc *Firecracker) PreExec(_ types.ExecArgs) error { + return nil +} + +// buildFirecrackerConfig builds the microVM configuration from args and +// ukernel. Used both to write the JSON config file (config-file-based boot) +// and to drive the same configuration over the API socket (API-based boot), +// so both paths always agree on what the guest actually gets configured +// with. +func buildFirecrackerConfig(args types.ExecArgs, ukernel types.Unikernel) *FirecrackerConfig { // VM config for Firecracker fcMem := DefaultMemory if args.MemSizeB != 0 { @@ -184,27 +223,195 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel } } - FCConfig := &FirecrackerConfig{ + return &FirecrackerConfig{ Source: FCSource, Machine: FCMachine, Drives: FCDrives, NetIfs: FCNet, VSock: FCVSockDev, } - FCConfigJSON, err := json.Marshal(FCConfig) - if err != nil { - return nil, fmt.Errorf("failed to marshal Firecracker config: %w", err) +} + +// FirecrackerSession is a Firecracker child process being configured over its +// API socket in stages, while the caller performs its own setup work between +// the stages. Create it with SpawnSocketVMM, feed it configuration with the +// Configure* methods as each piece becomes available, boot the guest with +// StartGuest, then hand the calling process over with Supervise. +type FirecrackerSession struct { + cmd *exec.Cmd + client *firecrackerClient +} + +// SpawnSocketVMM starts Firecracker as a child process with only its API +// socket enabled, and establishes the single persistent connection all later +// configuration stages use (it survives a later changeRoot; see +// firecrackerClient). +// +// This is called early in Exec, before the monitor rootfs is prepared and +// before changeRoot, so unlike the exec path the child keeps the caller's +// current (pre-pivot) mount view for its lifetime. It does inherit the +// sandbox's network namespace, which Exec has already joined. When uid/gid +// are non-zero the child is started directly under that credential, since +// the caller only drops its own privileges (setupUser) much later. +func (fc *Firecracker) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*FirecrackerSession, error) { + socketPath := args.SocketPath + if socketPath == "" { + socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") } - if err = os.WriteFile(JSONConfigFile, FCConfigJSON, 0o644); err != nil { //nolint: gosec - return nil, fmt.Errorf("failed to save Firecracker json config: %w", err) + // Firecracker binds this path itself; a stale socket file left from an + // earlier run (e.g. a crash) would make its bind fail with "address + // already in use", so remove any leftover first. + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to remove stale socket %q: %w", socketPath, err) + } + execCmd := []string{fc.Path(), "--api-sock", socketPath} + if !args.Seccomp { + execCmd = append(execCmd, "--no-seccomp") } - vmmLog.WithField("Json", string(FCConfigJSON)).Debug("Firecracker json config") - exArgs := strings.Split(cmdString, " ") - return exArgs, nil + cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec + cmd.Env = args.Environment + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if uid != 0 || gid != 0 { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uid, Gid: gid}, + } + } + vmmLog.WithField("command", execCmd).Debug("starting Firecracker as a supervised child") + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start firecracker: %w", err) + } + + client := newFirecrackerClient(socketPath) + if err := client.connect(5 * time.Second); err != nil { + s := &FirecrackerSession{cmd: cmd, client: client} + s.Kill() + return nil, fmt.Errorf("firecracker socket never became ready: %w", err) + } + return &FirecrackerSession{cmd: cmd, client: client}, nil } -// PreExec performs pre-execution setup. Firecracker has no special pre-exec requirements. -func (fc *Firecracker) PreExec(_ types.ExecArgs) error { - return nil +// ConfigureMachine sends the vCPU/memory configuration. Nothing but the +// container spec is needed for this, so it is the first stage, sent right +// after the spawn. +func (s *FirecrackerSession) ConfigureMachine(ctx context.Context, args types.ExecArgs) error { + fcMem := DefaultMemory + if args.MemSizeB != 0 { + fcMem = bytesToMiB(args.MemSizeB) + if fcMem == 0 { + fcMem = DefaultMemory + } + } + machine := FirecrackerMachine{ + VcpuCount: args.VCPUs, + MemSizeMiB: fcMem, + Smt: false, + TrackDirtyPages: false, + } + vmmLog.Debug("staged boot: sending machine-config") + return s.client.putMachineConfig(ctx, machine) +} + +// ConfigureNetwork attaches the container's network interface. Firecracker +// opens the tap device during this call, so it must only run once the +// network setup that creates the tap has finished. No-op without a tap. +func (s *FirecrackerSession) ConfigureNetwork(ctx context.Context, net types.NetDevParams) error { + if net.TapDev == "" { + return nil + } + iface := FirecrackerNet{ + IfaceID: "net1", + GuestMAC: net.MAC, + HostIF: net.TapDev, + } + vmmLog.Debug("staged boot: sending network-interfaces") + return s.client.putNetworkIface(ctx, iface) +} + +// ConfigureGuest sends everything that depends on unikernel.Init having run: +// the block devices (their IDs/paths come from the unikernel's +// MonitorBlockCli), the boot source (its boot args are the unikernel command +// line, which bakes in the resolved network), and the vsock device if any. +// +// The child was spawned before changeRoot, so it resolves paths in the +// pre-pivot mount view; monRootfs (the directory the caller will pivot into) +// is therefore prefixed onto every path the child has to open. +func (s *FirecrackerSession) ConfigureGuest(ctx context.Context, args types.ExecArgs, ukernel types.Unikernel, monRootfs string) error { + cfg := buildFirecrackerConfig(args, ukernel) + + cfg.Source.ImagePath = filepath.Join(monRootfs, cfg.Source.ImagePath) + if cfg.Source.InitrdPath != "" { + cfg.Source.InitrdPath = filepath.Join(monRootfs, cfg.Source.InitrdPath) + } + for i := range cfg.Drives { + cfg.Drives[i].HostPath = filepath.Join(monRootfs, cfg.Drives[i].HostPath) + } + if cfg.VSock.UDSPath != "" { + cfg.VSock.UDSPath = filepath.Join(monRootfs, cfg.VSock.UDSPath) + } + + vmmLog.Debug("staged boot: sending drives, boot-source and vsock") + if err := s.client.putDrives(ctx, cfg.Drives); err != nil { + return err + } + if err := s.client.putBootSource(ctx, cfg.Source); err != nil { + return err + } + return s.client.putVSock(ctx, cfg.VSock) +} + +// StartGuest powers on the configured microVM. +func (s *FirecrackerSession) StartGuest(ctx context.Context) error { + vmmLog.Debug("staged boot: sending InstanceStart") + return s.client.startGuest(ctx) +} + +// Kill terminates the child and reaps it. For error paths before Supervise. +func (s *FirecrackerSession) Kill() { + _ = s.cmd.Process.Kill() + _, _ = s.cmd.Process.Wait() +} + +// Supervise hands the calling process over to the child for the rest of its +// life: it forwards SIGTERM/SIGINT and, once the child exits, exits this +// process with the child's exit code, mirroring the semantics syscall.Exec +// would have had. The caller must not exit before the child, since it is +// the container's init process. +// +// On success this function does not return: it calls os.Exit with the +// child's exit status once the child exits. +func (s *FirecrackerSession) Supervise() error { + // Forward the signals containerd would send to stop the container. + // SIGKILL cannot be caught, so it is not listed here: if it arrives, + // this process dies immediately and the child is left running, a known + // gap for this bounded experiment, not yet handled. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) + go func() { + sig, ok := <-sigCh + if !ok { + return + } + if sg, ok := sig.(syscall.Signal); ok { + _ = s.cmd.Process.Signal(sg) + } + }() + + waitErr := s.cmd.Wait() + signal.Stop(sigCh) + close(sigCh) + + exitCode := 0 + if waitErr != nil { + var exitErr *exec.ExitError + if errors.As(waitErr, &exitErr) { + exitCode = exitErr.ExitCode() + } else { + vmmLog.WithError(waitErr).Error("firecracker exited with an unexpected error") + exitCode = 1 + } + } + os.Exit(exitCode) + return nil // unreachable } diff --git a/pkg/unikontainers/hypervisors/firecracker_client.go b/pkg/unikontainers/hypervisors/firecracker_client.go new file mode 100644 index 000000000..77bbd26fd --- /dev/null +++ b/pkg/unikontainers/hypervisors/firecracker_client.go @@ -0,0 +1,173 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "sync" + "time" +) + +// firecrackerClient drives a running Firecracker process over its HTTP-over-Unix +// API socket, one configuration resource at a time, so the caller can send each +// piece of configuration as soon as it becomes available. +// +// The client establishes a single connection in connect() and keeps it alive +// for all requests. This matters because the caller may change its root +// (pivot_root/chroot) between configuration stages: the socket path stops +// being resolvable from the new root, but the already-open connection keeps +// working, since open file descriptors survive a root change. +// +// This type is the API driver only; starting (and owning) the Firecracker +// process is handled by the caller (see SpawnSocketVMM). +type firecrackerClient struct { + socketPath string + httpClient *http.Client + + dialMu sync.Mutex + heldConn net.Conn +} + +// newFirecrackerClient returns a client that talks to the Firecracker API socket +// at socketPath. "http://localhost/" requests actually travel over the +// socket. Call connect() before issuing any request. +func newFirecrackerClient(socketPath string) *firecrackerClient { + c := &firecrackerClient{socketPath: socketPath} + transport := &http.Transport{ + DialContext: c.dialContext, + // Keep the single connection alive for the whole staged boot: it is + // established before the caller may change root, and the socket path + // is not resolvable afterwards, so an idle close between stages + // would break every later stage. + IdleConnTimeout: 0, + MaxIdleConns: 1, + MaxIdleConnsPerHost: 1, + } + c.httpClient = &http.Client{Transport: transport} + return c +} + +// dialContext hands the transport the connection pre-established by connect(), +// and falls back to dialing the socket path directly if that connection was +// already consumed (which only works while the path is still resolvable). +func (c *firecrackerClient) dialContext(ctx context.Context, _, _ string) (net.Conn, error) { + c.dialMu.Lock() + defer c.dialMu.Unlock() + if c.heldConn != nil { + conn := c.heldConn + c.heldConn = nil + return conn, nil + } + var d net.Dialer + return d.DialContext(ctx, "unix", c.socketPath) +} + +// connect blocks until the Firecracker API socket exists and accepts +// connections, or the timeout elapses. Firecracker creates the socket shortly +// after it starts. The successful connection is kept and reused for all +// requests (see the type comment for why). +func (c *firecrackerClient) connect(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("unix", c.socketPath, 50*time.Millisecond) + if err == nil { + c.dialMu.Lock() + c.heldConn = conn + c.dialMu.Unlock() + return nil + } + lastErr = err + // Firecracker binds the socket within ~1ms of starting, so poll + // tightly: a 1ms interval captures nearly all of the readiness + // latency a coarser interval would waste, without busy-spinning. + time.Sleep(1 * time.Millisecond) + } + if lastErr == nil { + lastErr = context.DeadlineExceeded + } + return fmt.Errorf("firecracker API socket %q not ready within %s: %w", c.socketPath, timeout, lastErr) +} + +// putMachineConfig configures vCPUs and memory. This needs nothing but the +// container spec, so it can be sent as the very first stage. +func (c *firecrackerClient) putMachineConfig(ctx context.Context, machine FirecrackerMachine) error { + return c.put(ctx, "/machine-config", machine) +} + +// putNetworkIface attaches one network interface. Firecracker opens the tap +// device during this call, so it must not be sent before the tap exists. +func (c *firecrackerClient) putNetworkIface(ctx context.Context, iface FirecrackerNet) error { + return c.put(ctx, "/network-interfaces/"+iface.IfaceID, iface) +} + +// putDrives attaches the guest's block devices. +func (c *firecrackerClient) putDrives(ctx context.Context, drives []FirecrackerDrive) error { + for _, drive := range drives { + if err := c.put(ctx, "/drives/"+drive.DriveID, drive); err != nil { + return err + } + } + return nil +} + +// putBootSource configures the kernel image, boot arguments and initrd. +func (c *firecrackerClient) putBootSource(ctx context.Context, source FirecrackerBootSource) error { + return c.put(ctx, "/boot-source", source) +} + +// putVSock configures the vsock device. No-op when unset (empty uds_path). +func (c *firecrackerClient) putVSock(ctx context.Context, vsock FirecrackerVSockDev) error { + if vsock.UDSPath == "" { + return nil + } + return c.put(ctx, "/vsock", vsock) +} + +// startGuest issues the InstanceStart action, which powers on the microVM and +// boots the guest. Firecracker allows this to succeed only once. +func (c *firecrackerClient) startGuest(ctx context.Context) error { + return c.put(ctx, "/actions", map[string]string{"action_type": "InstanceStart"}) +} + +// put marshals body to JSON and sends it as an HTTP PUT to the given API path +// over the Unix socket, returning an error for any non-2xx response. +func (c *firecrackerClient) put(ctx context.Context, path string, body any) error { + payload, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("marshal %s request: %w", path, err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://localhost"+path, bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("build %s request: %w", path, err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("send %s request: %w", path, err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("%s returned HTTP %d: %s", path, resp.StatusCode, string(respBody)) + } + return nil +} diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2cc..5ef6faffb 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -96,6 +96,8 @@ type UnikernelParams struct { // FIXME: add extra fields if required by additional VMM's type ExecArgs struct { ContainerID string // The container ID + SocketPath string // Optional user-configured path for the monitor's control socket. Empty means the monitor uses its default. + BootMode string // "api" (default) drives the monitor over its control socket. "config-file" lets the monitor boot itself from a config file, socket only for later use. Environment []string // The environment variables of the monitor Command string // The unikernel's command line Seccomp bool // Enable or disable seccomp filters for the VMM @@ -133,7 +135,9 @@ type ExtraBinConfig struct { type MonitorConfig struct { DefaultMemoryMB uint `toml:"default_memory_mb"` DefaultVCPUs uint `toml:"default_vcpus"` - BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary - DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) - Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary + DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) + Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket. If not specified, the monitor uses its default. + BootMode string `toml:"boot_mode,omitempty"` // Optional: "api" (default) or "config-file". Currently only used by Firecracker. } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index fc297f937..7650bdc85 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -16,6 +16,7 @@ package unikontainers import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -231,6 +232,19 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { return netArgs, nil } +// HasNetwork does a cheap check for whether this container has a network +// interface to configure, without doing the more expensive tap device +// creation and configuration that SetupNet does. It exists so callers can +// learn this fact early, while the full SetupNet runs concurrently. +func (u *Unikontainer) HasNetwork() (bool, error) { + networkType := u.getNetworkType() + netManager, err := network.NewNetworkManager(networkType) + if err != nil { + return false, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) + } + return netManager.HasNetwork() +} + // chooseRootfs determines the best rootfs configuration based on available options // Priority order: // 1. Initrd (if specified) @@ -354,10 +368,14 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath + bootMode := u.UruncCfg.Monitors[vmmType].BootMode // ExecArgs vmmArgs := types.ExecArgs{ ContainerID: u.State.ID, + SocketPath: socketPath, + BootMode: bootMode, UnikernelPath: unikernelPath, InitrdPath: initrdPath, Seccomp: true, // Enable Seccomp by default @@ -402,19 +420,76 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } // handle network - netArgs, err := u.SetupNet() - if err != nil { - uniklog.Errorf("failed to setup network: %v", err) - return err - } - metrics.Capture(m.TS16) - withTUNTAP := netArgs.IP != "" + // + // For Firecracker's API-based boot mode, the VMM is spawned right here, + // before any of the expensive setup work, and configured in stages while + // that work runs: machine-config immediately (it needs nothing but the + // spec), the network interface as soon as network setup finishes, and + // everything unikernel-dependent (drives, boot source) right after + // unikernel.Init. The guest itself is only started (InstanceStart) after + // the start-success handshake, preserving OCI start ordering. + // + // Rootfs prep (below) only needs the cheap "does this container have a + // network interface" fact, not the fully finished network setup (see + // prepareMonRootfs's needsTAP argument below), so the real SetupNet runs + // concurrently with rootfs prep, joined right before unikernel.Init, + // since every supported unikernel type bakes the resolved IP/gateway/MAC + // into the guest's boot command line. For every other case, this stays + // exactly as sequential as today. + isAPIBoot := vmmType == string(hypervisors.FirecrackerVmm) && bootMode != "config-file" + + var netArgs types.NetDevParams + var netSetupErr error + var netWG sync.WaitGroup + var withTUNTAP bool + + var fcSession *hypervisors.FirecrackerSession + fcHandedOff := false + defer func() { + // Any error return after the spawn must not leave the VMM child + // behind. Supervise never returns (os.Exit), so this only fires on + // error paths. + if fcSession != nil && !fcHandedOff { + fcSession.Kill() + } + }() + ctx := context.Background() - // UnikernelParams - unikernelParams.Net = netArgs + if isAPIBoot { + fc, ok := vmm.(*hypervisors.Firecracker) + if !ok { + return fmt.Errorf("boot_mode=api is only supported for the firecracker monitor") + } + fcSession, err = fc.SpawnSocketVMM(vmmArgs, procAttrs.UID, procAttrs.GID) + if err != nil { + uniklog.Errorf("failed to spawn firecracker: %v", err) + return err + } + if err = fcSession.ConfigureMachine(ctx, vmmArgs); err != nil { + uniklog.Errorf("failed to configure the machine over the socket: %v", err) + return err + } - // ExecArgs - vmmArgs.Net = netArgs + hasNet, err := u.HasNetwork() + if err != nil { + uniklog.Errorf("failed to check for a container network: %v", err) + return err + } + withTUNTAP = hasNet + netWG.Add(1) + go func() { + defer netWG.Done() + netArgs, netSetupErr = u.SetupNet() + }() + } else { + netArgs, err = u.SetupNet() + if err != nil { + uniklog.Errorf("failed to setup network: %v", err) + return err + } + withTUNTAP = netArgs.IP != "" + } + metrics.Capture(m.TS16) // virtiofsd config virtiofsdConfig := u.UruncCfg.ExtraBins["virtiofsd"] @@ -579,6 +654,25 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { vmmArgs.VSockDevID = idToGuestCID(u.State.ID) } + // If network setup is still running concurrently (isAPIBoot), this is + // where we actually need its result: every supported unikernel type + // bakes the resolved IP/gateway/MAC into the guest's boot command line + // inside Init below. The tap device exists from this point on, so the + // network interface is also sent to the VMM right away. + if isAPIBoot { + netWG.Wait() + if netSetupErr != nil { + uniklog.Errorf("failed to setup network: %v", netSetupErr) + return netSetupErr + } + if err = fcSession.ConfigureNetwork(ctx, netArgs); err != nil { + uniklog.Errorf("failed to configure the network over the socket: %v", err) + return err + } + } + unikernelParams.Net = netArgs + vmmArgs.Net = netArgs + // unikernel err = unikernel.Init(unikernelParams) if errors.Is(err, unikernels.ErrUndefinedVersion) || @@ -598,6 +692,16 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // ExecArgs vmmArgs.Command = unikernelCmd + // Everything unikernel-dependent is known now, so it goes to the VMM + // before changeRoot below: the child kept the pre-pivot mount view, so + // ConfigureGuest prefixes the monitor rootfs onto the paths it sends. + if isAPIBoot { + if err = fcSession.ConfigureGuest(ctx, vmmArgs, unikernel, rootfsParams.MonRootfs); err != nil { + uniklog.Errorf("failed to configure the guest over the socket: %v", err) + return err + } + } + // pivot _, err = findNS(u.Spec.Linux.Namespaces, specs.MountNamespace) // We just want to check if a mount namespace was define din the list @@ -637,10 +741,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // Build the VMM command once and verify it can be constructed successfully. // This ensures we don't report the container as started if command building fails. - execCmd, err := vmm.BuildExecCmd(vmmArgs, unikernel) - if err != nil { - uniklog.WithError(err).Error("failed to build VMM command") - return err + // For the API-based boot the VMM is already running and fully configured + // (the equivalent validation), so there is no command to build. + var execCmd []string + if !isAPIBoot { + execCmd, err = vmm.BuildExecCmd(vmmArgs, unikernel) + if err != nil { + uniklog.WithError(err).Error("failed to build VMM command") + return err + } } // Notify urunc start that the monitor is ready to execute. @@ -661,6 +770,18 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if isAPIBoot { + // The VMM is configured; boot the guest and hand this process over + // to supervising the child. This process must not exit before the + // child, since it is the container's init process. + if err = fcSession.StartGuest(ctx); err != nil { + uniklog.Errorf("failed to start the guest: %v", err) + return err + } + fcHandedOff = true + return fcSession.Supervise() + } + // Execute the VMM using the command we built earlier. uniklog.WithField("command", execCmd).Debug("Ready to execve VMM") return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 5f21d106e..52a028464 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -127,6 +127,8 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"socket_path"] = hvCfg.SocketPath + cfgMap[prefix+"boot_mode"] = hvCfg.BootMode } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -180,6 +182,10 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } else { hvCfg.Vhost = boolVal } + case "socket_path": + hvCfg.SocketPath = val + case "boot_mode": + hvCfg.BootMode = val } cfg.Monitors[hv] = hvCfg }