diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba..689a6bdc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,6 +112,8 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | +| `socket_path` | string | (empty) | Optional custom path for the monitor's control socket. If not specified, the monitor uses its default. Currently only used by Firecracker. | +| `boot_mode` | string | `api` | Optional: `api` drives the monitor's boot over its control socket, `config-file` lets the monitor boot itself from a config file (socket only used afterward). Currently only used by Firecracker. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for @@ -130,6 +132,8 @@ data_path = "/usr/local/share/" default_memory_mb = 512 default_vcpus = 2 path = "/opt/firecracker/firecracker" +socket_path = "/run/urunc/fc.sock" +boot_mode = "config-file" ``` ### Extra binaries Configuration diff --git a/pkg/network/network.go b/pkg/network/network.go index 375af431..67fad11b 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -38,6 +38,11 @@ type UnikernelNetworkInfo struct { EthDevice Interface } type Manager interface { + // HasNetwork does a cheap check for whether this container has a network + // interface to configure, without doing the more expensive tap device + // creation and configuration. It lets callers learn this fact early, + // before the full NetworkSetup (which does the expensive work) finishes. + HasNetwork() (bool, error) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) } @@ -113,6 +118,18 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L } } + // LinkAdd opens the tap device's file descriptor with O_CLOEXEC and sets + // TUNSETPERSIST, so the interface survives independent of any open fd. + // We don't need to keep it open past this point (the rest of setup, and + // whatever process later attaches to this tap by name, uses netlink/its + // own independent open, not this fd) - closing it now, rather than + // relying on a future exec to do it, matters because a process that + // keeps this fd open (instead of exec-ing away) would otherwise block + // anything else from opening the same single-queue tap device. + for _, tapFd := range tapLink.Fds { + _ = tapFd.Close() + } + err = netlink.LinkSetMTU(tapLink, mtu) if err != nil { return nil, fmt.Errorf("failed to set tap device MTU to %d: %w", mtu, err) diff --git a/pkg/network/network_dynamic.go b/pkg/network/network_dynamic.go index 1d6a2237..f1f5c991 100644 --- a/pkg/network/network_dynamic.go +++ b/pkg/network/network_dynamic.go @@ -32,6 +32,17 @@ type DynamicNetwork struct { // FIXME: CUrrently only one tap device per netns can provide functional networking. We need to find a proper way to handle networking // for multiple unikernels in the same pod/network namespace. // See: https://github.com/urunc-dev/urunc/issues/13 +// HasNetwork checks, cheaply, whether a container network interface exists +// in the current netns, without creating the tap device or doing any of the +// other, more expensive setup NetworkSetup does. +func (n DynamicNetwork) HasNetwork() (bool, error) { + _, err := discoverContainerIface() + if err != nil { + return false, nil + } + return true, nil +} + func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { tapIndex, err := getTapIndex() if err != nil { diff --git a/pkg/network/network_static.go b/pkg/network/network_static.go index fa46608c..dee81368 100644 --- a/pkg/network/network_static.go +++ b/pkg/network/network_static.go @@ -88,6 +88,17 @@ func setNATRule(iface string, sourceIP string) error { return nil } +// HasNetwork checks, cheaply, whether a container network interface exists +// in the current netns, without creating the tap device or doing any of the +// other, more expensive setup NetworkSetup does. +func (n StaticNetwork) HasNetwork() (bool, error) { + _, err := discoverContainerIface() + if err != nil { + return false, nil + } + return true, nil +} + func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { newTapName := strings.ReplaceAll(DefaultTap, "X", "0") addTCRules := false diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 9588a45b..d775fc3e 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -15,11 +15,17 @@ package hypervisors import ( + "context" "encoding/json" + "errors" "fmt" "os" + "os/exec" + "os/signal" "path/filepath" "strings" + "syscall" + "time" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" @@ -108,13 +114,46 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel // options in FC, since the string return value of the Monitor related // functions in the unikernel interface do not integrate well with FC's // json configuration. - cmdString := fc.Path() + " --no-api --config-file " + apiSockPath := args.SocketPath + if apiSockPath == "" { + apiSockPath = filepath.Join("/tmp/", args.ContainerID+".sock") + } + cmdString := fc.Path() + " --api-sock " + apiSockPath JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) - cmdString += JSONConfigFile + if args.BootMode == "config-file" { + // config-file-based: Firecracker boots itself from the JSON config file + // below; the socket stays open only for use after the guest is running. + cmdString += " --config-file " + JSONConfigFile + } if !args.Seccomp { cmdString += " --no-seccomp" } + FCConfig := buildFirecrackerConfig(args, ukernel) + FCConfigJSON, err := json.Marshal(FCConfig) + if err != nil { + return nil, fmt.Errorf("failed to marshal Firecracker config: %w", err) + } + if err = os.WriteFile(JSONConfigFile, FCConfigJSON, 0o644); err != nil { //nolint: gosec + return nil, fmt.Errorf("failed to save Firecracker json config: %w", err) + } + vmmLog.WithField("Json", string(FCConfigJSON)).Debug("Firecracker json config") + + exArgs := strings.Split(cmdString, " ") + return exArgs, nil +} + +// PreExec performs pre-execution setup. Firecracker has no special pre-exec requirements. +func (fc *Firecracker) PreExec(_ types.ExecArgs) error { + return nil +} + +// buildFirecrackerConfig builds the microVM configuration from args and +// ukernel. Used both to write the JSON config file (config-file-based boot) +// and to drive the same configuration over the API socket (API-based boot), +// so both paths always agree on what the guest actually gets configured +// with. +func buildFirecrackerConfig(args types.ExecArgs, ukernel types.Unikernel) *FirecrackerConfig { // VM config for Firecracker fcMem := DefaultMemory if args.MemSizeB != 0 { @@ -184,27 +223,96 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel } } - FCConfig := &FirecrackerConfig{ + return &FirecrackerConfig{ Source: FCSource, Machine: FCMachine, Drives: FCDrives, NetIfs: FCNet, VSock: FCVSockDev, } - FCConfigJSON, err := json.Marshal(FCConfig) - if err != nil { - return nil, fmt.Errorf("failed to marshal Firecracker config: %w", err) +} + +// RunSocketBoot starts Firecracker as a child process instead of replacing +// the current process via exec. Since nothing here changes the process's +// root (no SysProcAttr.Chroot is set), the child simply inherits whatever +// confinement changeRoot already established on the caller earlier in Exec +// (pivot_root or chroot, whichever the container spec calls for) - so it +// ends up exactly as confined as the exec path would have made it, with no +// separate confinement step needed here. +// +// It configures the guest over the API socket using the same config +// BuildExecCmd would have written to a file, starts the guest, then +// supervises the child until it exits: forwarding SIGTERM/SIGINT, and +// exiting this process with the child's exit code once it's done, mirroring +// the semantics syscall.Exec would have had. +// +// On success this function does not return: it calls os.Exit with the +// child's exit status once the child exits. It returns an error only if +// startup or configuration fails before the guest could ever run. +func (fc *Firecracker) RunSocketBoot(args types.ExecArgs, ukernel types.Unikernel, execCmd []string) error { + cfg := buildFirecrackerConfig(args, ukernel) + + cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec + cmd.Env = args.Environment + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to start firecracker: %w", err) } - if err = os.WriteFile(JSONConfigFile, FCConfigJSON, 0o644); err != nil { //nolint: gosec - return nil, fmt.Errorf("failed to save Firecracker json config: %w", err) + + socketPath := args.SocketPath + if socketPath == "" { + socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") } - vmmLog.WithField("Json", string(FCConfigJSON)).Debug("Firecracker json config") + client := newFirecrackerClient(socketPath) + ctx := context.Background() - exArgs := strings.Split(cmdString, " ") - return exArgs, nil -} + if err := client.waitForSocket(5 * time.Second); err != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + return fmt.Errorf("firecracker socket never became ready: %w", err) + } + if err := client.configure(ctx, cfg); err != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + return fmt.Errorf("failed to configure firecracker over the socket: %w", err) + } + if err := client.startGuest(ctx); err != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + return fmt.Errorf("failed to start the guest: %w", err) + } -// PreExec performs pre-execution setup. Firecracker has no special pre-exec requirements. -func (fc *Firecracker) PreExec(_ types.ExecArgs) error { - return nil + // Forward the signals containerd would send to stop the container. + // SIGKILL cannot be caught, so it is not listed here: if it arrives, + // this process dies immediately and the child is left running, a known + // gap for this bounded experiment, not yet handled. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) + go func() { + sig, ok := <-sigCh + if !ok { + return + } + if s, ok := sig.(syscall.Signal); ok { + _ = cmd.Process.Signal(s) + } + }() + + waitErr := cmd.Wait() + signal.Stop(sigCh) + close(sigCh) + + exitCode := 0 + if waitErr != nil { + var exitErr *exec.ExitError + if errors.As(waitErr, &exitErr) { + exitCode = exitErr.ExitCode() + } else { + vmmLog.WithError(waitErr).Error("firecracker exited with an unexpected error") + exitCode = 1 + } + } + os.Exit(exitCode) + return nil // unreachable } diff --git a/pkg/unikontainers/hypervisors/firecracker_client.go b/pkg/unikontainers/hypervisors/firecracker_client.go new file mode 100644 index 00000000..097ba929 --- /dev/null +++ b/pkg/unikontainers/hypervisors/firecracker_client.go @@ -0,0 +1,132 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "time" +) + +// firecrackerClient drives a running Firecracker process over its HTTP-over-Unix +// API socket: it waits for the socket, configures the microVM and starts the +// guest. +// +// This type is the API driver only; starting (and owning) the Firecracker +// process is handled by the caller (see RunSocketBoot). +type firecrackerClient struct { + socketPath string + httpClient *http.Client +} + +// newFirecrackerClient returns a client that talks to the Firecracker API socket +// at socketPath. The HTTP transport dials that Unix socket for every request, so +// "http://localhost/" requests actually travel over the socket. +func newFirecrackerClient(socketPath string) *firecrackerClient { + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socketPath) + }, + } + return &firecrackerClient{ + socketPath: socketPath, + httpClient: &http.Client{Transport: transport}, + } +} + +// waitForSocket blocks until the Firecracker API socket exists and accepts +// connections, or the timeout elapses. Firecracker creates the socket shortly +// after it starts, so the caller polls here before sending any configuration. +func (c *firecrackerClient) waitForSocket(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("unix", c.socketPath, 50*time.Millisecond) + if err == nil { + _ = conn.Close() + return nil + } + lastErr = err + time.Sleep(10 * time.Millisecond) + } + if lastErr == nil { + lastErr = context.DeadlineExceeded + } + return fmt.Errorf("firecracker API socket %q not ready within %s: %w", c.socketPath, timeout, lastErr) +} + +// configure sends the full microVM configuration over the API socket, in the +// order Firecracker expects, before the guest is started. +func (c *firecrackerClient) configure(ctx context.Context, cfg *FirecrackerConfig) error { + if err := c.put(ctx, "/machine-config", cfg.Machine); err != nil { + return err + } + if err := c.put(ctx, "/boot-source", cfg.Source); err != nil { + return err + } + for _, drive := range cfg.Drives { + if err := c.put(ctx, "/drives/"+drive.DriveID, drive); err != nil { + return err + } + } + for _, iface := range cfg.NetIfs { + if err := c.put(ctx, "/network-interfaces/"+iface.IfaceID, iface); err != nil { + return err + } + } + // vsock is only configured when vAccel-over-vsock is enabled (uds_path set). + if cfg.VSock.UDSPath != "" { + if err := c.put(ctx, "/vsock", cfg.VSock); err != nil { + return err + } + } + return nil +} + +// startGuest issues the InstanceStart action, which powers on the microVM and +// boots the guest. Firecracker allows this to succeed only once. +func (c *firecrackerClient) startGuest(ctx context.Context) error { + return c.put(ctx, "/actions", map[string]string{"action_type": "InstanceStart"}) +} + +// put marshals body to JSON and sends it as an HTTP PUT to the given API path +// over the Unix socket, returning an error for any non-2xx response. +func (c *firecrackerClient) put(ctx context.Context, path string, body any) error { + payload, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("marshal %s request: %w", path, err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://localhost"+path, bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("build %s request: %w", path, err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("send %s request: %w", path, err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("%s returned HTTP %d: %s", path, resp.StatusCode, string(respBody)) + } + return nil +} diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2c..5ef6faff 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -96,6 +96,8 @@ type UnikernelParams struct { // FIXME: add extra fields if required by additional VMM's type ExecArgs struct { ContainerID string // The container ID + SocketPath string // Optional user-configured path for the monitor's control socket. Empty means the monitor uses its default. + BootMode string // "api" (default) drives the monitor over its control socket. "config-file" lets the monitor boot itself from a config file, socket only for later use. Environment []string // The environment variables of the monitor Command string // The unikernel's command line Seccomp bool // Enable or disable seccomp filters for the VMM @@ -133,7 +135,9 @@ type ExtraBinConfig struct { type MonitorConfig struct { DefaultMemoryMB uint `toml:"default_memory_mb"` DefaultVCPUs uint `toml:"default_vcpus"` - BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary - DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) - Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary + DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) + Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket. If not specified, the monitor uses its default. + BootMode string `toml:"boot_mode,omitempty"` // Optional: "api" (default) or "config-file". Currently only used by Firecracker. } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index fc297f93..8eaae7dd 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -231,6 +231,19 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { return netArgs, nil } +// HasNetwork does a cheap check for whether this container has a network +// interface to configure, without doing the more expensive tap device +// creation and configuration that SetupNet does. It exists so callers can +// learn this fact early, while the full SetupNet runs concurrently. +func (u *Unikontainer) HasNetwork() (bool, error) { + networkType := u.getNetworkType() + netManager, err := network.NewNetworkManager(networkType) + if err != nil { + return false, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) + } + return netManager.HasNetwork() +} + // chooseRootfs determines the best rootfs configuration based on available options // Priority order: // 1. Initrd (if specified) @@ -354,10 +367,14 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath + bootMode := u.UruncCfg.Monitors[vmmType].BootMode // ExecArgs vmmArgs := types.ExecArgs{ ContainerID: u.State.ID, + SocketPath: socketPath, + BootMode: bootMode, UnikernelPath: unikernelPath, InitrdPath: initrdPath, Seccomp: true, // Enable Seccomp by default @@ -402,19 +419,45 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } // handle network - netArgs, err := u.SetupNet() - if err != nil { - uniklog.Errorf("failed to setup network: %v", err) - return err + // + // For Firecracker's API-based boot mode, rootfs prep (below) only needs + // the cheap "does this container have a network interface" fact, not the + // fully finished network setup (see prepareMonRootfs's needsTAP argument + // below) - the expensive part of network setup (creating the tap device, + // TC rules, fetching interface info) doesn't need to block it. So in + // that case we run the real SetupNet concurrently with rootfs prep, and + // only wait for it once we actually need the resolved IP/gateway/MAC - + // which is right before unikernel.Init, since every supported unikernel + // type bakes the resolved network info into the guest's boot command + // line. For every other case, this stays exactly as sequential as today. + isAPIBoot := vmmType == string(hypervisors.FirecrackerVmm) && bootMode != "config-file" + + var netArgs types.NetDevParams + var netSetupErr error + var netWG sync.WaitGroup + var withTUNTAP bool + + if isAPIBoot { + hasNet, err := u.HasNetwork() + if err != nil { + uniklog.Errorf("failed to check for a container network: %v", err) + return err + } + withTUNTAP = hasNet + netWG.Add(1) + go func() { + defer netWG.Done() + netArgs, netSetupErr = u.SetupNet() + }() + } else { + netArgs, err = u.SetupNet() + if err != nil { + uniklog.Errorf("failed to setup network: %v", err) + return err + } + withTUNTAP = netArgs.IP != "" } metrics.Capture(m.TS16) - withTUNTAP := netArgs.IP != "" - - // UnikernelParams - unikernelParams.Net = netArgs - - // ExecArgs - vmmArgs.Net = netArgs // virtiofsd config virtiofsdConfig := u.UruncCfg.ExtraBins["virtiofsd"] @@ -579,6 +622,20 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { vmmArgs.VSockDevID = idToGuestCID(u.State.ID) } + // If network setup is still running concurrently (isAPIBoot), this is + // where we actually need its result: every supported unikernel type + // bakes the resolved IP/gateway/MAC into the guest's boot command line + // inside Init below. + if isAPIBoot { + netWG.Wait() + if netSetupErr != nil { + uniklog.Errorf("failed to setup network: %v", netSetupErr) + return netSetupErr + } + } + unikernelParams.Net = netArgs + vmmArgs.Net = netArgs + // unikernel err = unikernel.Init(unikernelParams) if errors.Is(err, unikernels.ErrUndefinedVersion) || @@ -661,6 +718,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if isAPIBoot { + fc, ok := vmm.(*hypervisors.Firecracker) + if !ok { + return fmt.Errorf("boot_mode=api is only supported for the firecracker monitor") + } + uniklog.WithField("command", execCmd).Debug("Starting Firecracker as a supervised child, driving it over its control socket") + return fc.RunSocketBoot(vmmArgs, unikernel, execCmd) + } + // Execute the VMM using the command we built earlier. uniklog.WithField("command", execCmd).Debug("Ready to execve VMM") return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 5f21d106..52a02846 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -127,6 +127,8 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"socket_path"] = hvCfg.SocketPath + cfgMap[prefix+"boot_mode"] = hvCfg.BootMode } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -180,6 +182,10 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } else { hvCfg.Vhost = boolVal } + case "socket_path": + hvCfg.SocketPath = val + case "boot_mode": + hvCfg.BootMode = val } cfg.Monitors[hv] = hvCfg }