diff --git a/Cargo.lock b/Cargo.lock index a5d236261..9a435227d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -556,6 +556,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "boxlite-shared", + "chrono", "clap", "futures", "libcontainer", diff --git a/apps/api/src/box/dto/create-box.dto.ts b/apps/api/src/box/dto/create-box.dto.ts index b338f866f..b4339d364 100644 --- a/apps/api/src/box/dto/create-box.dto.ts +++ b/apps/api/src/box/dto/create-box.dto.ts @@ -161,6 +161,15 @@ export class CreateBoxDto { @IsBoolean() autoResume?: boolean + @ApiPropertyOptional({ + description: 'Capture init stdout/stderr as CRI records in the box-managed log path', + example: true, + }) + @IsOptional() + @IsBoolean() + captureLogs?: boolean + + @ApiPropertyOptional({ description: 'Array of volumes to attach to the box', type: [BoxVolume], diff --git a/apps/api/src/box/entities/box.entity.ts b/apps/api/src/box/entities/box.entity.ts index a4eb59fbf..01b5ff266 100644 --- a/apps/api/src/box/entities/box.entity.ts +++ b/apps/api/src/box/entities/box.entity.ts @@ -117,6 +117,10 @@ export class Box { @Column({ nullable: true }) networkAllowList?: string + @Column({ default: false, type: 'boolean' }) + captureLogs = false + + @Column('jsonb', { nullable: true }) labels: { [key: string]: string } diff --git a/apps/api/src/box/runner-adapter/runnerAdapter.v0.ts b/apps/api/src/box/runner-adapter/runnerAdapter.v0.ts index 520d380e1..beccc3908 100644 --- a/apps/api/src/box/runner-adapter/runnerAdapter.v0.ts +++ b/apps/api/src/box/runner-adapter/runnerAdapter.v0.ts @@ -18,6 +18,7 @@ import { DefaultApi, UpdateNetworkSettingsDTO, RecoverBoxDTO, + CreateBoxDTO, } from '@boxlite-ai/runner-api-client' import { Box } from '../entities/box.entity' import { BoxState } from '../enums/box-state.enum' @@ -250,7 +251,7 @@ export class RunnerAdapterV0 implements RunnerAdapter { } async createBox(box: Box, metadata?: { [key: string]: string }): Promise { - const response = await this.boxApiClient.create({ + const payload: CreateBoxDTO = { id: box.id, image: box.image ?? '', osUser: box.osUser, @@ -262,10 +263,12 @@ export class RunnerAdapterV0 implements RunnerAdapter { networkBlockAll: box.networkBlockAll, networkAllowList: box.networkAllowList, metadata, + captureLogs: box.captureLogs, authToken: box.authToken, organizationId: box.organizationId, regionId: box.region, - }) + } + const response = await this.boxApiClient.create(payload) if (!response?.data?.daemonVersion) { return undefined @@ -331,6 +334,7 @@ export class RunnerAdapterV0 implements RunnerAdapter { networkBlockAll: box.networkBlockAll, networkAllowList: box.networkAllowList, errorReason: box.errorReason, + captureLogs: box.captureLogs, } await this.boxApiClient.recover(box.id, recoverBoxDTO) } diff --git a/apps/api/src/box/runner-adapter/runnerAdapter.v2.ts b/apps/api/src/box/runner-adapter/runnerAdapter.v2.ts index d07c4bb16..bb44b8eea 100644 --- a/apps/api/src/box/runner-adapter/runnerAdapter.v2.ts +++ b/apps/api/src/box/runner-adapter/runnerAdapter.v2.ts @@ -136,6 +136,7 @@ export class RunnerAdapterV2 implements RunnerAdapter { networkBlockAll: box.networkBlockAll, networkAllowList: box.networkAllowList, metadata, + captureLogs: box.captureLogs, authToken: box.authToken, organizationId: box.organizationId, regionId: box.region, @@ -195,6 +196,7 @@ export class RunnerAdapterV2 implements RunnerAdapter { networkBlockAll: box.networkBlockAll, networkAllowList: box.networkAllowList, errorReason: box.errorReason, + captureLogs: box.captureLogs, } await this.jobService.createJob(null, JobType.RECOVER_BOX, this.runner.id, ResourceType.BOX, box.id, recoverBoxDTO) diff --git a/apps/api/src/box/services/box.service.ts b/apps/api/src/box/services/box.service.ts index 054230595..af71c3767 100644 --- a/apps/api/src/box/services/box.service.ts +++ b/apps/api/src/box/services/box.service.ts @@ -182,7 +182,11 @@ export class BoxService { } else if (image) { // No volumes requested — try to claim a pre-warmed box matching this image/spec // before creating a fresh one. - const skipWarmPool = (await this.redis.exists(`warm-pool:skip:${image}`)) === 1 + // Log capture must be configured before the container init process starts; + // a pre-warmed box cannot be retrofitted without losing early output. + const needsFreshBoxForLogCapture = createBoxDto.captureLogs + const skipWarmPool = + needsFreshBoxForLogCapture || (await this.redis.exists(`warm-pool:skip:${image}`)) === 1 if (!skipWarmPool) { const warmPoolBox = await this.warmPoolService.fetchWarmPoolBox({ organizationId: organization.id, @@ -236,6 +240,8 @@ export class BoxService { box.networkAllowList = this.resolveNetworkAllowList(createBoxDto.networkAllowList) } + box.captureLogs = createBoxDto.captureLogs || false + const lifecyclePolicy = this.resolveLifecyclePolicy({ autoPause: createBoxDto.autoPause, autoDelete: createBoxDto.autoDelete, @@ -310,6 +316,8 @@ export class BoxService { updateData.networkAllowList = this.resolveNetworkAllowList(createBoxDto.networkAllowList) } + updateData.captureLogs = createBoxDto.captureLogs || false + if (!warmPoolBox.runnerId) { throw new BoxError('Runner not found for warm pool box') } diff --git a/apps/api/src/boxlite-rest/dto/create-box.dto.ts b/apps/api/src/boxlite-rest/dto/create-box.dto.ts index 9e0310371..018fbda02 100644 --- a/apps/api/src/boxlite-rest/dto/create-box.dto.ts +++ b/apps/api/src/boxlite-rest/dto/create-box.dto.ts @@ -110,6 +110,11 @@ export class CreateBoxDto { @IsBoolean() auto_resume?: boolean + @IsOptional() + @IsBoolean() + capture_logs?: boolean + + @IsOptional() @ValidateNested() @Type(() => NetworkSpecDto) diff --git a/apps/api/src/boxlite-rest/mappers/box-to-box.mapper.ts b/apps/api/src/boxlite-rest/mappers/box-to-box.mapper.ts index f74a91e78..ea46b210c 100644 --- a/apps/api/src/boxlite-rest/mappers/box-to-box.mapper.ts +++ b/apps/api/src/boxlite-rest/mappers/box-to-box.mapper.ts @@ -42,6 +42,7 @@ export function createBoxToCreateBox(dto: RestCreateBoxDto, target?: string): Cr createDto.memory = dto.memory_mib ? Math.ceil(dto.memory_mib / 1024) : undefined createDto.disk = dto.disk_size_gb createDto.target = target + createDto.captureLogs = dto.capture_logs createDto.autoPause = dto.auto_pause createDto.autoDelete = dto.auto_delete createDto.autoResume = dto.auto_resume diff --git a/apps/api/src/migrations/pre-deploy/1762539090000-add-box-log-capture-migration.ts b/apps/api/src/migrations/pre-deploy/1762539090000-add-box-log-capture-migration.ts new file mode 100644 index 000000000..197eb3b03 --- /dev/null +++ b/apps/api/src/migrations/pre-deploy/1762539090000-add-box-log-capture-migration.ts @@ -0,0 +1,13 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class AddBoxLogCapture1762539090000 implements MigrationInterface { + name = 'AddBoxLogCapture1762539090000' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "box" ADD "captureLogs" boolean NOT NULL DEFAULT false`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "box" DROP COLUMN "captureLogs"`) + } +} diff --git a/apps/libs/runner-api-client/src/models/create-box-dto.ts b/apps/libs/runner-api-client/src/models/create-box-dto.ts index 2558b7e2c..55daf9495 100644 --- a/apps/libs/runner-api-client/src/models/create-box-dto.ts +++ b/apps/libs/runner-api-client/src/models/create-box-dto.ts @@ -22,6 +22,7 @@ import type { RegistryDTO } from './registry-dto'; export interface CreateBoxDTO { 'authToken'?: string; + 'captureLogs'?: boolean; 'cpuQuota'?: number; 'entrypoint'?: Array; 'env'?: { [key: string]: string; }; diff --git a/apps/libs/runner-api-client/src/models/recover-box-dto.ts b/apps/libs/runner-api-client/src/models/recover-box-dto.ts index 0338486f3..321d48883 100644 --- a/apps/libs/runner-api-client/src/models/recover-box-dto.ts +++ b/apps/libs/runner-api-client/src/models/recover-box-dto.ts @@ -19,6 +19,7 @@ import type { DtoVolumeDTO } from './dto-volume-dto'; export interface RecoverBoxDTO { 'backupErrorReason'?: string; + 'captureLogs'?: boolean; 'cpuQuota'?: number; 'env'?: { [key: string]: string; }; 'errorReason': string; diff --git a/apps/runner/pkg/api/docs/docs.go b/apps/runner/pkg/api/docs/docs.go index adda2ca5d..0e5156be3 100644 --- a/apps/runner/pkg/api/docs/docs.go +++ b/apps/runner/pkg/api/docs/docs.go @@ -1308,6 +1308,9 @@ const docTemplate = `{ "authToken": { "type": "string" }, + "captureLogs": { + "type": "boolean" + }, "cpuQuota": { "type": "integer", "minimum": 1 @@ -1487,6 +1490,9 @@ const docTemplate = `{ "backupErrorReason": { "type": "string" }, + "captureLogs": { + "type": "boolean" + }, "cpuQuota": { "type": "integer", "minimum": 1 diff --git a/apps/runner/pkg/api/docs/swagger.json b/apps/runner/pkg/api/docs/swagger.json index 784e50771..cac56b4c6 100644 --- a/apps/runner/pkg/api/docs/swagger.json +++ b/apps/runner/pkg/api/docs/swagger.json @@ -1216,6 +1216,9 @@ "authToken": { "type": "string" }, + "captureLogs": { + "type": "boolean" + }, "cpuQuota": { "type": "integer", "minimum": 1 @@ -1381,6 +1384,9 @@ "backupErrorReason": { "type": "string" }, + "captureLogs": { + "type": "boolean" + }, "cpuQuota": { "type": "integer", "minimum": 1 diff --git a/apps/runner/pkg/api/docs/swagger.yaml b/apps/runner/pkg/api/docs/swagger.yaml index e95fa2126..eef4f3142 100644 --- a/apps/runner/pkg/api/docs/swagger.yaml +++ b/apps/runner/pkg/api/docs/swagger.yaml @@ -51,6 +51,8 @@ definitions: properties: authToken: type: string + captureLogs: + type: boolean cpuQuota: minimum: 1 type: integer @@ -175,6 +177,8 @@ definitions: properties: backupErrorReason: type: string + captureLogs: + type: boolean cpuQuota: minimum: 1 type: integer diff --git a/apps/runner/pkg/api/dto/box.go b/apps/runner/pkg/api/dto/box.go index 596767b18..33a54eb13 100644 --- a/apps/runner/pkg/api/dto/box.go +++ b/apps/runner/pkg/api/dto/box.go @@ -19,6 +19,7 @@ type CreateBoxDTO struct { Volumes []VolumeDTO `json:"volumes,omitempty"` NetworkBlockAll *bool `json:"networkBlockAll,omitempty"` NetworkAllowList *string `json:"networkAllowList,omitempty"` + CaptureLogs bool `json:"captureLogs,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` AuthToken *string `json:"authToken,omitempty"` OtelEndpoint *string `json:"otelEndpoint,omitempty"` @@ -53,6 +54,7 @@ type RecoverBoxDTO struct { Volumes []VolumeDTO `json:"volumes,omitempty"` NetworkBlockAll *bool `json:"networkBlockAll,omitempty"` NetworkAllowList *string `json:"networkAllowList,omitempty"` + CaptureLogs bool `json:"captureLogs,omitempty"` ErrorReason string `json:"errorReason" validate:"required"` } // @name RecoverBoxDTO diff --git a/apps/runner/pkg/boxlite/client.go b/apps/runner/pkg/boxlite/client.go index eed31e44d..12b380442 100644 --- a/apps/runner/pkg/boxlite/client.go +++ b/apps/runner/pkg/boxlite/client.go @@ -248,6 +248,9 @@ func (c *Client) Create(ctx context.Context, boxDto dto.CreateBoxDTO) (string, s opts = append(opts, boxlite.WithEntrypoint(boxDto.Entrypoint...)) } + if boxDto.CaptureLogs { + opts = append(opts, boxlite.WithCaptureLogs(true)) + } volumeMounts, err := c.getVolumeMounts(ctx, boxDto.Volumes) if err != nil { return "", "", err diff --git a/apps/runner/pkg/boxlite/stubs.go b/apps/runner/pkg/boxlite/stubs.go index 2062f7f39..36411a778 100644 --- a/apps/runner/pkg/boxlite/stubs.go +++ b/apps/runner/pkg/boxlite/stubs.go @@ -95,6 +95,7 @@ func (c *Client) RecoverBox(ctx context.Context, boxId string, recoverDto dto.Re NetworkBlockAll: recoverDto.NetworkBlockAll, NetworkAllowList: recoverDto.NetworkAllowList, FromVolumeId: recoverDto.FromVolumeId, + CaptureLogs: recoverDto.CaptureLogs, } _, _, err := c.Create(ctx, createDto) diff --git a/openapi/box.openapi.yaml b/openapi/box.openapi.yaml index ee6e8d5ec..a56f5d526 100644 --- a/openapi/box.openapi.yaml +++ b/openapi/box.openapi.yaml @@ -1652,6 +1652,10 @@ components: description: | Secret placeholder rules for outbound HTTP(S) requests. Matching requests replace `placeholder` with `value` on the host side. + capture_logs: + type: boolean + description: Capture init stdout/stderr as CRI records in the box-managed log path + default: false detach: type: boolean description: Box survives parent process exit diff --git a/sdks/c/include/boxlite.h b/sdks/c/include/boxlite.h index afc4d36c8..8e22d9f1d 100644 --- a/sdks/c/include/boxlite.h +++ b/sdks/c/include/boxlite.h @@ -715,6 +715,8 @@ void boxlite_options_set_auto_resume_enabled(CBoxliteOptions *opts, int val); void boxlite_options_set_detach(CBoxliteOptions *opts, int val); +void boxlite_options_set_capture_logs(CBoxliteOptions *opts, int val); + // Apply a `CAdvancedBoxOptions` (security, mount isolation, health check) to a // `CBoxliteOptions`. Clones the advanced configuration into the box options — // the caller retains ownership of `advanced_opts` and is responsible for diff --git a/sdks/c/src/options.rs b/sdks/c/src/options.rs index fe133468f..da950fcb4 100644 --- a/sdks/c/src/options.rs +++ b/sdks/c/src/options.rs @@ -190,6 +190,11 @@ pub unsafe extern "C" fn boxlite_options_set_detach(opts: *mut CBoxliteOptions, options_set_detach(opts, val) } +#[unsafe(no_mangle)] +pub unsafe extern "C" fn boxlite_options_set_capture_logs(opts: *mut CBoxliteOptions, val: c_int) { + options_set_capture_logs(opts, val) +} + /// Apply a `CAdvancedBoxOptions` (security, mount isolation, health check) to a /// `CBoxliteOptions`. Clones the advanced configuration into the box options — /// the caller retains ownership of `advanced_opts` and is responsible for @@ -475,6 +480,14 @@ pub unsafe fn options_set_detach(handle: *mut OptionsHandle, val: c_int) { } } +pub unsafe fn options_set_capture_logs(handle: *mut OptionsHandle, val: c_int) { + unsafe { + if !handle.is_null() { + (*handle).options.capture_logs = val != 0; + } + } +} + pub unsafe fn options_set_entrypoint( handle: *mut OptionsHandle, args: *const *const c_char, diff --git a/sdks/go/options.go b/sdks/go/options.go index dc2f645af..2c098315a 100644 --- a/sdks/go/options.go +++ b/sdks/go/options.go @@ -171,25 +171,26 @@ type Secret struct { } type boxConfig struct { - name string - cpus int - memoryMiB int - diskSizeGB int - rootfsPath string - env [][2]string - volumes []volumeEntry - ports []PortSpec - workDir string - entrypoint []string - cmd []string - autoRemove *bool - autoPause *uint32 - autoDelete *uint32 - autoResume *bool - detach *bool - network *NetworkSpec - secrets []Secret - advanced *AdvancedBoxOptions // nil = runtime defaults; non-nil = caller-owned advanced opts applied via boxlite_options_set_advanced + name string + cpus int + memoryMiB int + diskSizeGB int + rootfsPath string + env [][2]string + volumes []volumeEntry + ports []PortSpec + workDir string + entrypoint []string + cmd []string + autoRemove *bool + autoPause *uint32 + autoDelete *uint32 + autoResume *bool + detach *bool + network *NetworkSpec + secrets []Secret + captureLogs bool + advanced *AdvancedBoxOptions // nil = runtime defaults; non-nil = caller-owned advanced opts applied via boxlite_options_set_advanced } type volumeEntry struct { @@ -327,6 +328,11 @@ func WithDetach(v bool) BoxOption { return func(c *boxConfig) { c.detach = &v } } +// WithCaptureLogs captures init stdout/stderr as CRI records in the box-managed log path. +func WithCaptureLogs(v bool) BoxOption { + return func(c *boxConfig) { c.captureLogs = v } +} + // buildAndFreeCOptions runs buildCOptions, immediately frees the C // handle on success, and returns just the error. Exists for unit // tests in `_test.go` files, which Go forbids from using cgo @@ -502,6 +508,9 @@ func buildCOptions(image string, cfg *boxConfig) (*C.CBoxliteOptions, error) { if cfg.detach != nil { C.boxlite_options_set_detach(cOpts, boolToCInt(*cfg.detach)) } + if cfg.captureLogs { + C.boxlite_options_set_capture_logs(cOpts, C.int(1)) + } if cfg.advanced != nil && cfg.advanced.handle != nil { // Clone the caller-owned advanced options (security, …) onto the box. // The Go-side handle stays caller-owned; the box has its own copy after diff --git a/sdks/node/lib/native-contracts.ts b/sdks/node/lib/native-contracts.ts index 6cc70cf4f..d521fbd09 100644 --- a/sdks/node/lib/native-contracts.ts +++ b/sdks/node/lib/native-contracts.ts @@ -151,6 +151,7 @@ export interface JsBoxOptions { security?: JsSecurityOptions; healthCheck?: JsHealthCheckOptions; secrets?: JsSecret[]; + captureLogs?: boolean; } export interface JsOptions { diff --git a/sdks/node/lib/simplebox.ts b/sdks/node/lib/simplebox.ts index a581c0143..afc9d5b48 100644 --- a/sdks/node/lib/simplebox.ts +++ b/sdks/node/lib/simplebox.ts @@ -262,6 +262,9 @@ export interface SimpleBoxOptions { */ user?: string; + /** Capture init stdout/stderr as CRI records in the box-managed log path. */ + captureLogs?: boolean; + /** Security isolation options for the box. */ security?: SecurityOptions; } @@ -405,6 +408,7 @@ export class SimpleBox { entrypoint: options.entrypoint, cmd: options.cmd, user: options.user, + captureLogs: options.captureLogs, security, secrets: options.secrets, }; diff --git a/sdks/node/src/options.rs b/sdks/node/src/options.rs index ab47cc2c1..437966375 100644 --- a/sdks/node/src/options.rs +++ b/sdks/node/src/options.rs @@ -236,6 +236,10 @@ pub struct JsBoxOptions { #[napi(js_name = "healthCheck")] pub health_check: Option, + /// Capture init stdout/stderr as CRI records in the box-managed log path. + #[napi(js_name = "captureLogs")] + pub capture_logs: Option, + /// Secrets to inject into outbound HTTPS requests via MITM proxy. pub secrets: Option>, } @@ -449,6 +453,7 @@ impl TryFrom for BoxOptions { // client that attaches to the main command, which the SDKs cannot // do until they grow `attach()` (see sdk-run-semantics-api.md). tty: false, + capture_logs: js_opts.capture_logs.unwrap_or(false), secrets, }) } @@ -736,6 +741,7 @@ mod tests { security: None, health_check: None, secrets: None, + capture_logs: None, }; let mut both = js.clone(); @@ -785,6 +791,7 @@ mod tests { hosts: Some(vec!["api.openai.com".into()]), placeholder: None, }]), + capture_logs: None, }; let opts = BoxOptions::try_from(js).unwrap(); diff --git a/sdks/python/src/options.rs b/sdks/python/src/options.rs index 9518650c6..74f2e87c2 100644 --- a/sdks/python/src/options.rs +++ b/sdks/python/src/options.rs @@ -412,6 +412,10 @@ pub(crate) struct PyBoxOptions { #[pyo3(get, set)] pub(crate) advanced: Option, + /// Capture init stdout/stderr as CRI records in the box-managed log path. + #[pyo3(get, set)] + pub(crate) capture_logs: Option, + /// Secrets to inject into outbound HTTPS requests via MITM proxy. #[pyo3(get, set)] pub(crate) secrets: Vec, @@ -440,6 +444,7 @@ impl PyBoxOptions { cmd=None, user=None, advanced=None, + capture_logs=None, secrets=vec![], ))] #[allow(clippy::too_many_arguments)] @@ -463,6 +468,7 @@ impl PyBoxOptions { cmd: Option>, user: Option, advanced: Option, + capture_logs: Option, secrets: Vec, ) -> Self { Self { @@ -485,6 +491,7 @@ impl PyBoxOptions { cmd, user, advanced, + capture_logs, secrets, } } @@ -545,6 +552,7 @@ impl TryFrom for BoxOptions { entrypoint: py_opts.entrypoint, cmd: py_opts.cmd, user: py_opts.user, + capture_logs: py_opts.capture_logs.unwrap_or(false), ..Default::default() }; diff --git a/src/boxlite/src/litebox/init/tasks/guest_init.rs b/src/boxlite/src/litebox/init/tasks/guest_init.rs index 574048587..cca280b21 100644 --- a/src/boxlite/src/litebox/init/tasks/guest_init.rs +++ b/src/boxlite/src/litebox/init/tasks/guest_init.rs @@ -33,6 +33,7 @@ impl PipelineTask for GuestInitTask { volume_mgr, rootfs_init, container_mounts, + log_capture_path, network_spec, ca_cert_pem, tty, @@ -57,6 +58,7 @@ impl PipelineTask for GuestInitTask { BoxliteError::Internal("vmm_spawn task must run first".into()) })?; let network_spec = ctx.config.options.network.clone(); + let log_capture_path = ctx.log_capture_path.clone(); let ca_cert_pem = ctx.ca_cert_pem.clone(); let tty = ctx.config.options.tty; ( @@ -66,6 +68,7 @@ impl PipelineTask for GuestInitTask { volume_mgr, rootfs_init, container_mounts, + log_capture_path, network_spec, ca_cert_pem, tty, @@ -79,6 +82,7 @@ impl PipelineTask for GuestInitTask { &volume_mgr, &rootfs_init, &container_mounts, + log_capture_path, &network_spec, ca_cert_pem.as_deref(), tty, @@ -109,6 +113,7 @@ async fn run_guest_init( volume_mgr: &GuestVolumeManager, rootfs_init: &ContainerRootfsInitConfig, container_mounts: &[ContainerMount], + log_capture_path: Option, network_spec: &NetworkSpec, ca_cert_pem: Option<&str>, tty: bool, @@ -151,6 +156,7 @@ async fn run_guest_init( container_mounts.to_vec(), ca_certs, tty, + log_capture_path, ) .await?; tracing::info!(container_id = %returned_id, "Container created"); diff --git a/src/boxlite/src/litebox/init/tasks/vmm_spawn.rs b/src/boxlite/src/litebox/init/tasks/vmm_spawn.rs index 31f29fa8e..bbf6335ce 100644 --- a/src/boxlite/src/litebox/init/tasks/vmm_spawn.rs +++ b/src/boxlite/src/litebox/init/tasks/vmm_spawn.rs @@ -11,7 +11,7 @@ use crate::litebox::init::types::resolve_user_volumes; use crate::net::{NetworkBackend, NetworkBackendConfig}; use crate::pipeline::PipelineTask; use crate::rootfs::guest::{GuestRootfs, Strategy}; -use crate::runtime::constants::{guest_paths, mount_tags}; +use crate::runtime::constants::{guest_paths, logs, mount_tags}; use crate::runtime::id::BoxID; use crate::runtime::layout::BoxFilesystemLayout; use crate::runtime::options::BoxOptions; @@ -27,7 +27,7 @@ use async_trait::async_trait; use boxlite_shared::BoxTransport; use boxlite_shared::errors::{BoxliteError, BoxliteResult}; use std::collections::{HashMap, HashSet}; -use std::path::Path; +use std::path::{Path, PathBuf}; pub struct VmmSpawnTask; @@ -77,20 +77,26 @@ impl PipelineTask for VmmSpawnTask { }; // Build config and get outputs - let (instance_spec, volume_mgr, rootfs_init, container_mounts, network_backend) = - build_config( - &box_id, - &options, - &layout, - &container_image_config, - &container_disk_path, - guest_disk_path.as_deref(), - &container_id, - &runtime, - reuse_rootfs, - ) - .await - .inspect_err(|e| log_task_error(&box_id, task_name, e))?; + let ( + instance_spec, + volume_mgr, + rootfs_init, + container_mounts, + network_backend, + log_capture_path, + ) = build_config( + &box_id, + &options, + &layout, + &container_image_config, + &container_disk_path, + guest_disk_path.as_deref(), + &container_id, + &runtime, + reuse_rootfs, + ) + .await + .inspect_err(|e| log_task_error(&box_id, task_name, e))?; // Spawn VM let handler = spawn_vm(&box_id, &instance_spec, &options, &layout) @@ -101,6 +107,7 @@ impl PipelineTask for VmmSpawnTask { ctx.guard.set_handler(handler); ctx.volume_mgr = Some(volume_mgr); ctx.rootfs_init = Some(rootfs_init); + ctx.log_capture_path = log_capture_path; ctx.container_mounts = Some(container_mounts); // Hand the box's one network backend to LiveState assembly (init/mod.rs). ctx.network_backend = network_backend; @@ -135,6 +142,7 @@ async fn build_config( crate::portal::interfaces::ContainerRootfsInitConfig, Vec, Option>, + Option, )> { // BoxTransport setup let transport = BoxTransport::unix(layout.socket_path()); @@ -151,6 +159,7 @@ async fn build_config( // SHARED virtiofs - needed by all strategies volume_mgr.add_fs_share(mount_tags::SHARED, layout.shared_dir(), None, false, None); + let log_capture_path = configure_log_capture(options, layout, container_id, &mut volume_mgr)?; // Add container rootfs disk (COW overlay workflow): // 1. Base disk: Pre-built ext4 image with container layers merged @@ -254,13 +263,60 @@ async fn build_config( exit_file: layout.exit_file_path(), detach: options.detach, }; - Ok(( instance_spec, volume_mgr, rootfs_init, container_mounts, network_backend, + log_capture_path, + )) +} + +fn configure_log_capture( + options: &BoxOptions, + layout: &BoxFilesystemLayout, + container_id: &ContainerID, + volume_mgr: &mut GuestVolumeManager, +) -> BoxliteResult> { + if !options.capture_logs { + return Ok(None); + } + + let managed_log_path = layout.container_log_path(container_id.as_str()); + let managed_log_dir = layout.container_logs_dir(container_id.as_str()); + std::fs::create_dir_all(&managed_log_dir).map_err(|e| { + BoxliteError::Storage(format!( + "failed to create container log directory {}: {}", + managed_log_dir.display(), + e + )) + })?; + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&managed_log_path) + .map_err(|e| { + BoxliteError::Storage(format!( + "failed to create container log file {}: {}", + managed_log_path.display(), + e + )) + })?; + + volume_mgr.add_fs_share( + mount_tags::CONTAINER_LOGS, + managed_log_dir, + Some(logs::GUEST_DIR), + false, + None, + ); + + Ok(Some( + PathBuf::from(logs::GUEST_DIR) + .join("console.log") + .to_string_lossy() + .into_owned(), )) } diff --git a/src/boxlite/src/litebox/init/types.rs b/src/boxlite/src/litebox/init/types.rs index b6001332c..58b428e40 100644 --- a/src/boxlite/src/litebox/init/types.rs +++ b/src/boxlite/src/litebox/init/types.rs @@ -298,6 +298,7 @@ pub struct InitPipelineContext { pub volume_mgr: Option, pub rootfs_init: Option, pub container_mounts: Option>, + pub log_capture_path: Option, pub guest_session: Option, /// The box's one network backend (set by vmm_spawn on first start/restart, or /// by vmm_attach on reattach; moved into LiveState for runtime control). @@ -330,6 +331,7 @@ impl InitPipelineContext { volume_mgr: None, rootfs_init: None, container_mounts: None, + log_capture_path: None, guest_session: None, network_backend: None, ca_cert_pem: None, diff --git a/src/boxlite/src/portal/interfaces/container.rs b/src/boxlite/src/portal/interfaces/container.rs index ce812baca..13ab48d99 100644 --- a/src/boxlite/src/portal/interfaces/container.rs +++ b/src/boxlite/src/portal/interfaces/container.rs @@ -101,6 +101,7 @@ impl ContainerInterface { mounts: Vec, ca_certs: Vec, tty: bool, + log_capture_path: Option, ) -> BoxliteResult { let proto_config = ProtoContainerConfig { entrypoint: image_config.final_cmd(), @@ -149,6 +150,7 @@ impl ContainerInterface { // `LiteBox::attach()` can address the main command with the same id // it sent, instead of both sides separately hard-coding it. execution_id: container_id.to_string(), + log_capture_path, }; let response = self.client.init(request).await?.into_inner(); diff --git a/src/boxlite/src/rest/types.rs b/src/boxlite/src/rest/types.rs index 4ae496828..f8ee06633 100644 --- a/src/boxlite/src/rest/types.rs +++ b/src/boxlite/src/rest/types.rs @@ -114,6 +114,8 @@ pub(crate) struct CreateBoxRequest { #[serde(skip_serializing_if = "Option::is_none")] pub secrets: Option>, #[serde(skip_serializing_if = "Option::is_none")] + pub capture_logs: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub detach: Option, /// A terminal for the main command (`run -t`). Only sent when asked for: /// the server rejects unknown fields, so an older one would 400 on it — @@ -175,6 +177,7 @@ impl CreateBoxRequest { cmd: options.cmd.clone(), user: options.user.clone(), secrets, + capture_logs: Some(options.capture_logs), detach: Some(options.detach), tty: options.tty.then_some(true), // The deprecated remove-on-stop flag was never applied by the cloud @@ -583,6 +586,7 @@ mod tests { hosts: vec!["api.openai.com".into()], placeholder: "".into(), }]), + capture_logs: None, detach: None, auto_pause: Some(900), auto_delete: Some(604800), diff --git a/src/boxlite/src/runtime/constants.rs b/src/boxlite/src/runtime/constants.rs index 22e65bf4a..aafef7690 100644 --- a/src/boxlite/src/runtime/constants.rs +++ b/src/boxlite/src/runtime/constants.rs @@ -4,7 +4,7 @@ //! Host controls all paths - guest receives these via GuestInitRequest. // Re-export shared constants from boxlite-core -pub use boxlite_shared::constants::{container, mount_tags, network}; +pub use boxlite_shared::constants::{container, logs, mount_tags, network}; /// Guest mount points (paths inside the guest). /// diff --git a/src/boxlite/src/runtime/layout.rs b/src/boxlite/src/runtime/layout.rs index 6c7b9ee06..3ab508090 100644 --- a/src/boxlite/src/runtime/layout.rs +++ b/src/boxlite/src/runtime/layout.rs @@ -330,7 +330,9 @@ impl FilesystemLayout { /// ├── shared/ # Guest-visible (ro bind mount → mounts/) /// ├── logs/ # Per-box logging /// │ ├── boxlite-shim.log # Shim tracing output -/// │ └── console.log # Kernel/init output +/// │ ├── console.log # Kernel/init output +/// │ └── {cid}/ +/// │ └── console.log # Optional CRI-formatted init stdout/stderr /// └── disks/ # Disk images /// ├── disk.qcow2 # Data disk (container rootfs COW disk) /// └── guest-rootfs.qcow2 # Guest rootfs COW overlay @@ -488,6 +490,20 @@ impl BoxFilesystemLayout { self.box_dir.join("logs") } + /// Per-container logs directory: ~/.boxlite/boxes/{box_id}/logs/{container_id} + /// + /// This is the only logs subtree exposed writable to the guest. VM console + /// and shim logs remain host-only siblings under [`Self::logs_dir`]. + pub fn container_logs_dir(&self, container_id: &str) -> PathBuf { + self.logs_dir().join(container_id) + } + + /// Container init stdout/stderr log path: + /// ~/.boxlite/boxes/{box_id}/logs/{container_id}/console.log + pub fn container_log_path(&self, container_id: &str) -> PathBuf { + self.container_logs_dir(container_id).join("console.log") + } + /// Per-box temp directory: ~/.boxlite/boxes/{box_id}/tmp /// /// Used for shim/libkrun transient files when jailer is enabled with the @@ -986,6 +1002,23 @@ mod tests { ); } + #[test] + fn test_box_layout_container_log_is_isolated_from_host_logs() { + let layout = test_box_layout("/home/.boxlite/boxes/mybox"); + assert_eq!( + layout.container_logs_dir("container-1"), + PathBuf::from("/home/.boxlite/boxes/mybox/logs/container-1") + ); + assert_eq!( + layout.container_log_path("container-1"), + PathBuf::from("/home/.boxlite/boxes/mybox/logs/container-1/console.log") + ); + assert_ne!( + layout.container_log_path("container-1"), + layout.console_output_path() + ); + } + #[test] fn test_box_layout_bin_dir() { let layout = test_box_layout("/home/.boxlite/boxes/mybox"); diff --git a/src/boxlite/src/runtime/options.rs b/src/boxlite/src/runtime/options.rs index 0e8c6b857..2555bb1c4 100644 --- a/src/boxlite/src/runtime/options.rs +++ b/src/boxlite/src/runtime/options.rs @@ -415,6 +415,14 @@ pub struct BoxOptions { #[serde(default)] pub tty: bool, + /// Capture the container init process stdout/stderr as CRI-formatted records. + /// + /// This is opt-in because it creates an additional writable guest share. + /// When enabled, logs are written to `logs/{container_id}/console.log` under + /// the box-managed directory, with stream identity, timestamps, and rotation. + #[serde(default)] + pub capture_logs: bool, + /// Secrets for MITM proxy injection into outbound HTTP(S) requests. /// /// Each secret maps a placeholder string to a real value. When the box @@ -511,6 +519,7 @@ fn default_detach() -> bool { impl Default for BoxOptions { fn default() -> Self { Self { + capture_logs: false, cpus: None, memory_mib: None, disk_size_gb: None, diff --git a/src/guest/Cargo.toml b/src/guest/Cargo.toml index 0713d2136..a2d1253f0 100644 --- a/src/guest/Cargo.toml +++ b/src/guest/Cargo.toml @@ -21,6 +21,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } nix = { version = "0.29", features = ["mount", "process", "fs", "sched", "socket", "poll"] } async-trait = "0.1" +chrono = "0.4" uuid = { version = "1.10", features = ["v4"] } tonic = "0.12" tokio-stream = { version = "0.1", features = ["sync"] } diff --git a/src/guest/src/container/lifecycle.rs b/src/guest/src/container/lifecycle.rs index de6ebe8e4..4f9c7fd7c 100644 --- a/src/guest/src/container/lifecycle.rs +++ b/src/guest/src/container/lifecycle.rs @@ -106,6 +106,7 @@ impl Container { user: &str, user_mounts: Vec, tty: bool, + log_capture: Option, ) -> BoxliteResult { let rootfs = rootfs.as_ref(); let workdir = workdir.as_ref(); @@ -199,7 +200,8 @@ impl Container { } else { // Pipes exist before the container does. The guest holds stdin's // write-end, so init's read() blocks instead of seeing EOF. - let (stdio, init_fds) = ContainerStdio::pipes()?; + let (mut stdio, init_fds) = ContainerStdio::pipes()?; + stdio.start_output_pump(log_capture)?; start::create_container_with_stdio( container_id, &state_root, @@ -369,10 +371,10 @@ impl Container { ) } - /// Drain init process stdout and stderr. + /// Take the bounded diagnostic tails for init stdout and stderr. /// - /// Reads all available data from the init process pipes using non-blocking I/O. - /// Can only be called once — subsequent calls return empty strings. + /// The always-on output pump owns the pipes and retains the latest bytes + /// per stream. Taking the tails clears them; subsequent calls are empty. /// /// # Returns /// @@ -404,7 +406,7 @@ impl Container { pub fn diagnose_exit(&mut self) -> String { let container_state_path = self.container_state_path(); - // Drain init process output before building diagnostics + // Take the bounded init output tails before building diagnostics let (init_stdout, init_stderr) = self.drain_init_output(); // Try to load container state from libcontainer @@ -483,7 +485,8 @@ impl Container { /// # Returns /// /// Ok(()) on successful shutdown, or if container was already stopped. - pub fn shutdown(&self, timeout_ms: u64) -> BoxliteResult<()> { + pub async fn shutdown(&mut self, timeout_ms: u64) -> BoxliteResult<()> { + const LOG_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); self.is_shutdown .store(true, std::sync::atomic::Ordering::SeqCst); @@ -492,12 +495,14 @@ impl Container { Ok(c) => c, Err(_) => { tracing::debug!(container_id = %self.id, "Container already gone, nothing to shutdown"); + self.stdio.finish_output(LOG_DRAIN_TIMEOUT).await; return Ok(()); } }; if !container.can_kill() { tracing::debug!(container_id = %self.id, "Container cannot be killed, skipping shutdown"); + self.stdio.finish_output(LOG_DRAIN_TIMEOUT).await; return Ok(()); } @@ -511,9 +516,10 @@ impl Container { while start.elapsed().as_millis() < timeout_ms as u128 { if !self.is_running() { tracing::info!(container_id = %self.id, "Container exited gracefully"); + self.stdio.finish_output(LOG_DRAIN_TIMEOUT).await; return Ok(()); } - std::thread::sleep(std::time::Duration::from_millis(100)); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; } // Step 3: SIGKILL if still running @@ -521,6 +527,7 @@ impl Container { let sigkill = Signal::try_from(9).expect("SIGKILL (9) is a valid signal"); let _ = container.kill(sigkill, true); + self.stdio.finish_output(LOG_DRAIN_TIMEOUT).await; Ok(()) } diff --git a/src/guest/src/container/mod.rs b/src/guest/src/container/mod.rs index 9695fab96..6db4f453b 100644 --- a/src/guest/src/container/mod.rs +++ b/src/guest/src/container/mod.rs @@ -16,10 +16,14 @@ //! # async fn example() -> Result<(), Box> { //! // Create and start container //! let container = Container::start( +//! "my-container", //! "/rootfs", //! vec!["sh".to_string()], //! vec!["PATH=/bin:/usr/bin".to_string()], //! "/", +//! "0:0", +//! Vec::new(), +//! None, //! )?; //! //! // Execute command diff --git a/src/guest/src/container/stdio.rs b/src/guest/src/container/stdio.rs index d6bf0309a..1ebdf0526 100644 --- a/src/guest/src/container/stdio.rs +++ b/src/guest/src/container/stdio.rs @@ -34,8 +34,24 @@ use boxlite_shared::errors::{BoxliteError, BoxliteResult}; use nix::unistd::pipe; +use std::collections::VecDeque; use std::io::Read; use std::os::unix::io::{AsRawFd, OwnedFd}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::unix::pipe::Receiver; +use tokio::task::JoinHandle; + +/// Maximum size of the active log before rotation. +const LOG_MAX_SIZE: u64 = 10 * 1024 * 1024; +/// Total retained files, including the active log. +const LOG_MAX_FILES: usize = 5; +/// Maximum raw content bytes in one CRI record. +const CRI_MAX_CONTENT_SIZE: usize = 16 * 1024; +/// Retained diagnostic output per stream. +const DIAGNOSTIC_TAIL_SIZE: usize = 8 * 1024; /// The guest's side of the container init process's stdio. /// @@ -66,10 +82,14 @@ pub enum ContainerStdio { /// session then owns keeping stdin open, and closing its stream /// delivers real EOF to init (docker `run -i` semantics). stdin_tx: Option, - /// Read-end of stdout (taken by drain_output for log capture) + /// Read-end of the stdout relay handed to the attach session. stdout_rx: Option, - /// Read-end of stderr (taken by drain_output for log capture) + /// Read-end of the stderr relay handed to the attach session. stderr_rx: Option, + /// One task that owns and drains the original stdout/stderr pipes. + output_task: Option>, + /// Bounded stdout/stderr tails used for startup diagnostics. + diagnostics: Arc>, }, Pty { /// PTY master received from libcontainer over the console socket. @@ -146,11 +166,13 @@ impl ContainerStdio { let (stderr_rx, stderr_tx) = pipe() .map_err(|e| BoxliteError::Internal(format!("Failed to create stderr pipe: {}", e)))?; - // nix::unistd::pipe() returns OwnedFd directly + let diagnostics = Arc::new(Mutex::new(DiagnosticTails::default())); let container_stdio = Self::Pipes { stdin_tx: Some(stdin_tx), stdout_rx: Some(stdout_rx), stderr_rx: Some(stderr_rx), + output_task: None, + diagnostics, }; let init_fds = InitStdioFds { @@ -164,22 +186,163 @@ impl ContainerStdio { Ok((container_stdio, init_fds)) } - /// Drain all available output from the init process. - /// - /// Takes ownership of the read-ends and reads with non-blocking I/O. - /// Can only be called once — subsequent calls return empty strings. - /// - /// # Returns - /// - /// `(stdout, stderr)` — captured output, truncated to 4 KiB each. On a PTY - /// stderr is always empty: the terminal merged it into stdout. + /// Start the single owner of init stdout/stderr. It always drains both + /// pipes, keeps bounded diagnostic tails, and relays raw bytes to attach. + /// When `path` is present, it also writes CRI-formatted rotating logs. + pub fn start_output_pump(&mut self, path: Option) -> BoxliteResult<()> { + let Self::Pipes { + stdout_rx, + stderr_rx, + output_task, + diagnostics, + .. + } = self + else { + return Err(BoxliteError::InvalidArgument( + "the pipe output pump is not used with TTY mode".to_string(), + )); + }; + + let writer = path + .map(|path| { + let log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + .map_err(|e| { + BoxliteError::Storage(format!( + "Failed to open container log {}: {}", + path.display(), + e + )) + })?; + Ok::<_, BoxliteError>(CaptureWriter::new(log, path)) + }) + .transpose()?; + + let stdout = stdout_rx + .take() + .map(Receiver::from_owned_fd) + .transpose()? + .ok_or_else(|| BoxliteError::Internal("stdout pipe is unavailable".to_string()))?; + let stderr = stderr_rx + .take() + .map(Receiver::from_owned_fd) + .transpose()? + .ok_or_else(|| BoxliteError::Internal("stderr pipe is unavailable".to_string()))?; + + let (stdout_relay_rx, stdout_relay_tx) = pipe().map_err(|e| { + BoxliteError::Internal(format!("Failed to create stdout relay pipe: {}", e)) + })?; + let (stderr_relay_rx, stderr_relay_tx) = pipe().map_err(|e| { + BoxliteError::Internal(format!("Failed to create stderr relay pipe: {}", e)) + })?; + set_nonblocking(&stdout_relay_tx)?; + set_nonblocking(&stderr_relay_tx)?; + *stdout_rx = Some(stdout_relay_rx); + *stderr_rx = Some(stderr_relay_rx); + + let diagnostics = Arc::clone(diagnostics); + tracing::debug!(capture = writer.is_some(), "Starting container init output pump"); + *output_task = Some(tokio::spawn(async move { + let mut writer = writer; + let mut stdout_cri = CriStream::new("stdout"); + let mut stderr_cri = CriStream::new("stderr"); + let mut stdout = stdout; + let mut stderr = stderr; + let mut stdout_open = true; + let mut stderr_open = true; + let mut stdout_relay = Some(stdout_relay_tx); + let mut stderr_relay = Some(stderr_relay_tx); + let mut stdout_buf = [0u8; 4096]; + let mut stderr_buf = [0u8; 4096]; + + while stdout_open || stderr_open { + tokio::select! { + result = stdout.read(&mut stdout_buf), if stdout_open => { + match result { + Ok(0) => { + stdout_open = false; + if let Some(writer) = writer.as_mut() { + stdout_cri.finish(writer).await; + } + } + Ok(n) => { + let chunk = &stdout_buf[..n]; + diagnostics.lock().expect("diagnostic tails poisoned").stdout.push(chunk); + relay(&mut stdout_relay, chunk); + if let Some(writer) = writer.as_mut() { + stdout_cri.push(writer, chunk).await; + } + } + Err(error) => { + tracing::warn!(%error, "Failed to read container init stdout"); + stdout_open = false; + if let Some(writer) = writer.as_mut() { + stdout_cri.finish(writer).await; + } + } + } + } + result = stderr.read(&mut stderr_buf), if stderr_open => { + match result { + Ok(0) => { + stderr_open = false; + if let Some(writer) = writer.as_mut() { + stderr_cri.finish(writer).await; + } + } + Ok(n) => { + let chunk = &stderr_buf[..n]; + diagnostics.lock().expect("diagnostic tails poisoned").stderr.push(chunk); + relay(&mut stderr_relay, chunk); + if let Some(writer) = writer.as_mut() { + stderr_cri.push(writer, chunk).await; + } + } + Err(error) => { + tracing::warn!(%error, "Failed to read container init stderr"); + stderr_open = false; + if let Some(writer) = writer.as_mut() { + stderr_cri.finish(writer).await; + } + } + } + } + } + } + + if let Some(writer) = writer.as_mut() { + writer.flush().await; + } + tracing::debug!("Container init output pump finished"); + })); + Ok(()) + } + + /// Wait for the output task to drain both pipes and flush any capture log. + pub async fn finish_output(&mut self, timeout: Duration) { + let Self::Pipes { output_task, .. } = self else { + return; + }; + let Some(mut task) = output_task.take() else { + return; + }; + + if tokio::time::timeout(timeout, &mut task).await.is_err() { + tracing::warn!(?timeout, "Timed out draining container init output"); + task.abort(); + let _ = task.await; + } + } + /// Return and clear the bounded diagnostic tail for each init stream. + /// On a PTY, stderr is always empty because the terminal merged it into stdout. pub fn drain_output(&mut self) -> (String, String) { match self { - Self::Pipes { - stdout_rx, - stderr_rx, - .. - } => (drain_fd(stdout_rx.take()), drain_fd(stderr_rx.take())), + Self::Pipes { diagnostics, .. } => diagnostics + .lock() + .expect("diagnostic tails poisoned") + .take(), Self::Pty { master } => (drain_fd(master.take()), String::new()), } } @@ -194,16 +357,15 @@ impl ContainerStdio { /// its stdin stream drops the last write-end and init sees real EOF /// (docker `run -i` piped semantics). A duplicate held here would make EOF /// impossible. - /// - /// The read-ends moving out also means a later `drain_output` (init exit - /// diagnostics) returns empty strings — when init is the user's workload, - /// its output belongs to the session, not to a 4 KiB diagnostic capture. + /// The output fds are relay read-ends. The output pump remains the sole + /// reader of init's original pipes and independently keeps diagnostic tails. pub fn take_init_io(&mut self) -> BoxliteResult> { match self { Self::Pipes { stdin_tx, stdout_rx, stderr_rx, + .. } => { let (Some(stdin), Some(stdout), Some(stderr)) = (stdin_tx.take(), stdout_rx.take(), stderr_rx.take()) @@ -221,6 +383,239 @@ impl ContainerStdio { } } +impl Drop for ContainerStdio { + fn drop(&mut self) { + if let Self::Pipes { + output_task: Some(task), + .. + } = self + { + task.abort(); + } + } +} + +#[derive(Debug, Default)] +pub(crate) struct DiagnosticTails { + stdout: ByteTail, + stderr: ByteTail, +} + +impl DiagnosticTails { + fn take(&mut self) -> (String, String) { + (self.stdout.take_string(), self.stderr.take_string()) + } +} + +#[derive(Debug, Default)] +struct ByteTail { + bytes: VecDeque, +} + +impl ByteTail { + fn push(&mut self, chunk: &[u8]) { + if chunk.len() >= DIAGNOSTIC_TAIL_SIZE { + self.bytes.clear(); + self.bytes + .extend(&chunk[chunk.len() - DIAGNOSTIC_TAIL_SIZE..]); + return; + } + + let overflow = self + .bytes + .len() + .saturating_add(chunk.len()) + .saturating_sub(DIAGNOSTIC_TAIL_SIZE); + self.bytes.drain(..overflow); + self.bytes.extend(chunk); + } + + fn take_string(&mut self) -> String { + let bytes: Vec = self.bytes.drain(..).collect(); + String::from_utf8_lossy(&bytes).into_owned() + } +} + +struct CriStream { + name: &'static str, + pending: Vec, +} + +impl CriStream { + fn new(name: &'static str) -> Self { + Self { + name, + pending: Vec::new(), + } + } + + async fn push(&mut self, writer: &mut CaptureWriter, chunk: &[u8]) { + self.pending.extend_from_slice(chunk); + + while let Some(newline) = self.pending.iter().position(|byte| *byte == b'\n') { + let mut line: Vec = self.pending.drain(..=newline).collect(); + line.pop(); + self.write_complete_line(writer, &line).await; + } + + while self.pending.len() > CRI_MAX_CONTENT_SIZE { + let partial: Vec = self.pending.drain(..CRI_MAX_CONTENT_SIZE).collect(); + writer.write_cri(self.name, 'P', &partial).await; + } + } + + async fn finish(&mut self, writer: &mut CaptureWriter) { + if !self.pending.is_empty() { + let final_fragment = std::mem::take(&mut self.pending); + self.write_complete_line(writer, &final_fragment).await; + } + } + + async fn write_complete_line(&self, writer: &mut CaptureWriter, line: &[u8]) { + if line.is_empty() { + writer.write_cri(self.name, 'F', line).await; + return; + } + + let mut chunks = line.chunks(CRI_MAX_CONTENT_SIZE).peekable(); + while let Some(chunk) = chunks.next() { + let tag = if chunks.peek().is_some() { 'P' } else { 'F' }; + writer.write_cri(self.name, tag, chunk).await; + } + } +} + +fn relay(fd: &mut Option, chunk: &[u8]) { + let Some(relay) = fd.as_ref() else { + return; + }; + + match nix::unistd::write(relay, chunk) { + Ok(_) | Err(nix::errno::Errno::EAGAIN) => {} + Err(nix::errno::Errno::EPIPE) => *fd = None, + Err(error) => { + tracing::warn!(%error, "Failed to relay container init output"); + *fd = None; + } + } +} + +fn set_nonblocking(fd: &OwnedFd) -> BoxliteResult<()> { + let flags = nix::fcntl::fcntl(fd.as_raw_fd(), nix::fcntl::FcntlArg::F_GETFL) + .map_err(|e| BoxliteError::Internal(format!("Failed to read relay pipe flags: {}", e)))?; + let mut flags = nix::fcntl::OFlag::from_bits_truncate(flags); + flags.insert(nix::fcntl::OFlag::O_NONBLOCK); + nix::fcntl::fcntl(fd.as_raw_fd(), nix::fcntl::FcntlArg::F_SETFL(flags)) + .map_err(|e| BoxliteError::Internal(format!("Failed to update relay pipe flags: {}", e)))?; + Ok(()) +} + +struct CaptureWriter { + file: tokio::fs::File, + path: PathBuf, + current_size: u64, + max_size: u64, + max_files: usize, + failed: bool, +} + +impl CaptureWriter { + fn new(file: std::fs::File, path: PathBuf) -> Self { + let current_size = file.metadata().map(|metadata| metadata.len()).unwrap_or(0); + Self { + file: tokio::fs::File::from_std(file), + path, + current_size, + max_size: LOG_MAX_SIZE, + max_files: LOG_MAX_FILES, + failed: false, + } + } + + async fn write_cri(&mut self, stream: &'static str, tag: char, content: &[u8]) { + if self.failed { + return; + } + + let timestamp = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Nanos, true); + let content = String::from_utf8_lossy(content); + let record = format!("{timestamp} {stream} {tag} {content}\n"); + let record = record.as_bytes(); + + if self.current_size > 0 && self.current_size + record.len() as u64 > self.max_size { + if let Err(error) = self.rotate().await { + tracing::warn!( + path = %self.path.display(), + %error, + "Failed to rotate container init log; continuing with current file" + ); + } + } + + if let Err(error) = self.file.write_all(record).await { + self.failed = true; + tracing::warn!( + stream, + path = %self.path.display(), + %error, + "Container init log write failed; continuing to drain output" + ); + } else { + self.current_size += record.len() as u64; + } + } + + async fn rotate(&mut self) -> std::io::Result<()> { + self.file.flush().await?; + + let mut replacement_path = self.path.as_os_str().to_os_string(); + replacement_path.push(".tmp"); + let replacement_path = PathBuf::from(replacement_path); + let replacement = tokio::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&replacement_path) + .await?; + + let archive_count = self.max_files - 1; + let oldest = self.archive_path(archive_count); + if tokio::fs::try_exists(&oldest).await? { + tokio::fs::remove_file(oldest).await?; + } + + for index in (1..archive_count).rev() { + let source = self.archive_path(index); + if tokio::fs::try_exists(&source).await? { + tokio::fs::rename(source, self.archive_path(index + 1)).await?; + } + } + + let newest = self.archive_path(1); + tokio::fs::rename(&self.path, &newest).await?; + if let Err(error) = tokio::fs::rename(&replacement_path, &self.path).await { + tokio::fs::rename(newest, &self.path).await?; + return Err(error); + } + + self.file = replacement; + self.current_size = 0; + Ok(()) + } + + fn archive_path(&self, index: usize) -> PathBuf { + let mut path = self.path.as_os_str().to_os_string(); + path.push(format!(".{index}")); + path.into() + } + + async fn flush(&mut self) { + if !self.failed { + let _ = self.file.flush().await; + } + } +} + /// Read all available data from an fd using non-blocking I/O. fn drain_fd(fd: Option) -> String { const MAX_CAPTURE: usize = 4096; @@ -267,6 +662,33 @@ mod tests { use std::io::Write; use std::os::unix::io::AsRawFd; + fn capture_writer(path: &std::path::Path, max_size: u64, max_files: usize) -> CaptureWriter { + let file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .unwrap(); + let current_size = file.metadata().unwrap().len(); + CaptureWriter { + file: tokio::fs::File::from_std(file), + path: path.to_path_buf(), + current_size, + max_size, + max_files, + failed: false, + } + } + + fn parse_cri_record(line: &str) -> (String, String, &str, String) { + let mut fields = line.splitn(4, ' '); + let timestamp = fields.next().expect("timestamp").to_string(); + let stream = fields.next().expect("stream").to_string(); + let tag = fields.next().expect("tag"); + let message = fields.next().unwrap_or_default().to_string(); + chrono::DateTime::parse_from_rfc3339(×tamp).expect("RFC3339 timestamp"); + (timestamp, stream, tag, message) + } + /// Borrow the parent-side pipe fds. Panics on a `Pty`, which these tests /// never build. fn pipe_fds(stdio: &ContainerStdio) -> [i32; 3] { @@ -274,6 +696,7 @@ mod tests { stdin_tx, stdout_rx, stderr_rx, + .. } = stdio else { panic!("expected pipes"); @@ -335,38 +758,276 @@ mod tests { assert!(stdio.take_init_io().unwrap().is_none()); } - #[test] - fn test_drain_output_captures_data() { + #[tokio::test] + async fn test_output_pump_keeps_and_clears_diagnostic_tails() { let (mut stdio, init_fds) = ContainerStdio::pipes().unwrap(); + stdio.start_output_pump(None).unwrap(); + let _relay = stdio.take_init_io().unwrap().unwrap(); - // Write to the init side of pipes (simulating init process output) let mut stdout_writer = std::fs::File::from(init_fds.stdout); let mut stderr_writer = std::fs::File::from(init_fds.stderr); stdout_writer.write_all(b"hello stdout").unwrap(); stderr_writer.write_all(b"hello stderr").unwrap(); drop(stdout_writer); drop(stderr_writer); + stdio.finish_output(Duration::from_secs(1)).await; let (stdout, stderr) = stdio.drain_output(); assert_eq!(stdout, "hello stdout"); assert_eq!(stderr, "hello stderr"); + assert_eq!(stdio.drain_output(), (String::new(), String::new())); } - #[test] - fn test_drain_output_returns_empty_on_second_call() { + #[tokio::test] + async fn test_output_pump_retains_only_the_diagnostic_tail() { let (mut stdio, init_fds) = ContainerStdio::pipes().unwrap(); - + stdio.start_output_pump(None).unwrap(); + let _relay = stdio.take_init_io().unwrap().unwrap(); + let payload = vec![b'x'; DIAGNOSTIC_TAIL_SIZE + 1024]; let mut stdout_writer = std::fs::File::from(init_fds.stdout); - stdout_writer.write_all(b"data").unwrap(); + drop(init_fds.stderr); + stdout_writer.write_all(&payload).unwrap(); drop(stdout_writer); + stdio.finish_output(Duration::from_secs(1)).await; + + let (stdout, stderr) = stdio.drain_output(); + assert_eq!(stdout.len(), DIAGNOSTIC_TAIL_SIZE); + assert!(stdout.bytes().all(|byte| byte == b'x')); + assert!(stderr.is_empty()); + } + + #[tokio::test] + async fn test_capture_off_does_not_block_when_relay_is_unread() { + let (mut stdio, init_fds) = ContainerStdio::pipes().unwrap(); + let payload = vec![b'x'; 256 * 1024]; + + stdio.start_output_pump(None).unwrap(); + let _unread_relay = stdio.take_init_io().unwrap().unwrap(); + let mut stdout_writer = std::fs::File::from(init_fds.stdout); drop(init_fds.stderr); + let write_task = tokio::task::spawn_blocking(move || { + stdout_writer.write_all(&payload).unwrap(); + }); + + tokio::time::timeout(Duration::from_secs(2), write_task) + .await + .expect("output pump blocked behind an unread relay") + .unwrap(); + stdio.finish_output(Duration::from_secs(1)).await; let (stdout, _) = stdio.drain_output(); - assert_eq!(stdout, "data"); + assert_eq!(stdout.len(), DIAGNOSTIC_TAIL_SIZE); + } + + #[tokio::test] + async fn test_capture_on_does_not_block_when_relay_is_unread() { + let (mut stdio, init_fds) = ContainerStdio::pipes().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("console.log"); + let payload = vec![b'y'; 256 * 1024]; + let expected = payload.len(); + + stdio.start_output_pump(Some(log_path.clone())).unwrap(); + let _unread_relay = stdio.take_init_io().unwrap().unwrap(); + let mut stdout_writer = std::fs::File::from(init_fds.stdout); + drop(init_fds.stderr); + let write_task = tokio::task::spawn_blocking(move || { + stdout_writer.write_all(&payload).unwrap(); + }); + + tokio::time::timeout(Duration::from_secs(2), write_task) + .await + .expect("capture blocked behind an unread relay") + .unwrap(); + stdio.finish_output(Duration::from_secs(1)).await; + + let content = std::fs::read_to_string(log_path).unwrap(); + let records: Vec<_> = content.lines().map(parse_cri_record).collect(); + assert!(records.iter().all(|record| record.1 == "stdout")); + assert_eq!( + records.iter().map(|record| record.3.len()).sum::(), + expected + ); + } + + #[tokio::test] + async fn test_output_pump_writes_cri_log_and_relays_raw_output() { + let (mut stdio, init_fds) = ContainerStdio::pipes().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("console.log"); + + stdio.start_output_pump(Some(log_path.clone())).unwrap(); + let InitIo::Pipes { stdout, stderr, .. } = stdio.take_init_io().unwrap().unwrap() else { + panic!("expected relay pipes"); + }; + + let mut stdout_writer = std::fs::File::from(init_fds.stdout); + let mut stderr_writer = std::fs::File::from(init_fds.stderr); + stdout_writer.write_all(b"hello stdout\n").unwrap(); + stderr_writer.write_all(b"hello stderr\n").unwrap(); + drop(stdout_writer); + drop(stderr_writer); + stdio.finish_output(Duration::from_secs(1)).await; + + let content = std::fs::read_to_string(&log_path).unwrap(); + let records: Vec<_> = content.lines().map(parse_cri_record).collect(); + assert!(records.iter().any(|(_, stream, tag, message)| { + stream == "stdout" && *tag == "F" && message == "hello stdout" + })); + assert!(records.iter().any(|(_, stream, tag, message)| { + stream == "stderr" && *tag == "F" && message == "hello stderr" + })); + assert_eq!(drain_fd(Some(stdout)), "hello stdout\n"); + assert_eq!(drain_fd(Some(stderr)), "hello stderr\n"); + } + + #[tokio::test] + async fn test_cri_stream_frames_partial_lines_and_eof() { + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("console.log"); + let mut writer = capture_writer(&log_path, u64::MAX, 3); + let mut stream = CriStream::new("stdout"); + + stream.push(&mut writer, b"first").await; + stream.push(&mut writer, b" line\nsecond").await; + stream.finish(&mut writer).await; + writer.flush().await; + + let content = std::fs::read_to_string(log_path).unwrap(); + let records: Vec<_> = content.lines().map(parse_cri_record).collect(); + assert_eq!(records.len(), 2); + assert_eq!(records[0].1, "stdout"); + assert_eq!(records[0].2, "F"); + assert_eq!(records[0].3, "first line"); + assert_eq!(records[1].1, "stdout"); + assert_eq!(records[1].2, "F"); + assert_eq!(records[1].3, "second"); + } + + #[tokio::test] + async fn test_cri_stream_emits_empty_lines_and_lossy_utf8() { + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("console.log"); + let mut writer = capture_writer(&log_path, u64::MAX, 3); + let mut stream = CriStream::new("stdout"); + + stream.push(&mut writer, b"\ninvalid: \xff\n").await; + writer.flush().await; + + let content = std::fs::read_to_string(log_path).unwrap(); + let records: Vec<_> = content.lines().map(parse_cri_record).collect(); + assert_eq!(records.len(), 2); + assert!(records[0].3.is_empty()); + assert_eq!(records[1].3, "invalid: \u{fffd}"); + } + + #[tokio::test] + async fn test_cri_streams_do_not_splice_partial_lines_together() { + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("console.log"); + let mut writer = capture_writer(&log_path, u64::MAX, 3); + let mut stdout = CriStream::new("stdout"); + let mut stderr = CriStream::new("stderr"); + + stdout.push(&mut writer, b"stdout partial").await; + stderr.push(&mut writer, b"stderr complete\n").await; + stdout.push(&mut writer, b" completed\n").await; + writer.flush().await; + + let content = std::fs::read_to_string(log_path).unwrap(); + let records: Vec<_> = content.lines().map(parse_cri_record).collect(); + assert_eq!(records.len(), 2); + assert_eq!(records[0].1, "stderr"); + assert_eq!(records[0].3, "stderr complete"); + assert_eq!(records[1].1, "stdout"); + assert_eq!(records[1].3, "stdout partial completed"); + } + + #[tokio::test] + async fn test_cri_stream_splits_long_lines_with_partial_tags() { + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("console.log"); + let mut writer = capture_writer(&log_path, u64::MAX, 3); + let mut stream = CriStream::new("stderr"); + let line = vec![b'z'; CRI_MAX_CONTENT_SIZE * 2 + 7]; + + stream.push(&mut writer, &line).await; + stream.push(&mut writer, b"\n").await; + writer.flush().await; + + let content = std::fs::read_to_string(log_path).unwrap(); + let records: Vec<_> = content.lines().map(parse_cri_record).collect(); + assert_eq!(records.len(), 3); + assert_eq!(records[0].2, "P"); + assert_eq!(records[1].2, "P"); + assert_eq!(records[2].2, "F"); + assert_eq!( + records.iter().map(|record| record.3.len()).sum::(), + line.len() + ); + assert!(records.iter().all(|record| record.1 == "stderr")); + } + + #[tokio::test] + async fn test_capture_writer_rotates_by_size_and_prunes_oldest() { + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("console.log"); + let mut writer = capture_writer(&log_path, 5, 3); + + writer.write_cri("stdout", 'F', b"first").await; + writer.write_cri("stdout", 'F', b"second").await; + writer.write_cri("stdout", 'F', b"third").await; + writer.flush().await; + + let active = std::fs::read_to_string(&log_path).unwrap(); + let archive1 = std::fs::read_to_string(log_path.with_extension("log.1")).unwrap(); + let archive2 = std::fs::read_to_string(log_path.with_extension("log.2")).unwrap(); + assert!(active.contains("third")); + assert!(archive1.contains("second")); + assert!(archive2.contains("first")); + assert_eq!(active.lines().count(), 1); + assert_eq!(archive1.lines().count(), 1); + assert_eq!(archive2.lines().count(), 1); + let mut replacement_path = log_path.as_os_str().to_os_string(); + replacement_path.push(".tmp"); + assert!(!PathBuf::from(replacement_path).exists()); + } + + #[tokio::test] + async fn test_capture_writer_resumes_size_from_existing_file() { + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("console.log"); + std::fs::write(&log_path, b"1234").unwrap(); + let mut writer = capture_writer(&log_path, 5, 3); + + writer.write_cri("stdout", 'F', b"XY").await; + writer.flush().await; + + let active = std::fs::read_to_string(&log_path).unwrap(); + assert!(active.contains("XY")); + assert_eq!( + std::fs::read(log_path.with_extension("log.1")).unwrap(), + b"1234" + ); + } + + #[tokio::test] + async fn test_capture_writer_keeps_active_log_when_replacement_creation_fails() { + let dir = tempfile::tempdir().unwrap(); + let log_path = dir.path().join("console.log"); + std::fs::write(&log_path, b"12345").unwrap(); + let mut replacement_path = log_path.as_os_str().to_os_string(); + replacement_path.push(".tmp"); + std::fs::create_dir(PathBuf::from(replacement_path)).unwrap(); + let mut writer = capture_writer(&log_path, 5, 3); + + writer.write_cri("stderr", 'F', b"X").await; + writer.flush().await; - // Second call returns empty (fds already taken) - let (stdout2, stderr2) = stdio.drain_output(); - assert_eq!(stdout2, ""); - assert_eq!(stderr2, ""); + let active = std::fs::read_to_string(&log_path).unwrap(); + assert!(active.starts_with("12345")); + assert!(active.contains(" stderr F X\n")); + assert!(!log_path.with_extension("log.1").exists()); + assert!(!writer.failed); } } diff --git a/src/guest/src/service/container.rs b/src/guest/src/service/container.rs index c8af7db7e..236d476ed 100644 --- a/src/guest/src/service/container.rs +++ b/src/guest/src/service/container.rs @@ -3,7 +3,7 @@ //! //! Handles OCI container lifecycle (Init RPC). -use std::path::Path; +use std::path::{Path, PathBuf}; use crate::service::server::GuestServer; use boxlite_shared::{ @@ -19,6 +19,20 @@ use crate::container::{Container, UserMount}; use crate::layout::GuestLayout; use crate::storage::block_device::BlockDeviceMount; +fn log_capture_config(path: Option) -> Result, String> { + let Some(path) = path else { + return Ok(None); + }; + let path = PathBuf::from(path); + if path.as_os_str().is_empty() { + return Err("log_capture_path is required when log capture is enabled".to_string()); + } + if !path.is_absolute() { + return Err("log_capture_path must be absolute".to_string()); + } + Ok(Some(path)) +} + /// Prepare container rootfs based on the initialization strategy. /// /// Handles three strategies: @@ -249,6 +263,25 @@ impl ContainerService for GuestServer { }) .collect(); + let log_capture = match log_capture_config(init_req.log_capture_path) { + Ok(config) => config, + Err(reason) => { + error!(%reason, "Invalid log capture configuration"); + return Ok(Response::new(ContainerInitResponse { + result: Some(container_init_response::Result::Error(ContainerInitError { + reason, + })), + })); + } + }; + if config.tty && log_capture.is_some() { + return Ok(Response::new(ContainerInitResponse { + result: Some(container_init_response::Result::Error(ContainerInitError { + reason: "log capture is not supported with TTY mode".to_string(), + })), + })); + } + debug!( entrypoint = ?config.entrypoint, workdir = %config.workdir, @@ -295,6 +328,7 @@ impl ContainerService for GuestServer { &config.user, user_mounts, config.tty, + log_capture, ) { Ok(mut container) => { // Init is created, not yet running — Init never runs it; the diff --git a/src/guest/src/service/exec/state.rs b/src/guest/src/service/exec/state.rs index af93b9015..68bae7670 100644 --- a/src/guest/src/service/exec/state.rs +++ b/src/guest/src/service/exec/state.rs @@ -17,7 +17,7 @@ pub(crate) trait InitHealthCheck: Send + Sync { fn is_running(&self) -> bool; /// Diagnose why init exited. Includes status, PID, init stdout/stderr. - /// May only return full output once (drains init pipes). + /// May only return the buffered output once (takes the diagnostic tails). fn diagnose_exit(&mut self) -> String; } diff --git a/src/guest/src/service/guest.rs b/src/guest/src/service/guest.rs index 72bb6739a..2eb5850c6 100644 --- a/src/guest/src/service/guest.rs +++ b/src/guest/src/service/guest.rs @@ -109,8 +109,8 @@ impl GuestService for GuestServer { let containers = self.containers.lock().await; for (container_id, container_arc) in containers.iter() { info!(container_id = %container_id, "Shutting down container"); - let container = container_arc.lock().await; - if let Err(e) = container.shutdown(CONTAINER_SHUTDOWN_TIMEOUT_MS) { + let mut container = container_arc.lock().await; + if let Err(e) = container.shutdown(CONTAINER_SHUTDOWN_TIMEOUT_MS).await { error!(container_id = %container_id, error = %e, "Failed to shutdown container"); } } diff --git a/src/shared/proto/boxlite/v1/service.proto b/src/shared/proto/boxlite/v1/service.proto index 4069a9483..d3859ed64 100644 --- a/src/shared/proto/boxlite/v1/service.proto +++ b/src/shared/proto/boxlite/v1/service.proto @@ -227,6 +227,9 @@ message ContainerInitRequest { // guest registers what it is handed, rather than both sides separately // hard-coding the id==container_id rule on their own side of the wire. string execution_id = 6; + // Optional guest-visible path for container init stdout/stderr capture. + // Constructed by the trusted host runtime under /run/boxlite/logs. + optional string log_capture_path = 7; } message ContainerStartRequest { diff --git a/src/shared/src/constants.rs b/src/shared/src/constants.rs index 435d407cb..6af501238 100644 --- a/src/shared/src/constants.rs +++ b/src/shared/src/constants.rs @@ -53,4 +53,13 @@ pub mod mount_tags { /// Tag for shared container directory (contains overlayfs/ and rootfs/) pub const SHARED: &str = "BoxLiteShared"; + + /// Tag for container init stdout/stderr log capture share. + pub const CONTAINER_LOGS: &str = "BoxLiteContainerLogs"; +} + +/// Container log capture constants. +pub mod logs { + /// Guest mount point for the dedicated container logs virtiofs share. + pub const GUEST_DIR: &str = "/run/boxlite/logs"; }