diff --git a/lib/builds/builder_agent/main.go b/lib/builds/builder_agent/main.go index a6bca84d..0730b865 100644 --- a/lib/builds/builder_agent/main.go +++ b/lib/builds/builder_agent/main.go @@ -523,7 +523,6 @@ func runBuildProcess() { // Run the build log.Println("=== Starting Build ===") digest, _, err := runBuild(ctx, config, logWriter) - // Note: buildLogs is already written to logWriter via io.MultiWriter in runBuild duration := time.Since(start).Milliseconds() @@ -763,10 +762,10 @@ func runBuild(ctx context.Context, config *BuildConfig, logWriter io.Writer) (st // Build arguments var outputOpts string if useInsecureFlag { - outputOpts = fmt.Sprintf("type=image,name=%s,push=true,registry.insecure=true,oci-mediatypes=true", outputRef) + outputOpts = fmt.Sprintf("type=image,name=%s,push=true,registry.insecure=true,oci-mediatypes=true,compression=zstd,force-compression=true", outputRef) log.Printf("Using HTTP registry (insecure mode): %s", registryHost) } else { - outputOpts = fmt.Sprintf("type=image,name=%s,push=true,oci-mediatypes=true", outputRef) + outputOpts = fmt.Sprintf("type=image,name=%s,push=true,oci-mediatypes=true,compression=zstd,force-compression=true", outputRef) log.Printf("Using HTTPS registry (secure mode): %s", registryHost) } @@ -849,19 +848,38 @@ func runBuild(ctx context.Context, config *BuildConfig, logWriter io.Writer) (st buildkitdConfig := "/home/builder/.config/buildkit/buildkitd.toml" log.Printf("Using buildkitd config: %s", buildkitdConfig) + // Mount a tmpfs for BuildKit's data directory. + // The VM rootfs is an overlayfs (read-only ext4 + writable ext4 upper layer). + // BuildKit's native overlayfs snapshotter creates char device 0:0 for whiteout + // markers, but mknod(char 0:0) fails on an overlayfs mount because the kernel + // treats it as an overlayfs whiteout rather than a regular device node. + // Using tmpfs avoids this nested-overlayfs conflict. + buildkitRoot := "/var/lib/buildkit" + if err := os.MkdirAll(buildkitRoot, 0755); err != nil { + return "", "", fmt.Errorf("create buildkit root dir: %w", err) + } + mountCmd := exec.Command("mount", "-t", "tmpfs", "-o", "size=3G", "tmpfs", buildkitRoot) + if output, err := mountCmd.CombinedOutput(); err != nil { + return "", "", fmt.Errorf("mount tmpfs at %s (required for native overlayfs snapshotter): %v: %s", buildkitRoot, err, output) + } + log.Printf("Mounted tmpfs at %s for BuildKit snapshotter", buildkitRoot) + log.Printf("Running: buildctl-daemonless.sh %s", strings.Join(args, " ")) // Run buildctl-daemonless.sh + // buildctl writes progress (#1, #2, etc.) to stderr and a duplicate summary to stdout. + // Only pipe stderr to logWriter to avoid doubled output in build logs. cmd := exec.CommandContext(ctx, "buildctl-daemonless.sh", args...) - cmd.Stdout = io.MultiWriter(logWriter, &buildLogs) + cmd.Stdout = &buildLogs cmd.Stderr = io.MultiWriter(logWriter, &buildLogs) // Set environment: // - HOME and DOCKER_CONFIG: ensures buildctl finds the auth config at /root/.docker/config.json // - BUILDKITD_FLAGS: tells buildkitd to use our custom config for registry TLS settings + // and to use native overlayfs snapshotter with a tmpfs-backed root directory // Filter out existing values to avoid duplicates (first value wins in shell) env := make([]string, 0, len(os.Environ())+3) for _, e := range os.Environ() { - if !strings.HasPrefix(e, "DOCKER_CONFIG=") && + if !strings.HasPrefix(e, "DOCKER_CONFIG=") && !strings.HasPrefix(e, "BUILDKITD_FLAGS=") && !strings.HasPrefix(e, "HOME=") { env = append(env, e) @@ -869,7 +887,7 @@ func runBuild(ctx context.Context, config *BuildConfig, logWriter io.Writer) (st } env = append(env, "HOME=/root") env = append(env, "DOCKER_CONFIG=/root/.docker") - env = append(env, fmt.Sprintf("BUILDKITD_FLAGS=--config=%s", buildkitdConfig)) + env = append(env, fmt.Sprintf("BUILDKITD_FLAGS=--config=%s --oci-worker-snapshotter=overlayfs --root=%s", buildkitdConfig, buildkitRoot)) cmd.Env = env if err := cmd.Run(); err != nil { diff --git a/lib/builds/images/generic/Dockerfile b/lib/builds/images/generic/Dockerfile index 5420641b..5f485f33 100644 --- a/lib/builds/images/generic/Dockerfile +++ b/lib/builds/images/generic/Dockerfile @@ -1,5 +1,5 @@ # Generic Builder Image -# Contains rootless BuildKit + builder agent + guest-agent for debugging +# Contains BuildKit + builder agent + guest-agent for debugging # Builds any Dockerfile provided by the user # Use non-rootless buildkit to avoid potential credential handling issues @@ -33,8 +33,7 @@ FROM alpine:3.21 RUN apk add --no-cache \ ca-certificates \ git \ - curl \ - fuse-overlayfs + curl # Copy BuildKit binaries from official image COPY --from=buildkit /usr/bin/buildctl /usr/bin/buildctl @@ -46,17 +45,17 @@ COPY --from=buildkit /usr/bin/buildkit-runc /usr/bin/runc COPY --from=agent-builder /builder-agent /usr/bin/builder-agent COPY --from=agent-builder /guest-agent /usr/bin/guest-agent -# Create unprivileged user for rootless BuildKit +# Create builder user and directories (kept for config path compatibility) +# Running as root inside ephemeral microVM - the VM is the security boundary RUN adduser -D -u 1000 builder && \ mkdir -p /home/builder/.local/share/buildkit /config /run/secrets /src && \ chown -R builder:builder /home/builder /config /run/secrets /src -# Switch to unprivileged user -USER builder WORKDIR /src # Set environment for buildkit in microVM -ENV BUILDKITD_FLAGS="" +# Use native overlayfs snapshotter (faster than fuse-overlayfs, requires root) +ENV BUILDKITD_FLAGS="--oci-worker-snapshotter=overlayfs" ENV HOME=/home/builder ENV XDG_RUNTIME_DIR=/home/builder/.local/share diff --git a/lib/builds/manager.go b/lib/builds/manager.go index a23b77e4..6da77ff1 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -418,6 +418,14 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc {Repo: fmt.Sprintf("builds/%s", id), Scope: "push"}, } + // If the Dockerfile uses base images from the internal registry, grant pull access + for _, baseRepo := range extractInternalBaseImageRepos(req.Dockerfile, m.config.RegistryURL) { + repoAccess = append(repoAccess, RepoPermission{ + Repo: baseRepo, + Scope: "pull", + }) + } + if req.IsAdminBuild { // Admin build: push access to global cache if req.GlobalCacheKey != "" { @@ -1264,6 +1272,14 @@ func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) err {Repo: fmt.Sprintf("builds/%s", buildID), Scope: "push"}, } + // If the Dockerfile uses base images from the internal registry, grant pull access + for _, baseRepo := range extractInternalBaseImageRepos(req.Dockerfile, m.config.RegistryURL) { + repoAccess = append(repoAccess, RepoPermission{ + Repo: baseRepo, + Scope: "pull", + }) + } + if req.IsAdminBuild { // Admin build: push access to global cache if req.GlobalCacheKey != "" { @@ -1362,6 +1378,65 @@ func (m *manager) createBuildConfigVolume(buildID, volID string) (string, error) return diskPath, nil } +// extractInternalBaseImageRepos parses the Dockerfile's FROM lines and returns +// all repository paths that reference the internal registry. Returns nil +// if no base images reference the internal registry. +func extractInternalBaseImageRepos(dockerfile, registryURL string) []string { + if dockerfile == "" { + return nil + } + + registryHost := stripRegistryScheme(registryURL) + seen := make(map[string]bool) + var repos []string + + scanner := bufio.NewScanner(strings.NewReader(dockerfile)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + upper := strings.ToUpper(line) + if !strings.HasPrefix(upper, "FROM ") { + continue + } + + // Parse: FROM [--platform=...] image[:tag|@digest] [AS name] + parts := strings.Fields(line) + imageIdx := 1 + for imageIdx < len(parts) && strings.HasPrefix(parts[imageIdx], "--") { + imageIdx++ + } + if imageIdx >= len(parts) { + continue + } + + imageRef := parts[imageIdx] + if strings.ToLower(imageRef) == "scratch" { + continue + } + + // Check if the image references the internal registry + if !strings.HasPrefix(imageRef, registryHost+"/") { + continue + } + + // Strip the registry host to get the repo path, then strip digest and tag. + // An image ref can have both: registry/org/img:v1@sha256:abc123 + repo := strings.TrimPrefix(imageRef, registryHost+"/") + if idx := strings.LastIndex(repo, "@"); idx != -1 { + repo = repo[:idx] + } + if idx := strings.LastIndex(repo, ":"); idx != -1 { + repo = repo[:idx] + } + + if !seen[repo] { + seen[repo] = true + repos = append(repos, repo) + } + } + + return repos +} + // copyFile copies a file from src to dst func copyFile(src, dst string) error { // Ensure parent directory exists diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index fdbf60a1..74e148e4 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -981,3 +981,78 @@ eventLoop: } } } + +func TestExtractInternalBaseImageRepos(t *testing.T) { + registryURL := "http://10.102.0.1:8085" + + tests := []struct { + name string + dockerfile string + want []string + }{ + { + name: "empty dockerfile", + dockerfile: "", + want: nil, + }, + { + name: "external base image only", + dockerfile: "FROM alpine:latest\nRUN echo hello", + want: nil, + }, + { + name: "internal base image with tag", + dockerfile: "FROM 10.102.0.1:8085/onkernel/nodejs22-base:0.1.1\nRUN echo hello", + want: []string{"onkernel/nodejs22-base"}, + }, + { + name: "internal base image with digest", + dockerfile: "FROM 10.102.0.1:8085/onkernel/nodejs22-base@sha256:abcdef1234567890\nRUN echo hello", + want: []string{"onkernel/nodejs22-base"}, + }, + { + name: "internal base image with tag AND digest", + dockerfile: "FROM 10.102.0.1:8085/onkernel/nodejs22-base:v1@sha256:abcdef1234567890\nRUN echo hello", + want: []string{"onkernel/nodejs22-base"}, + }, + { + name: "multi-stage with multiple internal images", + dockerfile: `FROM 10.102.0.1:8085/onkernel/builder:latest AS builder +RUN make build +FROM 10.102.0.1:8085/onkernel/runtime:v2 +COPY --from=builder /app /app`, + want: []string{"onkernel/builder", "onkernel/runtime"}, + }, + { + name: "mix of internal and external", + dockerfile: `FROM alpine:latest AS deps +RUN apk add curl +FROM 10.102.0.1:8085/onkernel/base:latest +COPY --from=deps /usr/bin/curl /usr/bin/curl`, + want: []string{"onkernel/base"}, + }, + { + name: "deduplicates same repo", + dockerfile: `FROM 10.102.0.1:8085/onkernel/base:v1 AS stage1 +FROM 10.102.0.1:8085/onkernel/base:v2 AS stage2`, + want: []string{"onkernel/base"}, + }, + { + name: "with --platform flag", + dockerfile: "FROM --platform=linux/amd64 10.102.0.1:8085/onkernel/base:latest\nRUN echo hello", + want: []string{"onkernel/base"}, + }, + { + name: "scratch is ignored", + dockerfile: "FROM scratch\nCOPY binary /", + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractInternalBaseImageRepos(tt.dockerfile, registryURL) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/lib/builds/types.go b/lib/builds/types.go index 33d9ace0..247154b9 100644 --- a/lib/builds/types.go +++ b/lib/builds/types.go @@ -208,8 +208,8 @@ type BuildResult struct { func DefaultBuildPolicy() BuildPolicy { return BuildPolicy{ TimeoutSeconds: 600, // 10 minutes - MemoryMB: 2048, // 2GB - CPUs: 2, + MemoryMB: 4096, // 4GB + CPUs: 4, NetworkMode: "egress", // Allow outbound for dependency downloads } } diff --git a/lib/system/versions.go b/lib/system/versions.go index 5db0013d..b10bcbfb 100644 --- a/lib/system/versions.go +++ b/lib/system/versions.go @@ -6,22 +6,30 @@ import "runtime" type KernelVersion string const ( - // Kernel_202601152 is the current kernel version with vGPU support + // Kernel_202601152 is the previous kernel version with vGPU support Kernel_202601152 KernelVersion = "ch-6.12.8-kernel-1.3-202601152" + + // Kernel_202602101 is the current kernel version with overlayfs redirect_dir and index support + Kernel_202602101 KernelVersion = "ch-6.12.8-kernel-1.4-202602101" ) var ( // DefaultKernelVersion is the kernel version used for new instances - DefaultKernelVersion = Kernel_202601152 + DefaultKernelVersion = Kernel_202602101 // SupportedKernelVersions lists all supported kernel versions SupportedKernelVersions = []KernelVersion{ + Kernel_202602101, Kernel_202601152, } ) // KernelDownloadURLs maps kernel versions and architectures to download URLs var KernelDownloadURLs = map[KernelVersion]map[string]string{ + Kernel_202602101: { + "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.4-202602101/vmlinux-x86_64", + "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.4-202602101/Image-arm64", + }, Kernel_202601152: { "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.3-202601152/vmlinux-x86_64", "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.3-202601152/Image-arm64", @@ -31,6 +39,10 @@ var KernelDownloadURLs = map[KernelVersion]map[string]string{ // KernelHeaderURLs maps kernel versions and architectures to kernel header tarball URLs // These tarballs contain kernel headers needed for DKMS to build out-of-tree modules (e.g., NVIDIA vGPU drivers) var KernelHeaderURLs = map[KernelVersion]map[string]string{ + Kernel_202602101: { + "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.4-202602101/kernel-headers-x86_64.tar.gz", + "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.4-202602101/kernel-headers-aarch64.tar.gz", + }, Kernel_202601152: { "x86_64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.3-202601152/kernel-headers-x86_64.tar.gz", "aarch64": "https://github.com/kernel/linux/releases/download/ch-6.12.8-kernel-1.3-202601152/kernel-headers-aarch64.tar.gz",