patchpkg: patch libstdc++ (#2271)

This fixes an issue with TensorFlow not being able to find libstdc++.so
at runtime.

Author: gcurtis

internal/boxcli/patch.go

@@ -17,6 +17,7 @@ func patchCmd() *cobra.Command {
 		},
 	}
 	cmd.Flags().StringVar(&builder.Glibc, "glibc", "", "patch binaries to use a different glibc")
+	cmd.Flags().StringVar(&builder.Gcc, "gcc", "", "patch binaries to use a different gcc")
 	cmd.Flags().BoolVar(&builder.RestoreRefs, "restore-refs", false, "restore references to removed store paths")
 	return cmd
 }

internal/patchpkg/builder.go

@@ -31,8 +31,14 @@ type DerivationBuilder struct {
 	// Glibc is an optional store path to an alternative glibc version. If
 	// it's set, the builder will patch ELF binaries to use its shared
 	// libraries and dynamic linker.
-	Glibc        string
-	glibcPatcher *glibcPatcher
+	Glibc string
+
+	// Gcc is an optional store path to an alternative gcc version. If
+	// it's set, the builder will patch ELF binaries to use its shared
+	// libraries (such as libstdc++.so).
+	Gcc string
+
+	glibcPatcher *libPatcher
 
 	RestoreRefs bool
 	bytePatches map[string][]fileSlice
@@ -56,12 +62,23 @@ func (d *DerivationBuilder) init() error {
 		}
 	}
 	if d.Glibc != "" {
-		var err error
-		d.glibcPatcher, err = newGlibcPatcher(newPackageFS(d.Glibc))
+		if d.glibcPatcher == nil {
+			d.glibcPatcher = &libPatcher{}
+		}
+		err := d.glibcPatcher.setGlibc(newPackageFS(d.Glibc))
 		if err != nil {
 			return fmt.Errorf("patchpkg: can't patch glibc using %s: %v", d.Glibc, err)
 		}
 	}
+	if d.Gcc != "" {
+		if d.glibcPatcher == nil {
+			d.glibcPatcher = &libPatcher{}
+		}
+		err := d.glibcPatcher.setGcc(newPackageFS(d.Gcc))
+		if err != nil {
+			return fmt.Errorf("patchpkg: can't patch gcc using %s: %v", d.Gcc, err)
+		}
+	}
 	return nil
 }
 

internal/patchpkg/patch.go

@@ -12,56 +12,88 @@ import (
 	"strings"
 )
 
-// glibcPatcher patches ELF binaries to use an alternative version of glibc.
-type glibcPatcher struct {
+// libPatcher patches ELF binaries to use an alternative version of glibc.
+type libPatcher struct {
 	// ld is the absolute path to the new dynamic linker (ld.so).
 	ld string
 
 	// rpath is the new RPATH with the directories containing the new libc
 	// shared objects (libc.so) and other libraries.
 	rpath []string
-}
 
-// newGlibcPatcher creates a new glibcPatcher and verifies that it can find the
-// shared object files in glibc.
-func newGlibcPatcher(glibc *packageFS) (*glibcPatcher, error) {
-	patcher := &glibcPatcher{}
+	// needed are shared libraries to add as dependencies (DT_NEEDED).
+	needed []string
+}
 
+// setGlibc configures the patcher to use the dynamic linker and libc libraries
+// in pkg.
+func (p *libPatcher) setGlibc(pkg *packageFS) error {
 	// Verify that we can find a directory with libc in it.
 	glob := "lib*/libc.so*"
-	matches, _ := fs.Glob(glibc, glob)
+	matches, _ := fs.Glob(pkg, glob)
 	if len(matches) == 0 {
-		return nil, fmt.Errorf("cannot find libc.so file matching %q", glob)
+		return fmt.Errorf("cannot find libc.so file matching %q", glob)
 	}
 	for i := range matches {
 		matches[i] = path.Dir(matches[i])
 	}
-	slices.Sort(matches) // pick the shortest name: lib < lib32 < lib64 < libx32
+	// Pick the shortest name: lib < lib32 < lib64 < libx32
+	//
+	// - lib is usually a symlink to the correct arch (e.g., lib -> lib64)
+	// - *.so is usually a symlink to the correct version (e.g., foo.so -> foo.so.2)
+	slices.Sort(matches)
 
-	lib, err := glibc.OSPath(matches[0])
+	lib, err := pkg.OSPath(matches[0])
 	if err != nil {
-		return nil, err
+		return err
 	}
-	patcher.rpath = append(patcher.rpath, lib)
+	p.rpath = append(p.rpath, lib)
 	slog.Debug("found new libc directory", "path", lib)
 
 	// Verify that we can find the new dynamic linker.
 	glob = "lib*/ld-linux*.so*"
-	matches, _ = fs.Glob(glibc, glob)
+	matches, _ = fs.Glob(pkg, glob)
 	if len(matches) == 0 {
-		return nil, fmt.Errorf("cannot find ld.so file matching %q", glob)
+		return fmt.Errorf("cannot find ld.so file matching %q", glob)
 	}
 	slices.Sort(matches)
-	patcher.ld, err = glibc.OSPath(matches[0])
+	p.ld, err = pkg.OSPath(matches[0])
 	if err != nil {
-		return nil, err
+		return err
+	}
+	slog.Debug("found new dynamic linker", "path", p.ld)
+	return nil
+}
+
+// setGlibc configures the patcher to use the standard C++ and gcc libraries in
+// pkg.
+func (p *libPatcher) setGcc(pkg *packageFS) error {
+	// Verify that we can find a directory with libstdc++.so in it.
+	glob := "lib*/libstdc++.so*"
+	matches, _ := fs.Glob(pkg, glob)
+	if len(matches) == 0 {
+		return fmt.Errorf("cannot find libstdc++.so file matching %q", glob)
 	}
-	slog.Debug("found new dynamic linker", "path", patcher.ld)
+	for i := range matches {
+		matches[i] = path.Dir(matches[i])
+	}
+	// Pick the shortest name: lib < lib32 < lib64 < libx32
+	//
+	// - lib is usually a symlink to the correct arch (e.g., lib -> lib64)
+	// - *.so is usually a symlink to the correct version (e.g., foo.so -> foo.so.2)
+	slices.Sort(matches)
 
-	return patcher, nil
+	lib, err := pkg.OSPath(matches[0])
+	if err != nil {
+		return err
+	}
+	p.rpath = append(p.rpath, lib)
+	p.needed = append(p.needed, "libstdc++.so")
+	slog.Debug("found new libstdc++ directory", "path", lib)
+	return nil
 }
 
-func (g *glibcPatcher) prependRPATH(libPkg *packageFS) {
+func (p *libPatcher) prependRPATH(libPkg *packageFS) {
 	glob := "lib*/*.so*"
 	matches, _ := fs.Glob(libPkg, glob)
 	if len(matches) == 0 {
@@ -80,13 +112,13 @@ func (g *glibcPatcher) prependRPATH(libPkg *packageFS) {
 			continue
 		}
 	}
-	g.rpath = append(matches, g.rpath...)
+	p.rpath = append(p.rpath, matches...)
 	slog.Debug("prepended package lib dirs to RPATH", "pkg", libPkg.storePath, "dirs", matches)
 }
 
 // patch applies glibc patches to a binary and writes the patched result to
 // outPath. It does not modify the original binary in-place.
-func (g *glibcPatcher) patch(ctx context.Context, path, outPath string) error {
+func (p *libPatcher) patch(ctx context.Context, path, outPath string) error {
 	cmd := &patchelf{PrintInterpreter: true}
 	out, err := cmd.run(ctx, path)
 	if err != nil {
@@ -102,8 +134,9 @@ func (g *glibcPatcher) patch(ctx context.Context, path, outPath string) error {
 	oldRpath := strings.Split(string(out), ":")
 
 	cmd = &patchelf{
-		SetInterpreter: g.ld,
-		SetRPATH:       append(g.rpath, oldRpath...),
+		SetInterpreter: p.ld,
+		SetRPATH:       append(p.rpath, oldRpath...),
+		AddNeeded:      p.needed,
 		Output:         outPath,
 	}
 	slog.Debug("patching glibc on binary",
@@ -123,6 +156,8 @@ type patchelf struct {
 	SetInterpreter   string
 	PrintInterpreter bool
 
+	AddNeeded []string
+
 	Output string
 }
 
@@ -141,6 +176,9 @@ func (p *patchelf) run(ctx context.Context, elf string) ([]byte, error) {
 	if p.PrintInterpreter {
 		cmd.Args = append(cmd.Args, "--print-interpreter")
 	}
+	for _, needed := range p.AddNeeded {
+		cmd.Args = append(cmd.Args, "--add-needed", needed)
+	}
 	if p.Output != "" {
 		cmd.Args = append(cmd.Args, "--output", p.Output)
 	}

internal/shellgen/tmpl/glibc-patch.nix.tmpl

@@ -77,6 +77,7 @@
 
         isLinux = (builtins.match ".*linux.*" system) != null;
         glibc = if isLinux then nixpkgs-glibc.legacyPackages."${system}".glibc else null;
+        gcc = if isLinux then nixpkgs-glibc.legacyPackages."${system}".stdenv.cc.cc.lib else null;
 
         # Create a package that puts the local devbox binary in the conventional
         # bin subdirectory. This also ensures that the executable is named
@@ -97,6 +98,7 @@
         builder = "${devbox}/bin/devbox";
         args = [ "patch" "--restore-refs" ] ++
           (if glibc != null then [ "--glibc" "${glibc}" ] else [ ]) ++
+          (if gcc != null then [ "--gcc" "${gcc}" ] else [ ]) ++
           [ pkg ];
       };
     in