Allow locally installed libraries in sketch profiles. (#2930)

* Allow directories in profile libraries

* Updated docs

* Dump custom libraries in profiles with --dump-profile command

* Added integration tests

* Fix integration test in Windows

* Improved integration tests

* Fixed flaky intergration test.

Author: cmaglie

commands/instances.go

@@ -363,6 +363,24 @@ func (s *arduinoCoreServerImpl) Init(req *rpc.InitRequest, stream rpc.ArduinoCor
 	} else {
 		// Load libraries required for profile
 		for _, libraryRef := range profile.Libraries {
+			if libraryRef.InstallDir != nil {
+				libDir := libraryRef.InstallDir
+				if !libDir.IsAbs() {
+					libDir = paths.New(req.GetSketchPath()).JoinPath(libraryRef.InstallDir)
+				}
+				if !libDir.IsDir() {
+					return &cmderrors.InvalidArgumentError{
+						Message: i18n.Tr("Invalid library directory in sketch project: %s", libraryRef.InstallDir),
+					}
+				}
+				lmb.AddLibrariesDir(librariesmanager.LibrariesDir{
+					Path:            libDir,
+					Location:        libraries.Unmanaged,
+					IsSingleLibrary: true,
+				})
+				continue
+			}
+
 			uid := libraryRef.InternalUniqueIdentifier()
 			libRoot := s.settings.ProfilesCacheDir().Join(uid)
 			libDir := libRoot.Join(libraryRef.Library)

docs/sketch-project-file.md

@@ -14,7 +14,9 @@ Each profile will define:
 - The target core platform name and version (with the 3rd party platform index URL if needed)
 - A possible core platform name and version, that is a dependency of the target core platform (with the 3rd party
   platform index URL if needed)
-- The libraries used in the sketch (including their version)
+- A list of libraries used in the sketch. Each library could be:
+  - a library taken from the Arduino Libraries Index
+  - a library installed anywhere in the filesystem
 - The port and protocol to upload the sketch and monitor the board
 
 The format of the file is the following:
@@ -31,9 +33,8 @@ profiles:
       - platform: <PLATFORM_DEPENDENCY> (<PLATFORM_DEPENDENCY_VERSION>)
         platform_index_url: <3RD_PARTY_PLATFORM_DEPENDENCY_URL>
     libraries:
-      - <LIB_NAME> (<LIB_VERSION>)
-      - <LIB_NAME> (<LIB_VERSION>)
-      - <LIB_NAME> (<LIB_VERSION>)
+      - <INDEX_LIB_NAME> (<INDEX_LIB_VERSION>)
+      - dir: <LOCAL_LIB_PATH>
     port: <PORT_NAME>
     port_config:
       <PORT_SETTING_NAME>: <PORT_SETTING_VALUE>
@@ -55,7 +56,11 @@ otherwise below). The available fields are:
   information as `<PLATFORM>`, `<PLATFORM_VERSION>`, and `<3RD_PARTY_PLATFORM_URL>` respectively but for the core
   platform dependency of the main core platform. These fields are optional.
 - `libraries:` is a section where the required libraries to build the project are defined. This section is optional.
-- `<LIB_VERSION>` is the version required for the library, for example, `1.0.0`.
+  - `<INDEX_LIB_NAME> (<INDEX_LIB_VERSION>)` represents a library from the Arduino Libraries Index, for example,
+    `MyLib (1.0.0)`.
+  - `dir: <LOCAL_LIB_PATH>` represents a library installed in the filesystem and `<LOCAL_LIB_PATH>` is the path to the
+    library. The path could be absolute or relative to the sketch folder. This option is available since Arduino CLI
+    1.3.0.
 - `<USER_NOTES>` is a free text string available to the developer to add comments. This field is optional.
 - `<PROGRAMMER>` is the programmer that will be used. This field is optional.
 

internal/arduino/sketch/profiles.go

@@ -28,6 +28,7 @@ import (
 	"github.com/arduino/arduino-cli/internal/i18n"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"github.com/arduino/go-paths-helper"
+	"go.bug.st/f"
 	semver "go.bug.st/relaxed-semver"
 	"gopkg.in/yaml.v3"
 )
@@ -271,12 +272,26 @@ func (p *ProfilePlatformReference) UnmarshalYAML(unmarshal func(interface{}) err
 
 // ProfileLibraryReference is a reference to a library
 type ProfileLibraryReference struct {
-	Library string
-	Version *semver.Version
+	Library    string
+	InstallDir *paths.Path
+	Version    *semver.Version
 }
 
 // UnmarshalYAML decodes a ProfileLibraryReference from YAML source.
 func (l *ProfileLibraryReference) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var dataMap map[string]any
+	if err := unmarshal(&dataMap); err == nil {
+		if installDir, ok := dataMap["dir"]; !ok {
+			return errors.New(i18n.Tr("invalid library reference: %s", dataMap))
+		} else if installDir, ok := installDir.(string); !ok {
+			return fmt.Errorf("%s: %s", i18n.Tr("invalid library reference: %s"), dataMap)
+		} else {
+			l.InstallDir = paths.New(installDir)
+			l.Library = l.InstallDir.Base()
+			return nil
+		}
+	}
+
 	var data string
 	if err := unmarshal(&data); err != nil {
 		return err
@@ -294,16 +309,23 @@ func (l *ProfileLibraryReference) UnmarshalYAML(unmarshal func(interface{}) erro
 
 // AsYaml outputs the required library as Yaml
 func (l *ProfileLibraryReference) AsYaml() string {
-	res := fmt.Sprintf("      - %s (%s)\n", l.Library, l.Version)
-	return res
+	if l.InstallDir != nil {
+		return fmt.Sprintf("      - dir: %s\n", l.InstallDir)
+	}
+	return fmt.Sprintf("      - %s (%s)\n", l.Library, l.Version)
 }
 
 func (l *ProfileLibraryReference) String() string {
+	if l.InstallDir != nil {
+		return fmt.Sprintf("%s@dir:%s", l.Library, l.InstallDir)
+	}
 	return fmt.Sprintf("%s@%s", l.Library, l.Version)
 }
 
 // InternalUniqueIdentifier returns the unique identifier for this object
 func (l *ProfileLibraryReference) InternalUniqueIdentifier() string {
+	f.Assert(l.InstallDir == nil,
+		"InternalUniqueIdentifier should not be called for library references with an install directory")
 	id := l.String()
 	h := sha256.Sum256([]byte(id))
 	res := fmt.Sprintf("%s_%s", id, hex.EncodeToString(h[:])[:16])

internal/cli/compile/compile.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/arduino/arduino-cli/commands"
@@ -323,22 +324,23 @@ func runCompileCommand(cmd *cobra.Command, args []string, srv rpc.ArduinoCoreSer
 			// Output profile
 
 			libs := ""
-			hasVendoredLibs := false
 			for _, lib := range builderRes.GetUsedLibraries() {
 				if lib.GetLocation() != rpc.LibraryLocation_LIBRARY_LOCATION_USER && lib.GetLocation() != rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED {
 					continue
 				}
-				if lib.GetVersion() == "" {
-					hasVendoredLibs = true
-					continue
+				if lib.GetVersion() == "" || lib.Location == rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED {
+					libDir := paths.New(lib.GetInstallDir())
+					// If the library is installed in the sketch path, we want to output the relative path
+					// to the sketch path, so that the sketch is portable.
+					if ok, err := libDir.IsInsideDir(sketchPath); err == nil && ok {
+						if ref, err := libDir.RelFrom(sketchPath); err == nil {
+							libDir = paths.New(filepath.ToSlash(ref.String()))
+						}
+					}
+					libs += fmt.Sprintln("      - dir: " + libDir.String())
+				} else {
+					libs += fmt.Sprintln("      - " + lib.GetName() + " (" + lib.GetVersion() + ")")
 				}
-				libs += fmt.Sprintln("      - " + lib.GetName() + " (" + lib.GetVersion() + ")")
-			}
-			if hasVendoredLibs {
-				msg := "\n"
-				msg += i18n.Tr("WARNING: The sketch is compiled using one or more custom libraries.") + "\n"
-				msg += i18n.Tr("Currently, Build Profiles only support libraries available through Arduino Library Manager.")
-				feedback.Warning(msg)
 			}
 
 			newProfileName := "my_profile_name"

internal/integrationtest/sketch/profiles_test.go

@@ -0,0 +1,99 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2022-2025 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package sketch_test
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/arduino/arduino-cli/internal/integrationtest"
+	"github.com/arduino/go-paths-helper"
+	"github.com/stretchr/testify/require"
+	"go.bug.st/testifyjson/requirejson"
+)
+
+func TestSketchProfileDump(t *testing.T) {
+	env, cli := integrationtest.CreateArduinoCLIWithEnvironment(t)
+	t.Cleanup(env.CleanUp)
+
+	// Prepare the sketch with libraries
+	tmpDir, err := paths.MkTempDir("", "")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = tmpDir.RemoveAll })
+
+	sketchTemplate, err := paths.New("testdata", "SketchWithLibrary").Abs()
+	require.NoError(t, err)
+
+	sketch := tmpDir.Join("SketchWithLibrary")
+	libInside := sketch.Join("libraries", "MyLib")
+	err = sketchTemplate.CopyDirTo(sketch)
+	require.NoError(t, err)
+
+	libOutsideTemplate := sketchTemplate.Join("..", "MyLibOutside")
+	libOutside := sketch.Join("..", "MyLibOutside")
+	err = libOutsideTemplate.CopyDirTo(libOutside)
+	require.NoError(t, err)
+
+	// Install the required core and libraries
+	_, _, err = cli.Run("core", "install", "arduino:[email protected]")
+	require.NoError(t, err)
+	_, _, err = cli.Run("lib", "install", "Adafruit [email protected]", "--no-overwrite")
+	require.NoError(t, err)
+	_, _, err = cli.Run("lib", "install", "Adafruit GFX [email protected]", "--no-overwrite")
+	require.NoError(t, err)
+	_, _, err = cli.Run("lib", "install", "Adafruit [email protected]", "--no-overwrite")
+	require.NoError(t, err)
+
+	// Check if the profile dump:
+	// - keeps libraries in the sketch with a relative path
+	// - keeps libraries outside the sketch with an absolute path
+	// - keeps libraries installed in the system with just the name and version
+	out, _, err := cli.Run("compile", "-b", "arduino:avr:uno",
+		"--library", libInside.String(),
+		"--library", libOutside.String(),
+		"--dump-profile",
+		sketch.String())
+	require.NoError(t, err)
+	require.Equal(t, strings.TrimSpace(`
+profiles:
+  uno:
+    fqbn: arduino:avr:uno
+    platforms:
+      - platform: arduino:avr (1.8.6)
+    libraries:
+      - dir: libraries/MyLib
+      - dir: `+libOutside.String()+`
+      - Adafruit SSD1306 (2.5.14)
+      - Adafruit GFX Library (1.12.1)
+      - Adafruit BusIO (1.17.1)
+`), strings.TrimSpace(string(out)))
+
+	// Dump the profile in the sketch directory and compile with it again
+	err = sketch.Join("sketch.yaml").WriteFile(out)
+	require.NoError(t, err)
+	out, _, err = cli.Run("compile", "-m", "uno", "--json", sketch.String())
+	require.NoError(t, err)
+	// Check if local libraries are picked up correctly
+	libInsideJson, _ := json.Marshal(libInside.String())
+	libOutsideJson, _ := json.Marshal(libOutside.String())
+	j := requirejson.Parse(t, out).Query(".builder_result.used_libraries")
+	j.MustContain(`
+		[
+			{"name": "MyLib", "install_dir": ` + string(libInsideJson) + `},
+			{"name": "MyLibOutside", "install_dir": ` + string(libOutsideJson) + `}
+		]`)
+}

internal/integrationtest/sketch/testdata/MyLibOutside/MyLibOutside.h

@@ -0,0 +1,10 @@
+name=MyLibOutside
+version=1.3.7
+author=Arduino
+maintainer=Arduino <[email protected]>
+sentence=
+paragraph=
+category=Communication
+url=
+architectures=*
+includes=MyLibOutside.h

internal/integrationtest/sketch/testdata/MyLibOutside/library.properties

@@ -0,0 +1,6 @@
+#include <MyLib.h>
+#include <MyLibOutside.h>
+#include <Adafruit_SSD1306.h>
+
+void setup() {}
+void loop() {}

internal/integrationtest/sketch/testdata/SketchWithLibrary/SketchWithLibrary.ino

@@ -0,0 +1,10 @@
+name=MyLib
+version=1.3.7
+author=Arduino
+maintainer=Arduino <[email protected]>
+sentence=
+paragraph=
+category=Communication
+url=
+architectures=*
+includes=MyLib.h