[breaking] Remove auto detection of Arduino IDE built-in libraries and tools / Allow gRPC install of built-in libraries (#1817)

* Removed IDE-bundle autodetection

* Do not return bundled tools folder with HardwarePaths

* Created configuration functions to obtain data dir and downloads dir

* Bundled libraries configuration is now considere a single directory

* Allow library installation on bundled libs dir via gRPC

* Added integration test for library install in bundled directory

* Fixed nil pointer exception in build options extraction

* Updated docs

* Removed useless custom env bindings

* Applied code review changes

Co-authored-by: per1234 <[email protected]>

* Set builtin libraries dir by default when running as a daemon

* Added test for regression

See #1817 (review)

* Fixed regression: 'lib uninstall' must operate only on user dir

Co-authored-by: per1234 <[email protected]>

Author: cmaglie

arduino/cores/packagemanager/loader.go

@@ -38,7 +38,7 @@ func (pm *Builder) LoadHardware() []error {
 	hardwareDirs := configuration.HardwareDirectories(configuration.Settings)
 	merr := pm.LoadHardwareFromDirectories(hardwareDirs)
 
-	bundleToolDirs := configuration.BundleToolsDirectories(configuration.Settings)
+	bundleToolDirs := configuration.BuiltinToolsDirectories(configuration.Settings)
 	merr = append(merr, pm.LoadToolsFromBundleDirectories(bundleToolDirs)...)
 
 	return merr

arduino/cores/packagemanager/package_manager_test.go

@@ -224,7 +224,7 @@ func TestFindToolsRequiredForBoard(t *testing.T) {
 	pmb := packagemanager.NewBuilder(
 		dataDir1,
 		configuration.PackagesDir(configuration.Settings),
-		paths.New(configuration.Settings.GetString("directories.Downloads")),
+		configuration.DownloadsDir(configuration.Settings),
 		dataDir1,
 		"test",
 	)

arduino/libraries/libraries_location.go

@@ -93,7 +93,7 @@ func (d *LibraryLocation) UnmarshalJSON(b []byte) error {
 func (d *LibraryLocation) ToRPCLibraryLocation() rpc.LibraryLocation {
 	switch *d {
 	case IDEBuiltIn:
-		return rpc.LibraryLocation_LIBRARY_LOCATION_IDE_BUILTIN
+		return rpc.LibraryLocation_LIBRARY_LOCATION_BUILTIN
 	case PlatformBuiltIn:
 		return rpc.LibraryLocation_LIBRARY_LOCATION_PLATFORM_BUILTIN
 	case ReferencedPlatformBuiltIn:
@@ -110,7 +110,7 @@ func (d *LibraryLocation) ToRPCLibraryLocation() rpc.LibraryLocation {
 // FromRPCLibraryLocation converts a rpc.LibraryLocation to a LibraryLocation
 func FromRPCLibraryLocation(l rpc.LibraryLocation) LibraryLocation {
 	switch l {
-	case rpc.LibraryLocation_LIBRARY_LOCATION_IDE_BUILTIN:
+	case rpc.LibraryLocation_LIBRARY_LOCATION_BUILTIN:
 		return IDEBuiltIn
 	case rpc.LibraryLocation_LIBRARY_LOCATION_PLATFORM_BUILTIN:
 		return PlatformBuiltIn
@@ -124,3 +124,15 @@ func FromRPCLibraryLocation(l rpc.LibraryLocation) LibraryLocation {
 		panic(fmt.Sprintf("invalid rpc.LibraryLocation value %d", l))
 	}
 }
+
+// FromRPCLibraryInstallLocation converts a rpc.LibraryInstallLocation to a LibraryLocation
+func FromRPCLibraryInstallLocation(l rpc.LibraryInstallLocation) LibraryLocation {
+	switch l {
+	case rpc.LibraryInstallLocation_LIBRARY_INSTALL_LOCATION_BUILTIN:
+		return IDEBuiltIn
+	case rpc.LibraryInstallLocation_LIBRARY_INSTALL_LOCATION_USER:
+		return User
+	default:
+		panic(fmt.Sprintf("invalid rpc.LibraryInstallLocation value %d", l))
+	}
+}

arduino/libraries/librariesmanager/install.go

@@ -48,13 +48,13 @@ var (
 // InstallPrerequisiteCheck performs prequisite checks to install a library. It returns the
 // install path, where the library should be installed and the possible library that is already
 // installed on the same folder and it's going to be replaced by the new one.
-func (lm *LibrariesManager) InstallPrerequisiteCheck(indexLibrary *librariesindex.Release) (*paths.Path, *libraries.Library, error) {
+func (lm *LibrariesManager) InstallPrerequisiteCheck(indexLibrary *librariesindex.Release, installLocation libraries.LibraryLocation) (*paths.Path, *libraries.Library, error) {
 	saneName := utils.SanitizeName(indexLibrary.Library.Name)
 
 	var replaced *libraries.Library
 	if installedLibs, have := lm.Libraries[saneName]; have {
 		for _, installedLib := range installedLibs.Alternatives {
-			if installedLib.Location != libraries.User {
+			if installedLib.Location != installLocation {
 				continue
 			}
 			if installedLib.Version != nil && installedLib.Version.Equal(indexLibrary.Version) {
@@ -64,9 +64,12 @@ func (lm *LibrariesManager) InstallPrerequisiteCheck(indexLibrary *librariesinde
 		}
 	}
 
-	libsDir := lm.getUserLibrariesDir()
+	libsDir := lm.getLibrariesDir(installLocation)
 	if libsDir == nil {
-		return nil, nil, fmt.Errorf(tr("User directory not set"))
+		if installLocation == libraries.User {
+			return nil, nil, fmt.Errorf(tr("User directory not set"))
+		}
+		return nil, nil, fmt.Errorf(tr("Builtin libraries directory not set"))
 	}
 
 	libPath := libsDir.Join(saneName)
@@ -79,8 +82,8 @@ func (lm *LibrariesManager) InstallPrerequisiteCheck(indexLibrary *librariesinde
 }
 
 // Install installs a library on the specified path.
-func (lm *LibrariesManager) Install(indexLibrary *librariesindex.Release, libPath *paths.Path) error {
-	libsDir := lm.getUserLibrariesDir()
+func (lm *LibrariesManager) Install(indexLibrary *librariesindex.Release, libPath *paths.Path, installLocation libraries.LibraryLocation) error {
+	libsDir := lm.getLibrariesDir(installLocation)
 	if libsDir == nil {
 		return fmt.Errorf(tr("User directory not set"))
 	}
@@ -100,9 +103,9 @@ func (lm *LibrariesManager) Uninstall(lib *libraries.Library) error {
 	return nil
 }
 
-//InstallZipLib  installs a Zip library on the specified path.
+// InstallZipLib installs a Zip library on the specified path.
 func (lm *LibrariesManager) InstallZipLib(ctx context.Context, archivePath string, overwrite bool) error {
-	libsDir := lm.getUserLibrariesDir()
+	libsDir := lm.getLibrariesDir(libraries.User)
 	if libsDir == nil {
 		return fmt.Errorf(tr("User directory not set"))
 	}
@@ -184,9 +187,9 @@ func (lm *LibrariesManager) InstallZipLib(ctx context.Context, archivePath strin
 	return nil
 }
 
-//InstallGitLib  installs a library hosted on a git repository on the specified path.
+// InstallGitLib installs a library hosted on a git repository on the specified path.
 func (lm *LibrariesManager) InstallGitLib(gitURL string, overwrite bool) error {
-	libsDir := lm.getUserLibrariesDir()
+	libsDir := lm.getLibrariesDir(libraries.User)
 	if libsDir == nil {
 		return fmt.Errorf(tr("User directory not set"))
 	}

arduino/libraries/librariesmanager/librariesmanager.go

@@ -78,9 +78,9 @@ func (alts *LibraryAlternatives) Remove(library *libraries.Library) {
 }
 
 // FindVersion returns the library mathching the provided version or nil if not found
-func (alts *LibraryAlternatives) FindVersion(version *semver.Version) *libraries.Library {
+func (alts *LibraryAlternatives) FindVersion(version *semver.Version, installLocation libraries.LibraryLocation) *libraries.Library {
 	for _, lib := range alts.Alternatives {
-		if lib.Version.Equal(version) {
+		if lib.Version.Equal(version) && lib.Location == installLocation {
 			return lib
 		}
 	}
@@ -118,6 +118,7 @@ func NewLibraryManager(indexDir *paths.Path, downloadsDir *paths.Path) *Librarie
 // LoadIndex reads a library_index.json from a file and returns
 // the corresponding Index structure.
 func (lm *LibrariesManager) LoadIndex() error {
+	logrus.WithField("index", lm.IndexFile).Info("Loading libraries index file")
 	index, err := librariesindex.LoadIndex(lm.IndexFile)
 	if err != nil {
 		lm.Index = librariesindex.EmptyIndex
@@ -175,9 +176,9 @@ func (lm *LibrariesManager) RescanLibraries() []*status.Status {
 	return statuses
 }
 
-func (lm *LibrariesManager) getUserLibrariesDir() *paths.Path {
+func (lm *LibrariesManager) getLibrariesDir(installLocation libraries.LibraryLocation) *paths.Path {
 	for _, dir := range lm.LibrariesDir {
-		if dir.Location == libraries.User {
+		if dir.Location == installLocation {
 			return dir.Path
 		}
 	}
@@ -244,7 +245,7 @@ func (lm *LibrariesManager) LoadLibraryFromDir(libRootDir *paths.Path, location
 // FindByReference return the installed library matching the Reference
 // name and version or, if the version is nil, the library installed
 // in the User folder.
-func (lm *LibrariesManager) FindByReference(libRef *librariesindex.Reference) *libraries.Library {
+func (lm *LibrariesManager) FindByReference(libRef *librariesindex.Reference, installLocation libraries.LibraryLocation) *libraries.Library {
 	saneName := utils.SanitizeName(libRef.Name)
 	alternatives, have := lm.Libraries[saneName]
 	if !have {
@@ -253,11 +254,11 @@ func (lm *LibrariesManager) FindByReference(libRef *librariesindex.Reference) *l
 	// TODO: Move "search into user" into another method...
 	if libRef.Version == nil {
 		for _, candidate := range alternatives.Alternatives {
-			if candidate.Location == libraries.User {
+			if candidate.Location == installLocation {
 				return candidate
 			}
 		}
 		return nil
 	}
-	return alternatives.FindVersion(libRef.Version)
+	return alternatives.FindVersion(libRef.Version, installLocation)
 }

cli/cache/clean.go

@@ -40,8 +40,8 @@ func initCleanCommand() *cobra.Command {
 func runCleanCommand(cmd *cobra.Command, args []string) {
 	logrus.Info("Executing `arduino-cli cache clean`")
 
-	cachePath := configuration.Settings.GetString("directories.Downloads")
-	err := os.RemoveAll(cachePath)
+	cachePath := configuration.DownloadsDir(configuration.Settings)
+	err := cachePath.RemoveAll()
 	if err != nil {
 		feedback.Errorf(tr("Error cleaning caches: %v"), err)
 		os.Exit(errorcodes.ErrGeneric)

cli/cli.go

@@ -158,7 +158,7 @@ func preRun(cmd *cobra.Command, args []string) {
 	configFile := configuration.Settings.ConfigFileUsed()
 
 	// initialize inventory
-	err := inventory.Init(configuration.Settings.GetString("directories.Data"))
+	err := inventory.Init(configuration.DataDir(configuration.Settings).String())
 	if err != nil {
 		feedback.Errorf("Error: %v", err)
 		os.Exit(errorcodes.ErrBadArgument)

cli/config/validate.go

@@ -30,6 +30,8 @@ var validMap = map[string]reflect.Kind{
 	"directories.data":              reflect.String,
 	"directories.downloads":         reflect.String,
 	"directories.user":              reflect.String,
+	"directories.builtin.tools":     reflect.String,
+	"directories.builtin.libraries": reflect.String,
 	"library.enable_unsafe_install": reflect.Bool,
 	"logging.file":                  reflect.String,
 	"logging.format":                reflect.String,

cli/core/search.go

@@ -34,7 +34,6 @@ import (
 	"github.com/arduino/arduino-cli/configuration"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"github.com/arduino/arduino-cli/table"
-	"github.com/arduino/go-paths-helper"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -130,7 +129,7 @@ func (sr searchResults) String() string {
 // of 24 hours is used.
 // Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
 func indexesNeedUpdating(duration string) bool {
-	indexpath := paths.New(configuration.Settings.GetString("directories.Data"))
+	indexpath := configuration.DataDir(configuration.Settings)
 
 	now := time.Now()
 	modTimeThreshold, err := time.ParseDuration(duration)

cli/daemon/daemon.go

@@ -72,6 +72,9 @@ func NewCommand() *cobra.Command {
 func runDaemonCommand(cmd *cobra.Command, args []string) {
 	logrus.Info("Executing `arduino-cli daemon`")
 
+	// Bundled libraries support is enabled by default when running as a daemon
+	configuration.Settings.SetDefault("directories.builtin.Libraries", configuration.GetDefaultBuiltinLibrariesDir())
+
 	port := configuration.Settings.GetString("daemon.port")
 	gRPCOptions := []grpc.ServerOption{}
 	if debugFile != "" {