feat: [close Vanilla-OS#165] Differ API support

* Implement differ WIP

* Fix BaseImagePackageDiff

* Overlay package diff

* Unify package diff functions return types

* Show package diff on upgrade check

* Add upgrade check for packages

* Update vendored packages

* Package diff improvements

Author: matbme

cmd/upgrade.go

@@ -14,11 +14,14 @@ package cmd
 */
 
 import (
+	"fmt"
 	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
 
 	"github.com/vanilla-os/abroot/core"
+	"github.com/vanilla-os/differ/diff"
 	"github.com/vanilla-os/orchid/cmdr"
 )
 
@@ -50,11 +53,6 @@ func NewUpgradeCommand() *cmdr.Command {
 }
 
 func upgrade(cmd *cobra.Command, args []string) error {
-	if !core.RootCheck(false) {
-		cmdr.Error.Println(abroot.Trans("upgrade.rootRequired"))
-		return nil
-	}
-
 	checkOnly, err := cmd.Flags().GetBool("check-only")
 	if err != nil {
 		cmdr.Error.Println(err)
@@ -68,14 +66,49 @@ func upgrade(cmd *cobra.Command, args []string) error {
 	}
 
 	if checkOnly {
-		_, res := aBsys.CheckUpdate()
+		cmdr.Info.Println(abroot.Trans("upgrade.checkingSystemUpdate"))
+
+		// Check for image updates
+		newDigest, res := aBsys.CheckUpdate()
 		if res {
-			cmdr.Info.Println(abroot.Trans("upgrade.updateAvailable"))
-			os.Exit(0)
+			cmdr.Info.Println(abroot.Trans("upgrade.systemUpdateAvailable"))
+
+			added, upgraded, downgraded, removed, err := core.BaseImagePackageDiff(aBsys.CurImage.Digest, newDigest)
+			if err != nil {
+				return err
+			}
+			err = renderPackageDiff(added, upgraded, downgraded, removed)
+			if err != nil {
+				return err
+			}
 		} else {
 			cmdr.Info.Println(abroot.Trans("upgrade.noUpdateAvailable"))
-			os.Exit(1)
 		}
+
+		// Check for package updates
+		cmdr.Info.Println(abroot.Trans("upgrade.checkingPackageUpdate"))
+		added, upgraded, downgraded, removed, err := core.OverlayPackageDiff()
+		if err != nil {
+			return err
+		}
+
+		sumChanges := len(added) + len(upgraded) + len(downgraded) + len(removed)
+		if sumChanges == 0 {
+			cmdr.Info.Println(abroot.Trans("upgrade.noUpdateAvailable"))
+		} else {
+			cmdr.Info.Sprintf(abroot.Trans("upgrade.packageUpdateAvailable"), sumChanges)
+
+			err = renderPackageDiff(added, upgraded, downgraded, removed)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	if !core.RootCheck(false) {
+		cmdr.Error.Println(abroot.Trans("upgrade.rootRequired"))
 		return nil
 	}
 
@@ -94,7 +127,7 @@ func upgrade(cmd *cobra.Command, args []string) error {
 
 	err = aBsys.RunOperation(operation)
 	if err != nil {
-		if err == core.NoUpdateError {
+		if err == core.ErrNoUpdate {
 			cmdr.Info.Println(abroot.Trans("upgrade.noUpdateAvailable"))
 			return err
 		}
@@ -106,3 +139,43 @@ func upgrade(cmd *cobra.Command, args []string) error {
 	os.Exit(0)
 	return nil
 }
+
+func renderPackageDiff(added, upgraded, downgraded, removed []diff.PackageDiff) error {
+	pkgFmt := "%s  '%s' -> '%s'"
+
+	// Calculate largest string for proper alignment
+	largestPkgName := 0
+	for _, pkgSet := range [][]diff.PackageDiff{added, upgraded, downgraded, removed} {
+		for _, pkg := range pkgSet {
+			if len(pkg.Name) > largestPkgName {
+				largestPkgName = len(pkg.Name)
+			}
+		}
+	}
+
+	for _, pkgSet := range []struct {
+		Set    []diff.PackageDiff
+		Header string
+		Color  cmdr.Color
+	}{
+		{added, abroot.Trans("upgrade.added"), cmdr.FgGreen},
+		{upgraded, abroot.Trans("upgrade.upgraded"), cmdr.FgBlue},
+		{downgraded, abroot.Trans("upgrade.downgraded"), cmdr.FgYellow},
+		{removed, abroot.Trans("upgrade.removed"), cmdr.FgRed},
+	} {
+		cmdr.NewStyle(cmdr.Bold, pkgSet.Color).Println(pkgSet.Header + ":")
+		bulletItems := []cmdr.BulletListItem{}
+		for _, pkg := range pkgSet.Set {
+			bulletItems = append(bulletItems, cmdr.BulletListItem{
+				Level: 1,
+				Text:  fmt.Sprintf(pkgFmt, pkg.Name+strings.Repeat(" ", largestPkgName-len(pkg.Name)), pkg.PreviousVersion, pkg.NewVersion),
+			})
+		}
+		err := cmdr.BulletList.WithItems(bulletItems).Render()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

config/abroot.json

@@ -14,6 +14,8 @@
     "iPkgMngRm": "apt remove -y",
     "iPkgMngApi": "https://packages.vanillaos.org/api/pkg/{packageName}",
 
+    "differURL": "https://differ.vanillaos.org",
+
     "partLabelVar": "vos-var",
     "partLabelA": "vos-a",
     "partLabelB": "vos-b",

core/package-diff.go

@@ -0,0 +1,120 @@
+package core
+
+/*	License: GPLv3
+	Authors:
+		Mirko Brombin <[email protected]>
+		Vanilla OS Contributors <https://github.com/vanilla-os/>
+	Copyright: 2023
+	Description:
+		ABRoot is utility which provides full immutability and
+		atomicity to a Linux system, by transacting between
+		two root filesystems. Updates are performed using OCI
+		images, to ensure that the system is always in a
+		consistent state.
+*/
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/vanilla-os/abroot/extras/dpkg"
+	"github.com/vanilla-os/abroot/settings"
+	"github.com/vanilla-os/differ/diff"
+)
+
+// BaseImagePackageDiff retrieves the added, removed, upgraded and downgraded
+// base packages (the ones bundled with the image).
+func BaseImagePackageDiff(currentDigest, newDigest string) (
+	added, upgraded, downgraded, removed []diff.PackageDiff,
+	err error,
+) {
+	PrintVerbose("PackageDiff.BaseImagePackageDiff: running...")
+
+	imageComponents := strings.Split(settings.Cnf.Name, "/")
+	imageName := imageComponents[len(imageComponents)-1]
+	reqUrl := fmt.Sprintf("%s/images/%s/diff", settings.Cnf.DifferURL, imageName)
+	body := fmt.Sprintf("{\"old_digest\": \"%s\", \"new_digest\": \"%s\"}", currentDigest, newDigest)
+
+	PrintVerbose("PackageDiff.BaseImagePackageDiff: Requesting base image diff to %s with body:\n%s", reqUrl, body)
+
+	request, err := http.NewRequest(http.MethodGet, reqUrl, strings.NewReader(body))
+	if err != nil {
+		PrintVerbose("PackageDiff.BaseImagePackageDiff:err: %s", err)
+		return
+	}
+	defer request.Body.Close()
+
+	resp, err := http.DefaultClient.Do(request)
+	if err != nil {
+		PrintVerbose("PackageDiff.BaseImagePackageDiff(1):err: %s", err)
+		return
+	}
+
+	contents, err := io.ReadAll(resp.Body)
+	if err != nil {
+		PrintVerbose("PackageDiff.BaseImagePackageDiff(2):err: %s", err)
+		return
+	}
+
+	pkgDiff := struct {
+		Added, Upgraded, Downgraded, Removed []diff.PackageDiff
+	}{}
+	err = json.Unmarshal(contents, &pkgDiff)
+	if err != nil {
+		PrintVerbose("PackageDiff.BaseImagePackageDiff(3):err: %s", err)
+		return
+	}
+
+	added = pkgDiff.Added
+	upgraded = pkgDiff.Upgraded
+	downgraded = pkgDiff.Downgraded
+	removed = pkgDiff.Removed
+
+	return
+}
+
+// OverlayPackageDiff retrieves the added, removed, upgraded and downgraded
+// overlay packages (the ones added manually via `abroot pkg add`).
+func OverlayPackageDiff() (
+	added, upgraded, downgraded, removed []diff.PackageDiff,
+	err error,
+) {
+	PrintVerbose("OverlayPackageDiff: running...")
+
+	pkgM := NewPackageManager(false)
+	addedPkgs, err := pkgM.GetAddPackages()
+	if err != nil {
+		PrintVerbose("PackageDiff.OverlayPackageDiff:err: %s", err)
+		return
+	}
+
+	localAddedVersions := dpkg.DpkgBatchGetPackageVersion(addedPkgs)
+	localAdded := map[string]string{}
+	for i := 0; i < len(addedPkgs); i++ {
+		if localAddedVersions[i] != "" {
+			localAdded[addedPkgs[i]] = localAddedVersions[i]
+		}
+	}
+
+	remoteAdded := map[string]string{}
+	var pkgInfo map[string]any
+	for pkgName := range localAdded {
+		pkgInfo, err = GetRepoContentsForPkg(pkgName)
+		if err != nil {
+			PrintVerbose("PackageDiff.OverlayPackageDiff(1):err: %s", err)
+			return
+		}
+		version, ok := pkgInfo["version"].(string)
+		if !ok {
+			err = fmt.Errorf("PackageDiff.OverlayPackageDiff(2):err: Unexpected value when retrieving upstream version of '%s'", pkgName)
+			return
+		}
+		remoteAdded[pkgName] = version
+	}
+
+	added, upgraded, downgraded, removed = diff.DiffPackages(localAdded, remoteAdded)
+	return
+}

core/packages.go

@@ -14,9 +14,12 @@ package core
 */
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -294,7 +297,7 @@ func (p *PackageManager) getPackages(file string) ([]string, error) {
 		return pkgs, err
 	}
 
-	pkgs = strings.Split(string(b), "\n")
+	pkgs = strings.Split(strings.TrimSpace(string(b)), "\n")
 
 	PrintVerbose("PackageManager.getPackages: returning packages")
 	return pkgs, nil
@@ -374,9 +377,10 @@ func (p *PackageManager) processApplyPackages() (string, string) {
 
 	var addPkgs, removePkgs []string
 	for _, pkg := range unstaged {
-		if pkg.Status == ADD {
+		switch pkg.Status {
+		case ADD:
 			addPkgs = append(addPkgs, pkg.Name)
-		} else if pkg.Status == REMOVE {
+		case REMOVE:
 			removePkgs = append(removePkgs, pkg.Name)
 		}
 	}
@@ -462,16 +466,36 @@ func (p *PackageManager) GetFinalCmd(operation ABSystemOperation) string {
 	return cmd
 }
 
-func (p *PackageManager) ExistsInRepo(pkg string) error {
-	PrintVerbose("PackageManager.ExistsInRepo: running...")
-
+// assertPkgMngApiSetUp checks whether the repo API is properly configured.
+// If a configuration exists but is malformed, returns an error.
+func assertPkgMngApiSetUp() (bool, error) {
 	if settings.Cnf.IPkgMngApi == "" {
-		PrintVerbose("PackageManager.ExistsInRepo: no API url set, will not check if package exists. This could lead to errors")
-		return nil
+		PrintVerbose("PackageManager.assertPkgMngApiSetUp: no API url set, will not check if package exists. This could lead to errors")
+		return false, nil
+	}
+
+	_, err := url.ParseRequestURI(settings.Cnf.IPkgMngApi)
+	if err != nil {
+		return false, fmt.Errorf("PackageManager.assertPkgMngApiSetUp: Value set as API url (%s) is not a valid URL", settings.Cnf.IPkgMngApi)
 	}
 
 	if !strings.Contains(settings.Cnf.IPkgMngApi, "{packageName}") {
-		return fmt.Errorf("PackageManager.ExistsInRepo: API url does not contain {packageName} placeholder. ABRoot is probably misconfigured, please report the issue to the maintainers of the distribution")
+		return false, fmt.Errorf("PackageManager.assertPkgMngApiSetUp: API url does not contain {packageName} placeholder. ABRoot is probably misconfigured, please report the issue to the maintainers of the distribution")
+	}
+
+	PrintVerbose("PackageManager.assertPkgMngApiSetUp: Repo is set up properly")
+	return true, nil
+}
+
+func (p *PackageManager) ExistsInRepo(pkg string) error {
+	PrintVerbose("PackageManager.ExistsInRepo: running...")
+
+	ok, err := assertPkgMngApiSetUp()
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return nil
 	}
 
 	url := strings.Replace(settings.Cnf.IPkgMngApi, "{packageName}", pkg, 1)
@@ -491,3 +515,40 @@ func (p *PackageManager) ExistsInRepo(pkg string) error {
 	PrintVerbose("PackageManager.ExistsInRepo: package exists in repo")
 	return nil
 }
+
+// GetRepoContentsForPkg retrieves package information from the repository API
+func GetRepoContentsForPkg(pkg string) (map[string]any, error) {
+	PrintVerbose("PackageManager.GetRepoContentsForPkg: running...")
+
+	ok, err := assertPkgMngApiSetUp()
+	if err != nil {
+		return map[string]any{}, err
+	}
+	if !ok {
+		return map[string]any{}, errors.New("PackageManager.GetRepoContentsForPkg: no API url set, cannot query package information")
+	}
+
+	url := strings.Replace(settings.Cnf.IPkgMngApi, "{packageName}", pkg, 1)
+	PrintVerbose("PackageManager.GetRepoContentsForPkg: fetching package information in: " + url)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		PrintVerbose("PackageManager.GetRepoContentsForPkg:err: " + err.Error())
+		return map[string]any{}, err
+	}
+
+	contents, err := io.ReadAll(resp.Body)
+	if err != nil {
+		PrintVerbose("PackageManager.GetRepoContentsForPkg(2):err: %s", err)
+		return map[string]any{}, err
+	}
+
+	pkgInfo := map[string]any{}
+	err = json.Unmarshal(contents, &pkgInfo)
+	if err != nil {
+		PrintVerbose("PackageManager.GetRepoContentsForPkg(3):err: %s", err)
+		return map[string]any{}, err
+	}
+
+	return pkgInfo, nil
+}