optimize tref out

Author: raphaelvigee

internal/engine/link.go

@@ -481,19 +481,16 @@ func (e *Engine) getDef(ctx context.Context, rs *RequestState, c DefContainer) (
 func addSrcTargetToInputs(def *pluginv1.TargetDef, currentTargetAddrHash func() string) {
 	inputs := def.GetInputs()
 	for _, input := range inputs {
-		if input.GetRef().GetPackage() != tref.QueryPackage {
-			continue
-		}
+		refo := input.GetRef()
+		ref := refo.GetTarget()
 
-		ref := input.GetRef()
-		args := ref.GetArgs()
-		if args == nil {
-			args = map[string]string{}
+		if ref.GetPackage() != tref.QueryPackage {
+			continue
 		}
-		args[querySrcTargetArg] = currentTargetAddrHash()
-		ref.SetArgs(args)
 
-		input.SetRef(ref)
+		ref = tref.WithArg(ref, querySrcTargetArg, currentTargetAddrHash())
+		refo.SetTarget(ref)
+		refo.ClearHash()
 	}
 	def.SetInputs(inputs)
 }
@@ -760,7 +757,7 @@ func (e *Engine) innerLink(ctx context.Context, rs *RequestState, def *TargetDef
 
 			if input.GetRef().HasOutput() {
 				if !slices.Contains(linkedDep.GetOutputs(), input.GetRef().GetOutput()) {
-					return fmt.Errorf("%v doesnt have a named output %q", tref.Format(input.GetRef()), input.GetRef().GetOutput())
+					return fmt.Errorf("%v doesnt have a named output %q", tref.FormatOut(input.GetRef()), input.GetRef().GetOutput())
 				}
 
 				outputs = []string{input.GetRef().GetOutput()}

internal/tmatch/match.go

@@ -25,6 +25,7 @@ func walkDirs(ctx context.Context, walkRoot, root string, filter func(path strin
 	}
 
 	return func(yield func(string, error) bool) {
+		stop := false
 		err := fsWalkCache.Walk(walkRoot, func(path string, d fs.DirEntry, err error) error {
 			if d == nil || !d.IsDir() {
 				return nil
@@ -48,13 +49,14 @@ func walkDirs(ctx context.Context, walkRoot, root string, filter func(path strin
 			}
 
 			if !yield(pkg, nil) {
+				stop = true
 				return fs.SkipAll
 			}
 
 			return nil
 		})
 		if err != nil {
-			if !yield("", err) {
+			if !stop && !yield("", err) {
 				return
 			}
 		}

lib/tref/format.go

@@ -7,8 +7,6 @@ import (
 	"strings"
 	"unsafe"
 
-	"github.com/hephbuild/heph/internal/hproto/hashpb"
-
 	"github.com/hephbuild/heph/internal/hsync"
 
 	cache "github.com/Code-Hex/go-generics-cache"
@@ -22,28 +20,12 @@ import (
 type Ref = pluginv1.TargetRef
 type RefOut = pluginv1.TargetRefWithOutput
 
-var _ Refable = (*Ref)(nil)
-var _ Refable = (*RefOut)(nil)
-var _ RefableOut = (*RefOut)(nil)
-
-type Refable interface {
-	GetArgs() map[string]string
-	GetPackage() string
-	GetName() string
-}
-
 type HashStore interface {
 	GetHash() uint64
 	HasHash() bool
 	SetHash(uint64)
 }
 
-type RefableOut interface {
-	Refable
-	GetOutput() string
-	GetFilters() []string
-}
-
 func FormatFile(pkg string, file string) string {
 	return Format(New(JoinPackage("@heph/file", pkg), "content", map[string]string{"f": file}))
 }
@@ -117,89 +99,84 @@ func ParseQuery(ref *pluginv1.TargetRef) (QueryOptions, error) {
 var formatCache = cache.New[uint64, string](cache.AsLFU[uint64, string](lfu.WithCapacity(10000)))
 var formatSf = hsingleflight.Group[uint64, string]{}
 
-var formatHashPool = hsync.Pool[*xxh3.Hasher]{New: xxh3.New}
-
-func sumRef(ref hashpb.StableWriter) uint64 {
-	h := formatHashPool.Get()
-	defer formatHashPool.Put(h)
-	h.Reset()
+var formatOutCache = cache.New[uint64, string](cache.AsLFU[uint64, string](lfu.WithCapacity(10000)))
+var formatOutSf = hsingleflight.Group[uint64, string]{}
 
-	switch ref := ref.(type) {
-	case *pluginv1.TargetRef:
-		sumRefTargetRef(h, ref)
-	case *pluginv1.TargetRefWithOutput:
-		sumRefTargetRefWithOutput(h, ref)
-	default:
-		hashpb.Hash(h, ref, nil)
-	}
+var formatHashPool = hsync.Pool[*xxh3.Hasher]{New: xxh3.New}
 
-	return h.Sum64()
-}
+func sumRefTargetRef(m *pluginv1.TargetRef) uint64 {
+	hasher := formatHashPool.Get()
+	defer formatHashPool.Put(hasher)
+	hasher.Reset()
 
-func sumRefTargetRef(hasher *xxh3.Hasher, m *pluginv1.TargetRef) {
 	_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(m.GetPackage()), len(m.GetPackage())))
 	_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(m.GetName()), len(m.GetName())))
 	for k, v := range hmaps.Sorted(m.GetArgs()) {
 		_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(k), len(k)))
 		_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(v), len(v)))
 	}
+
+	return hasher.Sum64()
 }
 
-func sumRefTargetRefWithOutput(hasher *xxh3.Hasher, m *pluginv1.TargetRefWithOutput) {
-	_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(m.GetPackage()), len(m.GetPackage())))
-	_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(m.GetName()), len(m.GetName())))
-	for k, v := range hmaps.Sorted(m.GetArgs()) {
-		_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(k), len(k)))
-		_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(v), len(v)))
-	}
-	_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(m.GetOutput()), len(m.GetOutput())))
-	if len(m.GetFilters()) > 0 {
-		for _, v := range m.GetFilters() {
+func sumRefTargetRefWithOutput(m *pluginv1.TargetRefWithOutput) uint64 {
+	hasher := formatHashPool.Get()
+	defer formatHashPool.Put(hasher)
+	hasher.Reset()
+
+	{
+		m := m.GetTarget()
+
+		_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(m.GetPackage()), len(m.GetPackage())))
+		_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(m.GetName()), len(m.GetName())))
+		for k, v := range hmaps.Sorted(m.GetArgs()) {
+			_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(k), len(k)))
 			_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(v), len(v)))
 		}
 	}
+
+	_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(m.GetOutput()), len(m.GetOutput())))
+	for _, v := range m.GetFilters() {
+		_, _ = hasher.Write(unsafe.Slice(unsafe.StringData(v), len(v)))
+	}
+
+	return hasher.Sum64()
 }
 
-func Format(ref Refable) string {
-	if refh, ok := ref.(hashpb.StableWriter); ok {
-		var sum uint64
-		if s, ok := ref.(HashStore); ok {
-			if s.HasHash() {
-				sum = s.GetHash()
-			} else {
-				sum = sumRef(refh)
-				s.SetHash(sum)
-			}
-		} else {
-			sum = sumRef(refh)
-		}
+func Format(ref *Ref) string {
+	var sum uint64
+	if ref.HasHash() {
+		sum = ref.GetHash()
+	} else {
+		sum = sumRefTargetRef(ref)
+		ref.SetHash(sum)
+	}
 
+	f, ok := formatCache.Get(sum)
+	if ok {
+		return f
+	}
+
+	f, _, _ = formatSf.Do(sum, func() (string, error) {
 		f, ok := formatCache.Get(sum)
 		if ok {
-			return f
+			return f, nil
 		}
 
-		f, _, _ = formatSf.Do(sum, func() (string, error) {
-			f, ok := formatCache.Get(sum)
-			if ok {
-				return f, nil
-			}
-
-			f = format(ref)
+		f = format(ref)
 
-			formatCache.Set(sum, f)
+		formatCache.Set(sum, f)
 
-			return f, nil
-		})
+		return f, nil
+	})
 
-		return f
-	}
+	return f
 
-	return format(ref)
 }
 
-func format(ref Refable) string {
+func format(ref *Ref) string {
 	var sb strings.Builder
+
 	sb.WriteString("//")
 	sb.WriteString(ref.GetPackage())
 	sb.WriteString(":")
@@ -225,24 +202,65 @@ func format(ref Refable) string {
 		}
 	}
 
-	if ref, ok := ref.(RefableOut); ok {
-		out := ref.GetOutput()
-		if out != "" {
-			sb.WriteString("|")
-			sb.WriteString(out)
+	return sb.String()
+}
+
+func FormatOut(ref *RefOut) string {
+	var sum uint64
+	if ref.HasHash() {
+		sum = ref.GetHash()
+	} else {
+		sum = sumRefTargetRefWithOutput(ref)
+		ref.SetHash(sum)
+	}
+
+	f, ok := formatOutCache.Get(sum)
+	if ok {
+		return f
+	}
+
+	f, _, _ = formatOutSf.Do(sum, func() (string, error) {
+		f, ok := formatOutCache.Get(sum)
+		if ok {
+			return f, nil
 		}
 
-		if len(ref.GetFilters()) > 0 {
-			sb.WriteString(" filters=")
-			first := true
-			for _, f := range ref.GetFilters() {
-				if !first {
-					sb.WriteString(",")
-				} else {
-					first = false
-				}
-				sb.WriteString(f)
+		f = formatOut(ref)
+
+		formatOutCache.Set(sum, f)
+
+		return f, nil
+	})
+
+	return f
+
+}
+
+func formatOut(ref *RefOut) string {
+	if ref.GetOutput() == "" && len(ref.GetFilters()) == 0 {
+		return Format(ref.GetTarget())
+	}
+
+	var sb strings.Builder
+
+	sb.WriteString(Format(ref.GetTarget()))
+
+	out := ref.GetOutput()
+	if out != "" {
+		sb.WriteString("|")
+		sb.WriteString(out)
+	}
+
+	if len(ref.GetFilters()) > 0 {
+		sb.WriteString(" filters=")
+		first := true
+		for _, f := range ref.GetFilters() {
+			if !first {
+				sb.WriteString(",")
+			} else {
+				first = false
 			}
+			sb.WriteString(f)
 		}
 	}
 

lib/tref/utils.go

@@ -14,27 +14,6 @@ func Equal(a, b *pluginv1.TargetRef) bool {
 	return a.GetPackage() == b.GetPackage() && a.GetName() == b.GetName() && maps.Equal(a.GetArgs(), b.GetArgs())
 }
 
-func CompareOut(a, b *pluginv1.TargetRefWithOutput) int {
-	if v := Compare(WithoutOut(a), WithoutOut(b)); v != 0 {
-		return v
-	}
-
-	if a.HasOutput() && b.HasOutput() {
-		if v := strings.Compare(a.GetOutput(), b.GetOutput()); v != 0 {
-			return v
-		}
-	} else {
-		if !a.HasOutput() {
-			return 1
-		}
-		if !b.HasOutput() {
-			return -1
-		}
-	}
-
-	return 0
-}
-
 func Compare(a, b *pluginv1.TargetRef) int {
 	if v := strings.Compare(a.GetPackage(), b.GetPackage()); v != 0 {
 		return v
@@ -113,20 +92,23 @@ func WithOut(ref *pluginv1.TargetRef, output string) *pluginv1.TargetRefWithOutp
 	}
 
 	return pluginv1.TargetRefWithOutput_builder{
-		Package: htypes.Ptr(ref.GetPackage()),
-		Name:    htypes.Ptr(ref.GetName()),
-		Args:    ref.GetArgs(),
-		Output:  outputp,
+		Target: ref,
+		Output: outputp,
 	}.Build()
 }
 
 func WithFilters(ref *pluginv1.TargetRefWithOutput, filters []string) *pluginv1.TargetRefWithOutput {
+	if len(filters) == 0 && len(ref.GetFilters()) == 0 {
+		return ref
+	}
+
 	ref = hproto.Clone(ref)
+	ref.ClearHash()
 	ref.SetFilters(filters)
 
 	return ref
 }
 
 func WithoutOut(ref *pluginv1.TargetRefWithOutput) *pluginv1.TargetRef {
-	return New(ref.GetPackage(), ref.GetName(), ref.GetArgs())
+	return ref.GetTarget()
 }

plugin/pluginexec/sandbox.go

@@ -74,7 +74,7 @@ func SetupSandbox(
 		for artifact := range ArtifactsForId(results, tool.GetId(), pluginv1.Artifact_TYPE_OUTPUT) {
 			listArtifact, err := SetupSandboxBinArtifact(ctx, artifact.GetArtifact(), binfs)
 			if err != nil {
-				return nil, fmt.Errorf("%v: %w", tref.Format(tool.GetRef()), err)
+				return nil, fmt.Errorf("%v: %w", tref.FormatOut(tool.GetRef()), err)
 			}
 			listArtifacts = append(listArtifacts, pluginv1.ArtifactWithOrigin_builder{
 				Artifact: listArtifact,