Only invoke a given cached partial once

Note that this is backed by a LRU cache (which we soon shall see more usage of), so if you're a heavy user of cached partials it may be evicted and
refreshed if needed. But in most cases every partial is only invoked once.

This commit also adds a timeout (the global `timeout` config option) to make infinite recursion in partials
easier to reason about.

```
name              old time/op    new time/op    delta
IncludeCached-10    8.92ms ± 0%    8.48ms ± 1%   -4.87%  (p=0.016 n=4+5)

name              old alloc/op   new alloc/op   delta
IncludeCached-10    6.65MB ± 0%    5.17MB ± 0%  -22.32%  (p=0.002 n=6+6)

name              old allocs/op  new allocs/op  delta
IncludeCached-10      117k ± 0%       71k ± 0%  -39.44%  (p=0.002 n=6+6)
```

Closes gohugoio#4086
Updates gohugoio#9588

Author: bep

deps/deps.go

@@ -11,6 +11,7 @@ import (
 	"github.com/gohugoio/hugo/cache/filecache"
 	"github.com/gohugoio/hugo/common/hexec"
 	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/config/security"
 	"github.com/gohugoio/hugo/helpers"
@@ -298,11 +299,14 @@ func New(cfg DepsCfg) (*Deps, error) {
 
 	sp := source.NewSourceSpec(ps, nil, fs.Source)
 
-	timeoutms := cfg.Language.GetInt("timeout")
-	if timeoutms <= 0 {
-		timeoutms = 3000
+	timeout := 30 * time.Second
+	if cfg.Cfg.IsSet("timeout") {
+		v := cfg.Cfg.Get("timeout")
+		d, err := types.ToDurationE(v)
+		if err == nil {
+			timeout = d
+		}
 	}
-
 	ignoreErrors := cast.ToStringSlice(cfg.Cfg.Get("ignoreErrors"))
 	ignorableLogger := loggers.NewIgnorableLogger(logger, ignoreErrors...)
 
@@ -329,7 +333,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 		BuildClosers:            &Closers{},
 		BuildState:              buildState,
 		Running:                 cfg.Running,
-		Timeout:                 time.Duration(timeoutms) * time.Millisecond,
+		Timeout:                 timeout,
 		globalErrHandler:        errorHandler,
 	}
 

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/bep/godartsass v0.16.0
 	github.com/bep/golibsass v1.1.0
 	github.com/bep/gowebp v0.2.0
+	github.com/bep/lazycache v0.2.0
 	github.com/bep/overlayfs v0.6.0
 	github.com/bep/tmc v0.5.1
 	github.com/clbanning/mxj/v2 v2.5.7
@@ -103,6 +104,7 @@ require (
 	github.com/google/wire v0.5.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.3.0 // indirect
 	github.com/googleapis/go-type-adapters v1.0.0 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/invopop/yaml v0.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect

go.sum

@@ -179,6 +179,8 @@ github.com/bep/golibsass v1.1.0 h1:pjtXr00IJZZaOdfryNa9wARTB3Q0BmxC3/V1KNcgyTw=
 github.com/bep/golibsass v1.1.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA=
 github.com/bep/gowebp v0.2.0 h1:ZVfK8i9PpZqKHEmthQSt3qCnnHycbLzBPEsVtk2ch2Q=
 github.com/bep/gowebp v0.2.0/go.mod h1:ZhFodwdiFp8ehGJpF4LdPl6unxZm9lLFjxD3z2h2AgI=
+github.com/bep/lazycache v0.2.0 h1:HKrlZTrDxHIrNKqmnurH42ryxkngCMYLfBpyu40VcwY=
+github.com/bep/lazycache v0.2.0/go.mod h1:xUIsoRD824Vx0Q/n57+ZO7kmbEhMBOnTjM/iPixNGbg=
 github.com/bep/overlayfs v0.6.0 h1:sgLcq/qtIzbaQNl2TldGXOkHvqeZB025sPvHOQL+DYo=
 github.com/bep/overlayfs v0.6.0/go.mod h1:NFjSmn3kCqG7KX2Lmz8qT8VhPPCwZap3UNogXawoQHM=
 github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI=
@@ -238,8 +240,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanw/esbuild v0.15.18 h1:CM7eAoUjjNkZs1LH0p6fkwtADrbFr4JV2SlT1bUMjEo=
-github.com/evanw/esbuild v0.15.18/go.mod h1:iINY06rn799hi48UqEnaQvVfZWe6W9bET78LbvN8VWk=
 github.com/evanw/esbuild v0.17.0 h1:gGx9TCZDO9k9x1PJdizx6syIpUq29RwrtHWlgDIdQH8=
 github.com/evanw/esbuild v0.17.0/go.mod h1:iINY06rn799hi48UqEnaQvVfZWe6W9bET78LbvN8VWk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
@@ -401,6 +401,8 @@ github.com/hairyhenderson/go-codeowners v0.2.3-0.20201026200250-cdc7c0759690 h1:
 github.com/hairyhenderson/go-codeowners v0.2.3-0.20201026200250-cdc7c0759690/go.mod h1:8Qu9UmnhCRunfRv365Z3w+mT/WfLGKJiK+vugY9qNCU=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
+github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=

helpers/general.go

@@ -23,16 +23,13 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"unicode"
 	"unicode/utf8"
 
 	"github.com/gohugoio/hugo/common/loggers"
 
-	"github.com/mitchellh/hashstructure"
-
 	"github.com/gohugoio/hugo/common/hugo"
 
 	"github.com/spf13/afero"
@@ -219,7 +216,7 @@ func ReaderContains(r io.Reader, subslice []byte) bool {
 // GetTitleFunc returns a func that can be used to transform a string to
 // title case.
 //
-// The supported styles are
+// # The supported styles are
 //
 // - "Go" (strings.Title)
 // - "AP" (see https://www.apstylebook.com/)
@@ -523,20 +520,3 @@ func PrintFs(fs afero.Fs, path string, w io.Writer) {
 		return nil
 	})
 }
-
-// HashString returns a hash from the given elements.
-// It will panic if the hash cannot be calculated.
-func HashString(elements ...any) string {
-	var o any
-	if len(elements) == 1 {
-		o = elements[0]
-	} else {
-		o = elements
-	}
-
-	hash, err := hashstructure.Hash(o, nil)
-	if err != nil {
-		panic(err)
-	}
-	return strconv.FormatUint(hash, 10)
-}

helpers/general_test.go

@@ -458,10 +458,3 @@ func BenchmarkUniqueStrings(b *testing.B) {
 		}
 	})
 }
-
-func TestHashString(t *testing.T) {
-	c := qt.New(t)
-
-	c.Assert(HashString("a", "b"), qt.Equals, "2712570657419664240")
-	c.Assert(HashString("ab"), qt.Equals, "590647783936702392")
-}

hugolib/page_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/bep/clock"
 	"github.com/gohugoio/hugo/htesting"
+	"github.com/gohugoio/hugo/identity"
 	"github.com/gohugoio/hugo/markup/asciidocext"
 	"github.com/gohugoio/hugo/markup/rst"
 	"github.com/gohugoio/hugo/tpl"
@@ -2001,3 +2002,37 @@ Page1: {{ $p1.Path }}
 
 	b.AssertFileContent("public/index.html", "Lang: no", filepath.FromSlash("Page1: a/B/C/Page1.md"))
 }
+
+func TestPageHashString(t *testing.T) {
+	files := `
+-- config.toml --
+baseURL = "https://example.org"
+[languages]
+[languages.en]
+weight = 1
+title = "English"
+[languages.no]
+weight = 2
+title = "Norsk"
+-- content/p1.md --
+---
+title: "p1"
+---
+-- content/p2.md --
+---
+title: "p2"
+---
+`
+
+	b := NewIntegrationTestBuilder(IntegrationTestConfig{
+		T:           t,
+		TxtarString: files,
+	}).Build()
+
+	p1 := b.H.Sites[0].RegularPages()[0]
+	p2 := b.H.Sites[0].RegularPages()[1]
+	sites := p1.Sites()
+
+	b.Assert(identity.HashString(p1), qt.Not(qt.Equals), identity.HashString(p2))
+	b.Assert(identity.HashString(sites[0]), qt.Not(qt.Equals), identity.HashString(sites[1]))
+}

hugolib/resource_chain_test.go

@@ -26,11 +26,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gohugoio/hugo/helpers"
-
 	qt "github.com/frankban/quicktest"
 
 	"github.com/gohugoio/hugo/common/loggers"
+	"github.com/gohugoio/hugo/identity"
 	"github.com/gohugoio/hugo/resources/resource_transformers/tocss/scss"
 )
 
@@ -122,7 +121,7 @@ FAILED REMOTE ERROR DETAILS CONTENT: |failed to fetch remote resource: Internal
 |StatusCode: 500|ContentLength: 16|ContentType: text/plain; charset=utf-8|
 
 
-`, helpers.HashString(ts.URL+"/sunset.jpg", map[string]any{})))
+`, identity.HashString(ts.URL+"/sunset.jpg", map[string]any{})))
 
 		b.AssertFileContent("public/styles.min.a1df58687c3c9cc38bf26532f7b4b2f2c2b0315dcde212376959995c04f11fef.css", "body{background-color:#add8e6}")
 		b.AssertFileContent("public//styles2.min.1cfc52986836405d37f9998a63fd6dd8608e8c410e5e3db1daaa30f78bc273ba.css", "body{background-color:orange}")

hugolib/site.go

@@ -778,6 +778,10 @@ func (s *SiteInfo) DisqusShortname() string {
 	return s.Config().Services.Disqus.Shortname
 }
 
+func (s *SiteInfo) GetIdentity() identity.Identity {
+	return identity.KeyValueIdentity{Key: "site", Value: s.language.Lang}
+}
+
 // SiteSocial is a place to put social details on a site level. These are the
 // standard keys that themes will expect to have available, but can be
 // expanded to any others on a per site basis

identity/identity.go

@@ -1,3 +1,16 @@
+// Copyright 2023 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package identity
 
 import (
@@ -107,7 +120,7 @@ func (id KeyValueIdentity) Name() string {
 	return id.Key
 }
 
-// Provider provides the hashable Identity.
+// Provider provides the comparable Identity.
 type Provider interface {
 	// GetIdentity is for internal use.
 	GetIdentity() Identity

identity/identityhash.go

@@ -0,0 +1,69 @@
+// Copyright 2023 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package identity
+
+import (
+	"strconv"
+
+	"github.com/mitchellh/hashstructure"
+)
+
+// HashString returns a hash from the given elements.
+// It will panic if the hash cannot be calculated.
+// Note that this hash should be used primarily for identity, not for change detection as
+// it in the more complex values (e.g. Page) will not hash the full content.
+func HashString(vs ...any) string {
+	hash := HashUint64(vs...)
+	return strconv.FormatUint(hash, 10)
+}
+
+// HashUint64 returns a hash from the given elements.
+// It will panic if the hash cannot be calculated.
+// Note that this hash should be used primarily for identity, not for change detection as
+// it in the more complex values (e.g. Page) will not hash the full content.
+func HashUint64(vs ...any) uint64 {
+	var o any
+	if len(vs) == 1 {
+		o = toHashable(vs[0])
+	} else {
+		elements := make([]any, len(vs))
+		for i, e := range vs {
+			elements[i] = toHashable(e)
+		}
+		o = elements
+	}
+
+	hash, err := hashstructure.Hash(o, nil)
+	if err != nil {
+		panic(err)
+	}
+	return hash
+}
+
+type keyer interface {
+	Key() string
+}
+
+// For structs, hashstructure.Hash only works on the exported fields,
+// so rewrite the input slice for known identity types.
+func toHashable(v any) any {
+	switch t := v.(type) {
+	case Provider:
+		return t.GetIdentity()
+	case keyer:
+		return t.Key()
+	default:
+		return v
+	}
+}

identity/identityhash_test.go

@@ -0,0 +1,45 @@
+// Copyright 2023 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package identity
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestHashString(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(HashString("a", "b"), qt.Equals, "2712570657419664240")
+	c.Assert(HashString("ab"), qt.Equals, "590647783936702392")
+
+	var vals []any = []any{"a", "b", tstKeyer{"c"}}
+
+	c.Assert(HashString(vals...), qt.Equals, "12599484872364427450")
+	c.Assert(vals[2], qt.Equals, tstKeyer{"c"})
+
+}
+
+type tstKeyer struct {
+	key string
+}
+
+func (t tstKeyer) Key() string {
+	return t.key
+}
+
+func (t tstKeyer) String() string {
+	return "key: " + t.key
+}

metrics/metrics.go

@@ -27,7 +27,7 @@ import (
 
 	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/compare"
-	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/identity"
 )
 
 // The Provider interface defines an interface for measuring metrics.
@@ -242,7 +242,7 @@ func howSimilar(a, b any) int {
 		return 90
 	}
 
-	h1, h2 := helpers.HashString(a), helpers.HashString(b)
+	h1, h2 := identity.HashString(a), identity.HashString(b)
 	if h1 == h2 {
 		return 100
 	}