LastImportTime for resource. (grafana#112153)

pstibrany · web-flow · commit d801b87db9f4 · 2025-10-09T11:27:11.000+02:00
* LastImportTime for resource.

* Make StorageBackendImpl implement GetResourceLastImportTimes

* More missing implementations of GetResourceLastImportTimes

* Fix import.

* Skip TestGetResourceLastImportTime in TestBadgerKVStorageBackend.

* Implement GetResourceLastImportTimes by mockStorageBackend

* Bump test tolerance.

* Fix postgres query and timezone.

* Fix postgres query and timezone.

* Make linter happy.
diff --git a/pkg/registry/apis/dashboard/legacy/storage.go b/pkg/registry/apis/dashboard/legacy/storage.go
@@ -242,6 +242,12 @@ func (a *dashboardSqlAccess) ListModifiedSince(ctx context.Context, key resource
 	}
 }
 
+func (a *dashboardSqlAccess) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[resource.ResourceLastImportTime, error] {
+	return func(yield func(resource.ResourceLastImportTime, error) bool) {
+		yield(resource.ResourceLastImportTime{}, errors.New("not implemented"))
+	}
+}
+
 // List implements StorageBackend.
 func (a *dashboardSqlAccess) ListIterator(ctx context.Context, req *resourcepb.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
 	if req.ResourceVersion != 0 {
diff --git a/pkg/registry/apis/iam/noopstorage/storage_backend.go b/pkg/registry/apis/iam/noopstorage/storage_backend.go
@@ -61,3 +61,9 @@ func (c *StorageBackendImpl) WatchWriteEvents(ctx context.Context) (<-chan *reso
 func (c *StorageBackendImpl) WriteEvent(context.Context, resource.WriteEvent) (int64, error) {
 	return 0, errNoopStorage
 }
+
+func (c *StorageBackendImpl) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[resource.ResourceLastImportTime, error] {
+	return func(yield func(resource.ResourceLastImportTime, error) bool) {
+		yield(resource.ResourceLastImportTime{}, errNoopStorage)
+	}
+}
diff --git a/pkg/registry/apis/iam/resourcepermission/storage_backend.go b/pkg/registry/apis/iam/resourcepermission/storage_backend.go
@@ -13,6 +13,7 @@ import (
 	"k8s.io/apiserver/pkg/endpoints/request"
 
 	"github.com/grafana/authlib/types"
+
 	"github.com/grafana/grafana/apps/iam/pkg/apis/iam/v0alpha1"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/registry/apis/iam/common"
@@ -295,3 +296,9 @@ func (s *ResourcePermSqlBackend) WriteEvent(ctx context.Context, event resource.
 
 	return rv, err
 }
+
+func (s *ResourcePermSqlBackend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[resource.ResourceLastImportTime, error] {
+	return func(yield func(resource.ResourceLastImportTime, error) bool) {
+		yield(resource.ResourceLastImportTime{}, errNotImplemented)
+	}
+}
diff --git a/pkg/storage/unified/resource/bulk.go b/pkg/storage/unified/resource/bulk.go
@@ -33,12 +33,13 @@ func grpcMetaValueIsTrue(vals []string) bool {
 }
 
 type BulkRequestIterator interface {
+	// Next advances the iterator to the next element if one exists.
 	Next() bool
 
-	// The next event we should process
+	// Request returns the current element. Only valid after Next() returns true.
 	Request() *resourcepb.BulkRequest
 
-	// Rollback requested
+	// RollbackRequested returns true if there was an error advancing the iterator. Checked after Next() returns true.
 	RollbackRequested() bool
 }
 
diff --git a/pkg/storage/unified/resource/cdk_backend.go b/pkg/storage/unified/resource/cdk_backend.go
@@ -75,6 +75,12 @@ type cdkBackend struct {
 	stream      chan<- *WrittenEvent
 }
 
+func (s *cdkBackend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[ResourceLastImportTime, error] {
+	return func(yield func(ResourceLastImportTime, error) bool) {
+		yield(ResourceLastImportTime{}, errors.New("not implemented"))
+	}
+}
+
 func (s *cdkBackend) ListModifiedSince(ctx context.Context, key NamespacedResource, sinceRv int64) (int64, iter.Seq2[*ModifiedResource, error]) {
 	return 0, func(yield func(*ModifiedResource, error) bool) {
 		yield(nil, errors.New("not implemented"))
diff --git a/pkg/storage/unified/resource/search_test.go b/pkg/storage/unified/resource/search_test.go
@@ -129,6 +129,12 @@ func (m *mockStorageBackend) ListModifiedSince(ctx context.Context, key Namespac
 	}
 }
 
+func (m *mockStorageBackend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[ResourceLastImportTime, error] {
+	return func(yield func(ResourceLastImportTime, error) bool) {
+		yield(ResourceLastImportTime{}, errors.New("not implemented"))
+	}
+}
+
 // mockSearchBackend implements SearchBackend for testing with tracking capabilities
 type mockSearchBackend struct {
 	openIndexes []NamespacedResource
diff --git a/pkg/storage/unified/resource/server.go b/pkg/storage/unified/resource/server.go
@@ -86,6 +86,11 @@ type BackendReadResponse struct {
 	Error *resourcepb.ErrorResult
 }
 
+type ResourceLastImportTime struct {
+	NamespacedResource
+	LastImportTime time.Time
+}
+
 // The StorageBackend is an internal abstraction that supports interacting with
 // the underlying raw storage medium.  This interface is never exposed directly,
 // it is provided by concrete instances that actually write values.
@@ -118,6 +123,9 @@ type StorageBackend interface {
 
 	// Get resource stats within the storage backend.  When namespace is empty, it will apply to all
 	GetResourceStats(ctx context.Context, namespace string, minCount int) ([]ResourceStats, error)
+
+	// GetResourceLastImportTimes returns import times for all namespaced resources in the backend.
+	GetResourceLastImportTimes(ctx context.Context) iter.Seq2[ResourceLastImportTime, error]
 }
 
 type ModifiedResource struct {
diff --git a/pkg/storage/unified/resource/storage_backend.go b/pkg/storage/unified/resource/storage_backend.go
@@ -15,12 +15,13 @@ import (
 
 	"github.com/bwmarrin/snowflake"
 	"github.com/grafana/grafana-app-sdk/logging"
-	"github.com/grafana/grafana/pkg/apimachinery/utils"
-	"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
-	"github.com/grafana/grafana/pkg/util/debouncer"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/trace"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/grafana/grafana/pkg/apimachinery/utils"
+	"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
+	"github.com/grafana/grafana/pkg/util/debouncer"
 )
 
 const (
@@ -1076,6 +1077,12 @@ func (k *kvStorageBackend) GetResourceStats(ctx context.Context, namespace strin
 	return k.dataStore.GetResourceStats(ctx, namespace, minCount)
 }
 
+func (k *kvStorageBackend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[ResourceLastImportTime, error] {
+	return func(yield func(ResourceLastImportTime, error) bool) {
+		yield(ResourceLastImportTime{}, fmt.Errorf("not implemented"))
+	}
+}
+
 // readAndClose reads all data from a ReadCloser and ensures it's closed,
 // combining any errors from both operations.
 func readAndClose(r io.ReadCloser) ([]byte, error) {
diff --git a/pkg/storage/unified/search/bleve_test.go b/pkg/storage/unified/search/bleve_test.go
@@ -1489,8 +1489,8 @@ func TestConcurrentIndexUpdateAndSearchWithIndexMinUpdateInterval(t *testing.T)
 					if rvDiff == 0 {
 						// OK
 					} else {
-						// Allow returned RV to be within 10% of minInterval.
-						require.InDelta(t, minInterval.Milliseconds(), rvDiff, float64(minInterval.Milliseconds())*0.10)
+						// Allow returned RV to be within 20% of minInterval (to account for slow CI machines).
+						require.InDelta(t, minInterval.Milliseconds(), rvDiff, float64(minInterval.Milliseconds())*0.20)
 					}
 				}
 
diff --git a/pkg/storage/unified/sql/backend.go b/pkg/storage/unified/sql/backend.go
@@ -955,3 +955,51 @@ func (b *backend) fetchLatestHistoryRV(ctx context.Context, x db.ContextExecer,
 	}
 	return res.ResourceVersion, nil
 }
+
+func (b *backend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[resource.ResourceLastImportTime, error] {
+	ctx, span := b.tracer.Start(ctx, tracePrefix+"GetLastImportTimes")
+	defer span.End()
+
+	rows, err := dbutil.QueryRows(ctx, b.db, sqlResourceLastImportTimeQuery, &sqlResourceLastImportTimeQueryRequest{SQLTemplate: sqltemplate.New(b.dialect)})
+	if err != nil {
+		return func(yield func(resource.ResourceLastImportTime, error) bool) {
+			yield(resource.ResourceLastImportTime{}, err)
+		}
+	}
+
+	return func(yield func(resource.ResourceLastImportTime, error) bool) {
+		closeOnDefer := true
+		defer func() {
+			if closeOnDefer {
+				_ = rows.Close() // Close while ignoring errors.
+			}
+		}()
+
+		for rows.Next() {
+			// If context has finished, return early.
+			if ctx.Err() != nil {
+				yield(resource.ResourceLastImportTime{}, ctx.Err())
+				return
+			}
+
+			row := resource.ResourceLastImportTime{}
+			err = rows.Scan(&row.Namespace, &row.Group, &row.Resource, &row.LastImportTime)
+			if err != nil {
+				yield(resource.ResourceLastImportTime{}, err)
+				return
+			}
+
+			if !yield(row, nil) {
+				return
+			}
+		}
+
+		closeOnDefer = false
+
+		// Close and report error, if any.
+		err := rows.Close()
+		if err != nil {
+			yield(resource.ResourceLastImportTime{}, err)
+		}
+	}
+}
diff --git a/pkg/storage/unified/sql/bulk.go b/pkg/storage/unified/sql/bulk.go
@@ -170,7 +170,7 @@ func (b *backend) processBulk(ctx context.Context, setting resource.BulkSettings
 		// Calculate the RV based on incoming request timestamps
 		rv := newBulkRV()
 
-		summaries := make(map[string]*resourcepb.BulkResponse_Summary, len(setting.Collection)*4)
+		summaries := make(map[string]*resourcepb.BulkResponse_Summary, len(setting.Collection))
 
 		// First clear everything in the transaction
 		if setting.RebuildCollection {
@@ -182,6 +182,14 @@ func (b *backend) processBulk(ctx context.Context, setting resource.BulkSettings
 				summaries[resource.NSGR(key)] = summary
 				rsp.Summary = append(rsp.Summary, summary)
 			}
+		} else {
+			for _, key := range setting.Collection {
+				summaries[resource.NSGR(key)] = &resourcepb.BulkResponse_Summary{
+					Namespace: key.Namespace,
+					Group:     key.Group,
+					Resource:  key.Resource,
+				}
+			}
 		}
 
 		obj := &unstructured.Unstructured{}
@@ -253,6 +261,12 @@ func (b *backend) processBulk(ctx context.Context, setting resource.BulkSettings
 			if err != nil {
 				b.log.Warn("error increasing RV", "error", err)
 			}
+
+			// Update the last import time. This is important to trigger reindexing
+			// of the resource for a given namespace.
+			if err := b.updateLastImportTime(ctx, tx, key, time.Now()); err != nil {
+				return rollbackWithError(err)
+			}
 		}
 		return nil
 	})
@@ -262,6 +276,19 @@ func (b *backend) processBulk(ctx context.Context, setting resource.BulkSettings
 	return rsp
 }
 
+func (b *backend) updateLastImportTime(ctx context.Context, tx db.Tx, key *resourcepb.ResourceKey, now time.Time) error {
+	if _, err := dbutil.Exec(ctx, tx, sqlResourceLastImportTimeInsert, sqlResourceLastImportTimeInsertRequest{
+		SQLTemplate:    sqltemplate.New(b.dialect),
+		Namespace:      key.Namespace,
+		Group:          key.Group,
+		Resource:       key.Resource,
+		LastImportTime: now.UTC(),
+	}); err != nil {
+		return fmt.Errorf("insert resource last import time: %w", err)
+	}
+	return nil
+}
+
 type bulkWroker struct {
 	ctx     context.Context
 	tx      db.ContextExecer
diff --git a/pkg/storage/unified/sql/data/resource_last_import_time_insert.sql b/pkg/storage/unified/sql/data/resource_last_import_time_insert.sql
@@ -0,0 +1,42 @@
+{{ if eq $.DialectName "mysql" }}
+  INSERT INTO {{ .Ident "resource_last_import_time" }} (
+      {{ .Ident "group" }},
+      {{ .Ident "resource" }},
+      {{ .Ident "namespace" }},
+      {{ .Ident "last_import_time" }}
+    ) VALUES (
+      {{ .Arg .Group }},
+      {{ .Arg .Resource }},
+      {{ .Arg .Namespace }},
+      {{ .Arg .LastImportTime }}
+    ) ON DUPLICATE KEY UPDATE {{ .Ident "last_import_time" }} = {{ .Arg .LastImportTime }}
+
+{{ else if eq $.DialectName "sqlite" }}
+   INSERT OR REPLACE INTO {{ .Ident "resource_last_import_time" }} (
+       {{ .Ident "group" }},
+       {{ .Ident "resource" }},
+       {{ .Ident "namespace" }},
+       {{ .Ident "last_import_time" }}
+   ) VALUES (
+       {{ .Arg .Group }},
+       {{ .Arg .Resource }},
+       {{ .Arg .Namespace }},
+       {{ .Arg .LastImportTime }}
+   )
+
+{{ else if eq $.DialectName "postgres" }}
+    INSERT INTO {{ .Ident "resource_last_import_time" }} (
+       {{ .Ident "group" }},
+       {{ .Ident "resource" }},
+       {{ .Ident "namespace" }},
+       {{ .Ident "last_import_time" }}
+     ) VALUES (
+       {{ .Arg .Group }},
+       {{ .Arg .Resource }},
+       {{ .Arg .Namespace }},
+       {{ .Arg .LastImportTime }}
+    ) ON CONFLICT ({{ .Ident "group" }}, {{ .Ident "resource" }}, {{ .Ident "namespace" }})
+      DO UPDATE SET {{ .Ident "last_import_time" }} = {{ .Arg .LastImportTime }}
+
+{{ end }}
+;
diff --git a/pkg/storage/unified/sql/data/resource_last_import_time_query.sql b/pkg/storage/unified/sql/data/resource_last_import_time_query.sql
@@ -0,0 +1,8 @@
+SELECT
+  {{ .Ident "namespace" }},
+  {{ .Ident "group" }},
+  {{ .Ident "resource" }},
+  {{ .Ident "last_import_time" }}
+FROM
+  {{ .Ident "resource_last_import_time" }}
+;
diff --git a/pkg/storage/unified/sql/db/migrations/resource_mig.go b/pkg/storage/unified/sql/db/migrations/resource_mig.go
@@ -116,6 +116,17 @@ func initResourceTables(mg *migrator.Migrator) string {
 		},
 	})
 
+	tables = append(tables, migrator.Table{
+		Name: "resource_last_import_time",
+		Columns: []*migrator.Column{
+			{Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
+			{Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
+			{Name: "namespace", Type: migrator.DB_NVarchar, Length: 63, Nullable: false},
+			{Name: "last_import_time", Type: migrator.DB_DateTime, Nullable: false},
+		},
+		PrimaryKeys: []string{"group", "resource", "namespace"},
+	})
+
 	// Initialize all tables
 	for t := range tables {
 		mg.AddMigration("drop table "+tables[t].Name, migrator.NewDropTableMigration(tables[t].Name))
diff --git a/pkg/storage/unified/sql/queries.go b/pkg/storage/unified/sql/queries.go
@@ -58,6 +58,9 @@ var (
 
 	sqlResourceBlobInsert = mustTemplate("resource_blob_insert.sql")
 	sqlResourceBlobQuery  = mustTemplate("resource_blob_query.sql")
+
+	sqlResourceLastImportTimeInsert = mustTemplate("resource_last_import_time_insert.sql")
+	sqlResourceLastImportTimeQuery  = mustTemplate("resource_last_import_time_query.sql")
 )
 
 // TxOptions.
@@ -454,3 +457,35 @@ func (r sqlResourceListModifiedSinceRequest) Validate() error {
 	}
 	return nil
 }
+
+type sqlResourceLastImportTimeInsertRequest struct {
+	sqltemplate.SQLTemplate
+	Namespace      string
+	Group          string
+	Resource       string
+	LastImportTime time.Time
+}
+
+func (r sqlResourceLastImportTimeInsertRequest) Validate() error {
+	if r.Namespace == "" {
+		return fmt.Errorf("missing namespace")
+	}
+	if r.Group == "" {
+		return fmt.Errorf("missing group")
+	}
+	if r.Resource == "" {
+		return fmt.Errorf("missing resource")
+	}
+	if r.LastImportTime.IsZero() {
+		return fmt.Errorf("last import time cannot be zero")
+	}
+	return nil
+}
+
+type sqlResourceLastImportTimeQueryRequest struct {
+	sqltemplate.SQLTemplate
+}
+
+func (r *sqlResourceLastImportTimeQueryRequest) Validate() error {
+	return nil
+}
diff --git a/pkg/storage/unified/sql/queries_test.go b/pkg/storage/unified/sql/queries_test.go
@@ -495,5 +495,25 @@ func TestUnifiedStorageQueries(t *testing.T) {
 					},
 				},
 			},
+			sqlResourceLastImportTimeInsert: {
+				{
+					Name: "insert",
+					Data: &sqlResourceLastImportTimeInsertRequest{
+						SQLTemplate:    mocks.NewTestingSQLTemplate(),
+						Namespace:      "ns",
+						Group:          "group",
+						Resource:       "res",
+						LastImportTime: time.Date(2025, 10, 07, 22, 30, 05, 0, time.UTC),
+					},
+				},
+			},
+			sqlResourceLastImportTimeQuery: {
+				{
+					Name: "insert",
+					Data: &sqlResourceLastImportTimeQueryRequest{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+					},
+				},
+			},
 		}})
 }
diff --git a/pkg/storage/unified/sql/testdata/mysql--resource_last_import_time_insert-insert.sql b/pkg/storage/unified/sql/testdata/mysql--resource_last_import_time_insert-insert.sql
diff --git a/pkg/storage/unified/sql/testdata/mysql--resource_last_import_time_query-insert.sql b/pkg/storage/unified/sql/testdata/mysql--resource_last_import_time_query-insert.sql
diff --git a/pkg/storage/unified/sql/testdata/postgres--resource_last_import_time_insert-insert.sql b/pkg/storage/unified/sql/testdata/postgres--resource_last_import_time_insert-insert.sql
diff --git a/pkg/storage/unified/sql/testdata/postgres--resource_last_import_time_query-insert.sql b/pkg/storage/unified/sql/testdata/postgres--resource_last_import_time_query-insert.sql
diff --git a/pkg/storage/unified/sql/testdata/sqlite--resource_last_import_time_insert-insert.sql b/pkg/storage/unified/sql/testdata/sqlite--resource_last_import_time_insert-insert.sql
diff --git a/pkg/storage/unified/sql/testdata/sqlite--resource_last_import_time_query-insert.sql b/pkg/storage/unified/sql/testdata/sqlite--resource_last_import_time_query-insert.sql
diff --git a/pkg/storage/unified/testing/storage_backend.go b/pkg/storage/unified/testing/storage_backend.go
diff --git a/pkg/storage/unified/testing/storage_backend_test.go b/pkg/storage/unified/testing/storage_backend_test.go

Original file line number	Diff line number	Diff line change
`@@ -242,6 +242,12 @@ func (a *dashboardSqlAccess) ListModifiedSince(ctx context.Context, key resource`
`242`	`242`	`}`
`243`	`243`	`}`
`244`	`244`
	`245`	`+func (a *dashboardSqlAccess) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[resource.ResourceLastImportTime, error] {`
	`246`	`+ return func(yield func(resource.ResourceLastImportTime, error) bool) {`
	`247`	`+ yield(resource.ResourceLastImportTime{}, errors.New("not implemented"))`
	`248`	`+ }`
	`249`	`+}`
	`250`	`+`
`245`	`251`	`// List implements StorageBackend.`
`246`	`252`	`func (a dashboardSqlAccess) ListIterator(ctx context.Context, req resourcepb.ListRequest, cb func(resource.ListIterator) error) (int64, error) {`
`247`	`253`	`if req.ResourceVersion != 0 {`
Original file line number	Diff line number	Diff line change
`@@ -33,12 +33,13 @@ func grpcMetaValueIsTrue(vals []string) bool {`
`33`	`33`	`}`
`34`	`34`
`35`	`35`	`type BulkRequestIterator interface {`
	`36`	`+ // Next advances the iterator to the next element if one exists.`
`36`	`37`	`Next() bool`
`37`	`38`
`38`		`- // The next event we should process`
	`39`	`+ // Request returns the current element. Only valid after Next() returns true.`
`39`	`40`	`Request() *resourcepb.BulkRequest`
`40`	`41`
`41`		`- // Rollback requested`
	`42`	`+ // RollbackRequested returns true if there was an error advancing the iterator. Checked after Next() returns true.`
`42`	`43`	`RollbackRequested() bool`
`43`	`44`	`}`
`44`	`45`
Original file line number	Diff line number	Diff line change
`@@ -129,6 +129,12 @@ func (m *mockStorageBackend) ListModifiedSince(ctx context.Context, key Namespac`
`129`	`129`	`}`
`130`	`130`	`}`
`131`	`131`
	`132`	`+func (m *mockStorageBackend) GetResourceLastImportTimes(ctx context.Context) iter.Seq2[ResourceLastImportTime, error] {`
	`133`	`+ return func(yield func(ResourceLastImportTime, error) bool) {`
	`134`	`+ yield(ResourceLastImportTime{}, errors.New("not implemented"))`
	`135`	`+ }`
	`136`	`+}`
	`137`	`+`
`132`	`138`	`// mockSearchBackend implements SearchBackend for testing with tracking capabilities`
`133`	`139`	`type mockSearchBackend struct {`
`134`	`140`	`openIndexes []NamespacedResource`
Original file line number	Diff line number	Diff line change
`@@ -1489,8 +1489,8 @@ func TestConcurrentIndexUpdateAndSearchWithIndexMinUpdateInterval(t *testing.T)`
`1489`	`1489`	`if rvDiff == 0 {`
`1490`	`1490`	`// OK`
`1491`	`1491`	`} else {`
`1492`		`- // Allow returned RV to be within 10% of minInterval.`
`1493`		`- require.InDelta(t, minInterval.Milliseconds(), rvDiff, float64(minInterval.Milliseconds())*0.10)`
	`1492`	`+ // Allow returned RV to be within 20% of minInterval (to account for slow CI machines).`
	`1493`	`+ require.InDelta(t, minInterval.Milliseconds(), rvDiff, float64(minInterval.Milliseconds())*0.20)`
`1494`	`1494`	`}`
`1495`	`1495`	`}`
`1496`	`1496`