K8s pod meta caching

fd/file.d.go

@@ -390,11 +390,11 @@ func (f *FileD) serveReady(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusServiceUnavailable)
 		return
 	}
-	logger.Infof("ready OK")
+	logger.Debug("ready OK")
 }
 
 func (f *FileD) serveLive(_ http.ResponseWriter, _ *http.Request) {
-	logger.Infof("live OK")
+	logger.Debug("live OK")
 }
 
 type valueChangerHandler struct {

go.mod

@@ -50,13 +50,13 @@ require (
 	go.uber.org/automaxprocs v1.5.3
 	go.uber.org/zap v1.25.0
 	golang.org/x/net v0.35.0
-	google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002
+	google.golang.org/protobuf v1.34.2
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.29.14
-	k8s.io/apimachinery v0.29.14
-	k8s.io/client-go v0.29.14
-	sigs.k8s.io/yaml v1.3.0
+	k8s.io/api v0.31.13
+	k8s.io/apimachinery v0.31.13
+	k8s.io/client-go v0.31.13
+	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
@@ -68,20 +68,21 @@ require (
 	github.com/cilium/ebpf v0.9.1 // indirect
 	github.com/containerd/cgroups/v3 v3.0.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dmarkham/enumer v1.5.8 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/go-faster/city v1.0.1 // indirect
 	github.com/go-faster/errors v0.6.1 // indirect
 	github.com/go-ini/ini v1.62.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
-	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/godbus/dbus/v5 v5.0.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
@@ -121,7 +122,7 @@ require (
 	github.com/opencontainers/runtime-spec v1.0.2 // indirect
 	github.com/pascaldekloe/name v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
@@ -132,6 +133,7 @@ require (
 	github.com/timtadh/data-structures v0.6.1 // indirect
 	github.com/twmb/franz-go/pkg/kmsg v1.8.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/yuin/gopher-lua v1.1.0 // indirect
@@ -141,19 +143,18 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.33.0 // indirect
 	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/oauth2 v0.10.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/sync v0.11.0 // indirect
 	golang.org/x/sys v0.30.0 // indirect
 	golang.org/x/term v0.29.0 // indirect
 	golang.org/x/text v0.22.0 // indirect
 	golang.org/x/time v0.10.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
-	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
-	k8s.io/klog/v2 v2.110.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

plugin/input/k8s/README.md

@@ -116,6 +116,16 @@ Example: ```component: '{{ index .pod.Labels "component" | default .k8s_containe
 
 <br>
 
+**`meta_file`** *`string`* 
+
+The filename to store current k8s pod meta. Meta are loaded only on initialization
+
+> This feature is used to reuse metadata that we have already received, in particular if the kube-apiserver is unavailable.
+>
+> The plugin considers kube-apiserver to be unavailable from the moment the SetWatchErrorHandler handler is called until one of the ResourceEventHandlerFuncs handlers is called.
+
+<br>
+
 
 ### Meta params
 **`pod_name`** - string

plugin/input/k8s/k8s.go

@@ -137,6 +137,15 @@ type Config struct {
 	// >
 	// > Example: ```component: '{{ index .pod.Labels "component" | default .k8s_container }}'```
 	K8sMeta cfg.MetaTemplates `json:"meta"` // *
+
+	// > @3@4@5@6
+	// >
+	// > The filename to store current k8s pod meta. Meta are loaded only on initialization
+	// >
+	// > > This feature is used to reuse metadata that we have already received, in particular if the kube-apiserver is unavailable.
+	// > >
+	// > > The plugin considers kube-apiserver to be unavailable from the moment the SetWatchErrorHandler handler is called until one of the ResourceEventHandlerFuncs handlers is called.
+	K8sMetaFile string `json:"meta_file" default:""` // *
 }
 
 var startCounter atomic.Int32
@@ -176,10 +185,11 @@ func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.InputPluginPa
 
 	if startCounter == 1 {
 		meta.DeletedPodsCacheSize = p.config.DeletedPodsCacheSize
+		meta.MetaFileSaver = meta.NewMetaSaver(p.config.K8sMetaFile)
 		meta.EnableGatherer(p.logger)
 	}
 
-	if meta.CriType == "docker" {
+	if meta.MetaData.CriType == "docker" {
 		p.params.Controller.SuggestDecoder(decoder.JSON)
 	} else {
 		p.params.Controller.SuggestDecoder(decoder.CRI)

plugin/input/k8s/k8s_test.go

@@ -242,5 +242,5 @@ func TestCleanUp(t *testing.T) {
 
 	meta.DisableGatherer()
 	p.Stop()
-	assert.Equal(t, 0, len(meta.MetaData))
+	assert.Equal(t, 0, len(meta.MetaData.PodMeta))
 }