From 3832d0c1520a54ab66a97f6c619e93d372388733 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Tue, 11 Feb 2025 12:07:01 -0500 Subject: [PATCH 01/22] [usm] add regular and raw tracepoints /sched_process_exit --- pkg/network/ebpf/c/protocols/flush.h | 22 ++++++++++++++++++++++ pkg/network/usm/ebpf_main.go | 24 ++++++++++++++++++++++++ pkg/network/usm/sharedlibraries/ebpf.go | 8 ++++++++ 3 files changed, 54 insertions(+) diff --git a/pkg/network/ebpf/c/protocols/flush.h b/pkg/network/ebpf/c/protocols/flush.h index cf040a66dc83f5..2c0744e5265092 100644 --- a/pkg/network/ebpf/c/protocols/flush.h +++ b/pkg/network/ebpf/c/protocols/flush.h @@ -28,4 +28,26 @@ int tracepoint__net__netif_receive_skb(void *ctx) { return 0; } +SEC("tracepoint/sched/sched_process_exit") +int tracepoint__sched__sched_process_exit(void *ctx) { + CHECK_BPF_PROGRAM_BYPASSED() + u64 pid_tgid = bpf_get_current_pid_tgid(); + + bpf_map_delete_elem(&ssl_read_args, &pid_tgid); + bpf_map_delete_elem(&ssl_read_ex_args, &pid_tgid); + + return 0; +} + +SEC("raw_tracepoint/sched_process_exit") +int raw_tracepoint__sched_process_exit(void *ctx) { + CHECK_BPF_PROGRAM_BYPASSED() + u64 pid_tgid = bpf_get_current_pid_tgid(); + + bpf_map_delete_elem(&ssl_read_args, &pid_tgid); + bpf_map_delete_elem(&ssl_read_ex_args, &pid_tgid); + + return 0; +} + #endif diff --git a/pkg/network/usm/ebpf_main.go b/pkg/network/usm/ebpf_main.go index e44ae89e8868ac..fdc22e3e1f9250 100644 --- a/pkg/network/usm/ebpf_main.go +++ b/pkg/network/usm/ebpf_main.go @@ -14,6 +14,7 @@ import ( "slices" "unsafe" + "github.com/DataDog/datadog-agent/pkg/util/kernel" manager "github.com/DataDog/ebpf-manager" "github.com/cilium/ebpf" "github.com/davecgh/go-spew/spew" @@ -149,6 +150,29 @@ func newEBPFProgram(c *config.Config, connectionProtocolMap *ebpf.Map) (*ebpfPro } } + if kversion, err := kernel.HostVersion(); err == nil && kversion >= kernel.VersionCode(4, 17, 0) { + // use a raw tracepoint on a supported kernel to intercept terminated threads and clear the corresponding maps. + mgr.Probes = append(mgr.Probes, []*manager.Probe{ + { + ProbeIdentificationPair: manager.ProbeIdentificationPair{ + EBPFFuncName: "raw_tracepoint__sched_process_exit", + UID: probeUID, + }, + TracepointName: "sched_process_exit", + }, + }...) + } else { + // use a regular tracepoint to intercept terminated threads. + mgr.Probes = append(mgr.Probes, []*manager.Probe{ + { + ProbeIdentificationPair: manager.ProbeIdentificationPair{ + EBPFFuncName: "tracepoint__sched__sched_process_exit", + UID: probeUID, + }, + }, + }...) + } + program := &ebpfProgram{ Manager: ddebpf.NewManager(mgr, "usm", &ebpftelemetry.ErrorsTelemetryModifier{}), cfg: c, diff --git a/pkg/network/usm/sharedlibraries/ebpf.go b/pkg/network/usm/sharedlibraries/ebpf.go index 6f6cf41d2654b8..4dd9afd8c3eee8 100644 --- a/pkg/network/usm/sharedlibraries/ebpf.go +++ b/pkg/network/usm/sharedlibraries/ebpf.go @@ -631,6 +631,14 @@ func (e *EbpfProgram) initializeProbes() { e.enabledProbes = tpProbes e.disabledProbes = tracingProbes } + + if kversion, err := kernel.HostVersion(); err == nil && kversion < kernel.VersionCode(4, 17, 0) { + // do not use a raw tracepoint on an unsupported kernel. + e.disabledProbes = append(e.disabledProbes, manager.ProbeIdentificationPair{ + EBPFFuncName: "raw_tracepoint__sched_process_exit", + UID: probeUID, + }) + } } func getAssetName(module string, debug bool) string { From d9e1a320b46abc267810a32deb62ad19cc75d1b9 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Wed, 12 Feb 2025 13:53:19 -0500 Subject: [PATCH 02/22] [usm] #if PREBUILT or CORE or (RUNTIME and kernel>4, 17, 0): raw_tracepoint__sched_process_exit() --- pkg/network/ebpf/c/protocols/flush.h | 3 +++ pkg/network/usm/ebpf_main.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/network/ebpf/c/protocols/flush.h b/pkg/network/ebpf/c/protocols/flush.h index 2c0744e5265092..3654854918560a 100644 --- a/pkg/network/ebpf/c/protocols/flush.h +++ b/pkg/network/ebpf/c/protocols/flush.h @@ -39,6 +39,8 @@ int tracepoint__sched__sched_process_exit(void *ctx) { return 0; } +#if defined(COMPILE_PREBUILT) || defined(COMPILE_CORE) || (defined(COMPILE_RUNTIME) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)) +// add this program when compiling CO-RE or runtime compilation for supported kernel versions. SEC("raw_tracepoint/sched_process_exit") int raw_tracepoint__sched_process_exit(void *ctx) { CHECK_BPF_PROGRAM_BYPASSED() @@ -49,5 +51,6 @@ int raw_tracepoint__sched_process_exit(void *ctx) { return 0; } +#endif #endif diff --git a/pkg/network/usm/ebpf_main.go b/pkg/network/usm/ebpf_main.go index fdc22e3e1f9250..ffa8771012d6f6 100644 --- a/pkg/network/usm/ebpf_main.go +++ b/pkg/network/usm/ebpf_main.go @@ -14,7 +14,6 @@ import ( "slices" "unsafe" - "github.com/DataDog/datadog-agent/pkg/util/kernel" manager "github.com/DataDog/ebpf-manager" "github.com/cilium/ebpf" "github.com/davecgh/go-spew/spew" @@ -36,6 +35,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/tracer/offsetguess" "github.com/DataDog/datadog-agent/pkg/network/usm/buildmode" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" + "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" ) From e2c3963f394e6b571833a3482a87e1fb729efeb7 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Wed, 12 Feb 2025 17:14:48 -0500 Subject: [PATCH 03/22] [usm] ebpfProgram.init() exclude raw_tracepoint if kernel<4.17.0 --- pkg/network/usm/ebpf_main.go | 5 +++++ pkg/network/usm/sharedlibraries/ebpf.go | 8 -------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/network/usm/ebpf_main.go b/pkg/network/usm/ebpf_main.go index ffa8771012d6f6..9fb82e9d4a489e 100644 --- a/pkg/network/usm/ebpf_main.go +++ b/pkg/network/usm/ebpf_main.go @@ -486,6 +486,11 @@ func (e *ebpfProgram) init(buf bytecode.AssetReader, options manager.Options) er } } + if kversion, err := kernel.HostVersion(); err == nil && kversion < kernel.VersionCode(4, 17, 0) { + // do not use a raw tracepoint on an unsupported kernel. + options.ExcludedFunctions = append(options.ExcludedFunctions, "raw_tracepoint__sched_process_exit") + } + err := e.InitWithOptions(buf, &options) if err != nil { cleanup() diff --git a/pkg/network/usm/sharedlibraries/ebpf.go b/pkg/network/usm/sharedlibraries/ebpf.go index 4dd9afd8c3eee8..6f6cf41d2654b8 100644 --- a/pkg/network/usm/sharedlibraries/ebpf.go +++ b/pkg/network/usm/sharedlibraries/ebpf.go @@ -631,14 +631,6 @@ func (e *EbpfProgram) initializeProbes() { e.enabledProbes = tpProbes e.disabledProbes = tracingProbes } - - if kversion, err := kernel.HostVersion(); err == nil && kversion < kernel.VersionCode(4, 17, 0) { - // do not use a raw tracepoint on an unsupported kernel. - e.disabledProbes = append(e.disabledProbes, manager.ProbeIdentificationPair{ - EBPFFuncName: "raw_tracepoint__sched_process_exit", - UID: probeUID, - }) - } } func getAssetName(module string, debug bool) string { From a58dd1bad5bee5108f474a2022bea2839fc79328 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Thu, 13 Feb 2025 08:04:01 -0500 Subject: [PATCH 04/22] [usm] ebpfProgram.init(), exclude 'tracepoint__sched__sched_process_exit' if raw tracepoint is supported --- pkg/network/usm/ebpf_main.go | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pkg/network/usm/ebpf_main.go b/pkg/network/usm/ebpf_main.go index 9fb82e9d4a489e..bb4d8521b723a5 100644 --- a/pkg/network/usm/ebpf_main.go +++ b/pkg/network/usm/ebpf_main.go @@ -14,7 +14,6 @@ import ( "slices" "unsafe" - manager "github.com/DataDog/ebpf-manager" "github.com/cilium/ebpf" "github.com/davecgh/go-spew/spew" @@ -37,6 +36,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/usm/utils" "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" + manager "github.com/DataDog/ebpf-manager" ) var ( @@ -150,7 +150,7 @@ func newEBPFProgram(c *config.Config, connectionProtocolMap *ebpf.Map) (*ebpfPro } } - if kversion, err := kernel.HostVersion(); err == nil && kversion >= kernel.VersionCode(4, 17, 0) { + if rawTracepointSupported() { // use a raw tracepoint on a supported kernel to intercept terminated threads and clear the corresponding maps. mgr.Probes = append(mgr.Probes, []*manager.Probe{ { @@ -486,8 +486,11 @@ func (e *ebpfProgram) init(buf bytecode.AssetReader, options manager.Options) er } } - if kversion, err := kernel.HostVersion(); err == nil && kversion < kernel.VersionCode(4, 17, 0) { - // do not use a raw tracepoint on an unsupported kernel. + if rawTracepointSupported() { + // exclude regular tracepoint if kernel supports raw tracepoint + options.ExcludedFunctions = append(options.ExcludedFunctions, "tracepoint__sched__sched_process_exit") + } else { + //exclude a raw tracepoint if kernel does not support it. options.ExcludedFunctions = append(options.ExcludedFunctions, "raw_tracepoint__sched_process_exit") } @@ -503,6 +506,14 @@ func (e *ebpfProgram) init(buf bytecode.AssetReader, options manager.Options) er return err } +func rawTracepointSupported() bool { + if kversion, err := kernel.HostVersion(); err == nil && kversion >= kernel.VersionCode(4, 17, 0) { + return true + } + + return false +} + func getAssetName(module string, debug bool) string { if debug { return fmt.Sprintf("%s-debug.o", module) From 9a74005f63cfe058bd7c250472aa2ec925119455 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Thu, 13 Feb 2025 10:33:00 -0500 Subject: [PATCH 05/22] [usm] remove #if conditon for SEC(raw_tracepoint/sched_process_exit) --- pkg/network/ebpf/c/protocols/flush.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/network/ebpf/c/protocols/flush.h b/pkg/network/ebpf/c/protocols/flush.h index 3654854918560a..2c0744e5265092 100644 --- a/pkg/network/ebpf/c/protocols/flush.h +++ b/pkg/network/ebpf/c/protocols/flush.h @@ -39,8 +39,6 @@ int tracepoint__sched__sched_process_exit(void *ctx) { return 0; } -#if defined(COMPILE_PREBUILT) || defined(COMPILE_CORE) || (defined(COMPILE_RUNTIME) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)) -// add this program when compiling CO-RE or runtime compilation for supported kernel versions. SEC("raw_tracepoint/sched_process_exit") int raw_tracepoint__sched_process_exit(void *ctx) { CHECK_BPF_PROGRAM_BYPASSED() @@ -51,6 +49,5 @@ int raw_tracepoint__sched_process_exit(void *ctx) { return 0; } -#endif #endif From cf2ceac1d61343b0c2e6a52b983249ce7c31ed3a Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Fri, 14 Feb 2025 12:03:58 -0500 Subject: [PATCH 06/22] [usm] sharedlibraries.Watcher, periodically clean dead pids from maps if kernel<=4.17 --- pkg/network/usm/ebpf_main.go | 13 ++--- pkg/network/usm/ebpf_ssl.go | 3 +- pkg/network/usm/sharedlibraries/watcher.go | 59 ++++++++++++++++++++++ 3 files changed, 66 insertions(+), 9 deletions(-) diff --git a/pkg/network/usm/ebpf_main.go b/pkg/network/usm/ebpf_main.go index bb4d8521b723a5..07713e6e222cea 100644 --- a/pkg/network/usm/ebpf_main.go +++ b/pkg/network/usm/ebpf_main.go @@ -150,7 +150,7 @@ func newEBPFProgram(c *config.Config, connectionProtocolMap *ebpf.Map) (*ebpfPro } } - if rawTracepointSupported() { + if rawTracepointsSupported() { // use a raw tracepoint on a supported kernel to intercept terminated threads and clear the corresponding maps. mgr.Probes = append(mgr.Probes, []*manager.Probe{ { @@ -486,7 +486,7 @@ func (e *ebpfProgram) init(buf bytecode.AssetReader, options manager.Options) er } } - if rawTracepointSupported() { + if rawTracepointsSupported() { // exclude regular tracepoint if kernel supports raw tracepoint options.ExcludedFunctions = append(options.ExcludedFunctions, "tracepoint__sched__sched_process_exit") } else { @@ -506,12 +506,9 @@ func (e *ebpfProgram) init(buf bytecode.AssetReader, options manager.Options) er return err } -func rawTracepointSupported() bool { - if kversion, err := kernel.HostVersion(); err == nil && kversion >= kernel.VersionCode(4, 17, 0) { - return true - } - - return false +func rawTracepointsSupported() bool { + kversion, err := kernel.HostVersion() + return err == nil && kversion >= kernel.VersionCode(4, 17, 0) } func getAssetName(module string, debug bool) string { diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index 4a7800de41d4b2..0b112b9e25c361 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -499,8 +499,9 @@ func (o *sslProgram) ConfigureOptions(_ *manager.Manager, options *manager.Optio } // PreStart is called before the start of the provided eBPF manager. -func (o *sslProgram) PreStart(*manager.Manager) error { +func (o *sslProgram) PreStart(m *manager.Manager) error { o.watcher.Start() + o.watcher.SetEbpfManager(m) o.istioMonitor.Start() o.nodeJSMonitor.Start() return nil diff --git a/pkg/network/usm/sharedlibraries/watcher.go b/pkg/network/usm/sharedlibraries/watcher.go index f13548414a0173..b0bf026ffa7e07 100644 --- a/pkg/network/usm/sharedlibraries/watcher.go +++ b/pkg/network/usm/sharedlibraries/watcher.go @@ -9,6 +9,7 @@ package sharedlibraries import ( "bufio" + "errors" "fmt" "os" "regexp" @@ -24,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/process/monitor" "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" + manager "github.com/DataDog/ebpf-manager" ) var ( @@ -71,6 +73,8 @@ type Watcher struct { // telemetry libHits *telemetry.Counter libMatches *telemetry.Counter + + ebpfManager *manager.Manager } // Validate that Watcher implements the Attacher interface. @@ -347,4 +351,59 @@ func (w *Watcher) sync() { for pid := range deletionCandidates { _ = w.registry.Unregister(pid) } + + if w.ebpfManager != nil && rawTracepointsNotSupported() { + err := w.CleanDeadPidsInMaps(w.ebpfManager, []string{"ssl_read_args", "ssl_read_ex_args"}, alivePIDs) + if err != nil { + log.Debugf("clean 'ssl_read_args' map error: %v", err) + } + } +} + +// SetEbpfManager assigns eBPF manager +func (w *Watcher) SetEbpfManager(m *manager.Manager) { + if w == nil { + return + } + w.ebpfManager = m +} + +// CleanDeadPidsInMaps finds a map by name and deletes dead processes, used for maps with the key 'u64 pid_tgid' +func (w *Watcher) CleanDeadPidsInMaps(manager *manager.Manager, mapNames []string, alivePIDs map[uint32]struct{}) error { + var errs []error + for _, n := range mapNames { + err := cleanDeadPidsInMap(manager, n, alivePIDs) + if err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func cleanDeadPidsInMap(manager *manager.Manager, mapName string, alivePIDs map[uint32]struct{}) error { + emap, _, err := manager.GetMap(mapName) + if err != nil { + return fmt.Errorf("dead process cleaner failed to get map: %q error: %s", emap, err) + } + iter := emap.Iterate() + var keysToDelete []uint64 + var key uint64 + value := make([]byte, emap.ValueSize()) + + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { + pid := uint32(key >> 32) + if _, exists := alivePIDs[pid]; !exists { + keysToDelete = append(keysToDelete, key) + } + } + for _, k := range keysToDelete { + emap.Delete(&k) + } + + return nil +} + +func rawTracepointsNotSupported() bool { + kversion, err := kernel.HostVersion() + return err == nil && kversion < kernel.VersionCode(4, 17, 0) } From 4b84b6773a48a0f8ee3ba104059c75d421d6a087 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Fri, 14 Feb 2025 13:46:21 -0500 Subject: [PATCH 07/22] [usm] sharedlibraries.Watcher, fix linter error, check err=emap.Delete() --- pkg/network/usm/sharedlibraries/watcher.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/network/usm/sharedlibraries/watcher.go b/pkg/network/usm/sharedlibraries/watcher.go index b0bf026ffa7e07..db0ef3ea25e900 100644 --- a/pkg/network/usm/sharedlibraries/watcher.go +++ b/pkg/network/usm/sharedlibraries/watcher.go @@ -396,11 +396,14 @@ func cleanDeadPidsInMap(manager *manager.Manager, mapName string, alivePIDs map[ keysToDelete = append(keysToDelete, key) } } + var lastError error for _, k := range keysToDelete { - emap.Delete(&k) + if err := emap.Delete(&k); err != nil { + lastError = err + } } - return nil + return lastError } func rawTracepointsNotSupported() bool { From d7728e5f6976ceea5bc1f308acd48270efec48fa Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Fri, 14 Feb 2025 16:21:33 -0500 Subject: [PATCH 08/22] [usm] unit tests, cleaner of the map 'ssl_read_ex_args' --- .../corechecks/ebpf/probe/ebpfcheck/probe.go | 12 ++ pkg/network/protocols/http/types.go | 1 + pkg/network/protocols/http/types_linux.go | 5 + pkg/network/usm/ebpf_ssl.go | 31 +++- pkg/network/usm/monitor_tls_test.go | 171 ++++++++++++++++++ 5 files changed, 217 insertions(+), 3 deletions(-) diff --git a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go index d09baa968c0a59..3da5bcde4717ae 100644 --- a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go +++ b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go @@ -965,3 +965,15 @@ func hashMapNumberOfEntriesWithHelper(mp *ebpf.Map, mapid ebpf.MapID, mphCache * return int64(res), nil } + +// HashMapNumberOfEntries returns the number of entries in the map +func HashMapNumberOfEntries(mp *ebpf.Map) (int64, error) { + if isPerCPU(mp.Type()) { + return -1, fmt.Errorf("unsupported map type: %s", mp.String()) + } + buffers := entryCountBuffers{ + keysBufferSizeLimit: 0, // No limit + valuesBufferSizeLimit: 0, // No limit + } + return hashMapNumberOfEntriesWithIteration(mp, &buffers, 1) +} diff --git a/pkg/network/protocols/http/types.go b/pkg/network/protocols/http/types.go index 3f0eb5c9639377..21bf1e91e8010f 100644 --- a/pkg/network/protocols/http/types.go +++ b/pkg/network/protocols/http/types.go @@ -17,6 +17,7 @@ import "C" type ConnTuple = C.conn_tuple_t type SslSock C.ssl_sock_t type SslReadArgs C.ssl_read_args_t +type SslReadExArgs C.ssl_read_ex_args_t type EbpfEvent C.http_event_t type EbpfTx C.http_transaction_t diff --git a/pkg/network/protocols/http/types_linux.go b/pkg/network/protocols/http/types_linux.go index e4b3a976234573..a227e08a2717bd 100644 --- a/pkg/network/protocols/http/types_linux.go +++ b/pkg/network/protocols/http/types_linux.go @@ -23,6 +23,11 @@ type SslReadArgs struct { Ctx *byte Buf *byte } +type SslReadExArgs struct { + Ctx *byte + Buf *byte + Out_param *uint64 +} type EbpfEvent struct { Tuple ConnTuple diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index 0b112b9e25c361..13528607c95b65 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -535,9 +535,34 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map) io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.ssl_read_args_t'\n") iter := currentMap.Iterate() var key uint64 - var value http.SslReadArgs - for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { - spew.Fdump(w, key, value) + // The wrapper struct prevents 'Fdump' from accessing content of pointers. + a := struct { + Ctx unsafe.Pointer + Buf unsafe.Pointer + }{ + Ctx: unsafe.Pointer(http.SslReadArgs{}.Ctx), + Buf: unsafe.Pointer(http.SslReadArgs{}.Buf), + } + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&a)) { + spew.Fdump(w, key, a) + } + + case "ssl_read_ex_args": // maps/ssl_read_ex_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_read_ex_args_t + io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.ssl_read_ex_args_t'\n") + iter := currentMap.Iterate() + var key uint64 + // The wrapper struct prevents 'Fdump' from accessing content of pointers. + a := struct { + Ctx unsafe.Pointer + Buf unsafe.Pointer + Size_out_param unsafe.Pointer + }{ + Ctx: unsafe.Pointer(http.SslReadExArgs{}.Ctx), + Buf: unsafe.Pointer(http.SslReadExArgs{}.Buf), + Size_out_param: unsafe.Pointer(http.SslReadExArgs{}.Out_param), + } + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&a)) { + spew.Fdump(w, key, a) } case "bio_new_socket_args": // maps/bio_new_socket_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.__u32 diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index 06700430167a78..86b8e1ffb67556 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/DataDog/datadog-agent/pkg/collector/corechecks/ebpf/probe/ebpfcheck" "github.com/DataDog/datadog-agent/pkg/ebpf/ebpftest" "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers" consumerstestutil "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers/testutil" @@ -46,8 +47,10 @@ import ( usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" "github.com/DataDog/datadog-agent/pkg/process/monitor" + "github.com/DataDog/datadog-agent/pkg/util/kernel" globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" + manager "github.com/DataDog/ebpf-manager" ) type tlsSuite struct { @@ -991,3 +994,171 @@ func (s *tlsSuite) TestNodeJSTLS() { } } } + +// TestSSLReadArgsMaps verifies proper clearance of SSL-related kernel maps. +func (s *tlsSuite) TestSSLReadArgsMaps() { + t := s.T() + // setup monitor + cfg := utils.NewUSMEmptyConfig() + cfg.EnableNativeTLSMonitoring = true + + addressOfHTTPPythonServer := "127.0.0.1:4443" + cmd := testutil.HTTPPythonServer(t, addressOfHTTPPythonServer, testutil.Options{ + EnableTLS: true, + }) + + monitor := setupUSMTLSMonitor(t, cfg, reInitEventConsumer) + // Giving the tracer time to install the hooks + utils.WaitForProgramsToBeTraced(t, consts.USMModuleName, "shared_libraries", cmd.Process.Pid, utils.ManualTracingFallbackEnabled) + if !utils.IsProgramTraced(consts.USMModuleName, "shared_libraries", cmd.Process.Pid) { + return + } + + // find probes for the programs we want to manipulate + probeSSLReadEx := getProbeByName(monitor.ebpfProgram.Manager.Manager, "uretprobe__SSL_read_ex") + require.NotNil(t, probeSSLReadEx) + + probeProcExit := getProbeProcExit(t, monitor) + require.NotNil(t, probeProcExit) + + // find the map + readExArgsMap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap("ssl_read_ex_args") + require.NotNil(t, readExArgsMap) + + // create client + client, requestFn := simpleGetRequestsGenerator(t, addressOfHTTPPythonServer) + + units := []struct { + name string + expected int64 + preRun func(t *testing.T) + postRun func(t *testing.T) + }{ + { + // disable both 'SSL_read_ex' and 'sched_process_exit', check the map is not empty + name: "eBPF programs disabled", + expected: 1, + preRun: func(t *testing.T) { + cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) + detachProbe(t, monitor.ebpfProgram.Manager.Manager, probeSSLReadEx) + err := probeProcExit.Pause() + assert.NoError(t, err) + }, + postRun: func(t *testing.T) { + client.CloseIdleConnections() + }, + }, + { + // disable both 'SSL_read_ex' and 'sched_process_exit', ensure the cleaner properly clears the map + name: "periodic cleaner", + expected: 0, + preRun: func(t *testing.T) { + cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) + detachProbe(t, monitor.ebpfProgram.Manager.Manager, probeSSLReadEx) + err := probeProcExit.Pause() + assert.NoError(t, err) + }, + postRun: func(t *testing.T) { + client.CloseIdleConnections() + err := monitor.ebpfProgram.cleanDeadPidsInSslMaps() + assert.NoError(t, err) + }, + }, + { + // check if only 'sched_process_exit' is present and process terminates then the map is empty. + // must be last test case, because it terminates the server + name: "check terminated server", + expected: 0, + preRun: func(t *testing.T) { + cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) + err := probeProcExit.Resume() + assert.NoError(t, err) + }, + postRun: func(t *testing.T) { + client.CloseIdleConnections() + cmd.Process.Kill() + cmd.Wait() + }, + }, + } + for _, unit := range units { + t.Run(unit.name, func(t *testing.T) { + unit.preRun(t) + + // send requests to server + for i := 0; i < numberOfRequests; i++ { + requestFn() + } + unit.postRun(t) + + require.Eventually(t, func() bool { + num, err := ebpfcheck.HashMapNumberOfEntries(readExArgsMap) + assert.NoError(t, err) + + if num == unit.expected { + return true + } + return false + }, 2*time.Second, 200*time.Millisecond, "unexpected map entries") + }) + if t.Failed() { + t.Logf("Unexpect number of entries in the map") + ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, "ssl_read_ex_args") + } + } +} + +// getProbeByName returns the probe of running or paused program that match the input name +func getProbeByName(m *manager.Manager, name string) *manager.Probe { + for _, probe := range m.Probes { + if probe.EBPFFuncName == name && probe.IsRunning() { + return probe + } + } + return nil +} + +// getProbeProcExit returns the 'Probe' for the process exit program, either as a regular trace or a raw trace. +func getProbeProcExit(t *testing.T, monitor *Monitor) *manager.Probe { + probeRawProcExit := getProbeByName(monitor.ebpfProgram.Manager.Manager, "raw_tracepoint__sched_process_exit") + if probeRawProcExit != nil { + return probeRawProcExit + } + if kversion, err := kernel.HostVersion(); err == nil && kversion >= kernel.VersionCode(4, 17, 0) { + // raw tracepoints are supported on kernel>=4.17 + require.FailNow(t, "raw tracepoint missing despite kernel supports it") + } + probeProcExit := getProbeByName(monitor.ebpfProgram.Manager.Manager, "tracepoint__sched__sched_process_exit") + return probeProcExit +} + +func detachProbe(t *testing.T, m *manager.Manager, probe *manager.Probe) { + id := manager.ProbeIdentificationPair{ + UID: probe.UID, + EBPFFuncName: probe.EBPFFuncName, + } + err := m.DetachHook(id) + assert.NoError(t, err) +} + +// cleanDeadPidsInSslMaps finds activated 'sslProgram' and calls map cleaner. +func (e *ebpfProgram) cleanDeadPidsInSslMaps() error { + for _, prot := range e.enabledProtocols { + if prot.Instance.Name() == "openssl" { + switch prot.Instance.(type) { + case *sslProgram: + p, ok := prot.Instance.(*sslProgram) + if ok { + return p.cleanDeadPidsInMaps(e.Manager.Manager) + } + default: + } + } + } + return nil +} + +// cleanDeadPidsInMaps clears terminated processes from SSL-related kernel maps. +func (o *sslProgram) cleanDeadPidsInMaps(manager *manager.Manager) error { + return o.watcher.CleanDeadPidsInMaps(manager, []string{"ssl_read_args", "ssl_read_ex_args"}, nil) +} From 9fba5d2c18b5ac2e058c2c871a6ce816760d415b Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Fri, 14 Feb 2025 17:08:28 -0500 Subject: [PATCH 09/22] [usm] fix linter errors in TestSSLReadArgsMaps() --- pkg/network/usm/ebpf_ssl.go | 12 ++++++------ pkg/network/usm/monitor_tls_test.go | 9 +++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index 13528607c95b65..b15821ff6d876d 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -553,13 +553,13 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map) var key uint64 // The wrapper struct prevents 'Fdump' from accessing content of pointers. a := struct { - Ctx unsafe.Pointer - Buf unsafe.Pointer - Size_out_param unsafe.Pointer + Ctx unsafe.Pointer + Buf unsafe.Pointer + SizeOutParam unsafe.Pointer }{ - Ctx: unsafe.Pointer(http.SslReadExArgs{}.Ctx), - Buf: unsafe.Pointer(http.SslReadExArgs{}.Buf), - Size_out_param: unsafe.Pointer(http.SslReadExArgs{}.Out_param), + Ctx: unsafe.Pointer(http.SslReadExArgs{}.Ctx), + Buf: unsafe.Pointer(http.SslReadExArgs{}.Buf), + SizeOutParam: unsafe.Pointer(http.SslReadExArgs{}.Out_param), } for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&a)) { spew.Fdump(w, key, a) diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index 86b8e1ffb67556..61aa9853df66a3 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -1044,7 +1044,7 @@ func (s *tlsSuite) TestSSLReadArgsMaps() { err := probeProcExit.Pause() assert.NoError(t, err) }, - postRun: func(t *testing.T) { + postRun: func(*testing.T) { client.CloseIdleConnections() }, }, @@ -1074,7 +1074,7 @@ func (s *tlsSuite) TestSSLReadArgsMaps() { err := probeProcExit.Resume() assert.NoError(t, err) }, - postRun: func(t *testing.T) { + postRun: func(*testing.T) { client.CloseIdleConnections() cmd.Process.Kill() cmd.Wait() @@ -1095,10 +1095,7 @@ func (s *tlsSuite) TestSSLReadArgsMaps() { num, err := ebpfcheck.HashMapNumberOfEntries(readExArgsMap) assert.NoError(t, err) - if num == unit.expected { - return true - } - return false + return num == unit.expected }, 2*time.Second, 200*time.Millisecond, "unexpected map entries") }) if t.Failed() { From 1e141f3836166e35f507e157aa12e9af6ff8792c Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Tue, 18 Feb 2025 07:36:11 -0500 Subject: [PATCH 10/22] [usm] TestSSLReadArgsMaps, test used map 'ssl_read_args' or 'ssl_read_ex_args' --- pkg/network/usm/monitor_tls_test.go | 77 +++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 21 deletions(-) diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index 61aa9853df66a3..9d670e17e29b7f 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/cilium/ebpf" krpretty "github.com/kr/pretty" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -891,9 +892,9 @@ func reinitializeEventConsumer(t *testing.T) { } const ( - // useExistingConsumer is used to indicate that we should use the existing consumer instance + // reInitEventConsumer is used to indicate that we should create a new consumer instance reInitEventConsumer = true - // useExistingConsumer is used to indicate that we should create a new consumer instance + // useExistingConsumer is used to indicate that we should use the existing consumer instance useExistingConsumer = false ) @@ -1015,21 +1016,22 @@ func (s *tlsSuite) TestSSLReadArgsMaps() { } // find probes for the programs we want to manipulate - probeSSLReadEx := getProbeByName(monitor.ebpfProgram.Manager.Manager, "uretprobe__SSL_read_ex") - require.NotNil(t, probeSSLReadEx) + sslReadProbe, useEx := getSSLReadProbe(t, monitor) + require.NotNil(t, sslReadProbe) - probeProcExit := getProbeProcExit(t, monitor) - require.NotNil(t, probeProcExit) + procExitProbe := getProcExitProbe(t, monitor) + require.NotNil(t, procExitProbe) // find the map - readExArgsMap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap("ssl_read_ex_args") - require.NotNil(t, readExArgsMap) + sslReadMap := getSSLReadMap(t, monitor, useEx) // create client client, requestFn := simpleGetRequestsGenerator(t, addressOfHTTPPythonServer) + require.NotNil(t, client) units := []struct { name string + found int64 expected int64 preRun func(t *testing.T) postRun func(t *testing.T) @@ -1040,8 +1042,8 @@ func (s *tlsSuite) TestSSLReadArgsMaps() { expected: 1, preRun: func(t *testing.T) { cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) - detachProbe(t, monitor.ebpfProgram.Manager.Manager, probeSSLReadEx) - err := probeProcExit.Pause() + detachProbe(t, monitor.ebpfProgram.Manager.Manager, sslReadProbe) + err := procExitProbe.Pause() assert.NoError(t, err) }, postRun: func(*testing.T) { @@ -1054,14 +1056,18 @@ func (s *tlsSuite) TestSSLReadArgsMaps() { expected: 0, preRun: func(t *testing.T) { cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) - detachProbe(t, monitor.ebpfProgram.Manager.Manager, probeSSLReadEx) - err := probeProcExit.Pause() + detachProbe(t, monitor.ebpfProgram.Manager.Manager, sslReadProbe) + err := procExitProbe.Pause() assert.NoError(t, err) }, postRun: func(t *testing.T) { + // allow eBPF maps to sync with kernel + assert.Eventually(t, func() bool { + err := monitor.ebpfProgram.cleanDeadPidsInSslMaps() + return err == nil + }, 200*time.Millisecond, 50*time.Millisecond, "failed to clean up dead processes") + client.CloseIdleConnections() - err := monitor.ebpfProgram.cleanDeadPidsInSslMaps() - assert.NoError(t, err) }, }, { @@ -1071,13 +1077,13 @@ func (s *tlsSuite) TestSSLReadArgsMaps() { expected: 0, preRun: func(t *testing.T) { cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) - err := probeProcExit.Resume() + err := procExitProbe.Resume() assert.NoError(t, err) }, postRun: func(*testing.T) { - client.CloseIdleConnections() cmd.Process.Kill() cmd.Wait() + client.CloseIdleConnections() }, }, } @@ -1092,20 +1098,22 @@ func (s *tlsSuite) TestSSLReadArgsMaps() { unit.postRun(t) require.Eventually(t, func() bool { - num, err := ebpfcheck.HashMapNumberOfEntries(readExArgsMap) + num, err := ebpfcheck.HashMapNumberOfEntries(sslReadMap) assert.NoError(t, err) + unit.found = num return num == unit.expected }, 2*time.Second, 200*time.Millisecond, "unexpected map entries") }) if t.Failed() { - t.Logf("Unexpect number of entries in the map") + t.Logf("'%s' expect number of entries '%d' got '%d' in the map '%v'", unit.name, unit.expected, unit.found, sslReadMap) ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, "ssl_read_ex_args") + ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, "ssl_read_args") } } } -// getProbeByName returns the probe of running or paused program that match the input name +// getProbeByName returns the probe of running or paused program that match the input name. func getProbeByName(m *manager.Manager, name string) *manager.Probe { for _, probe := range m.Probes { if probe.EBPFFuncName == name && probe.IsRunning() { @@ -1115,8 +1123,21 @@ func getProbeByName(m *manager.Manager, name string) *manager.Probe { return nil } -// getProbeProcExit returns the 'Probe' for the process exit program, either as a regular trace or a raw trace. -func getProbeProcExit(t *testing.T, monitor *Monitor) *manager.Probe { +// getSSLReadProbe it searches an available 'SSL_read ex' or 'SSL_read' program. +func getSSLReadProbe(t *testing.T, monitor *Monitor) (*manager.Probe, bool) { + probe := getProbeByName(monitor.ebpfProgram.Manager.Manager, "uretprobe__SSL_read_ex") + if probe != nil && probe.IsRunning() { + t.Logf("found probe: %v", probe.GetEBPFFuncName()) + return probe, true + } + + probe = getProbeByName(monitor.ebpfProgram.Manager.Manager, "uretprobe__SSL_read") + t.Logf("found probe: %v", probe.GetEBPFFuncName()) + return probe, false +} + +// getProcExitProbe returns the 'Probe' for the process exit program, either as a regular trace or a raw trace. +func getProcExitProbe(t *testing.T, monitor *Monitor) *manager.Probe { probeRawProcExit := getProbeByName(monitor.ebpfProgram.Manager.Manager, "raw_tracepoint__sched_process_exit") if probeRawProcExit != nil { return probeRawProcExit @@ -1129,6 +1150,20 @@ func getProbeProcExit(t *testing.T, monitor *Monitor) *manager.Probe { return probeProcExit } +// getSSLReadMap returns a map corresponding to the program under test. +func getSSLReadMap(t *testing.T, monitor *Monitor, useEx bool) *ebpf.Map { + if useEx { + sslReadMap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap("ssl_read_ex_args") + require.NotNil(t, sslReadMap) + t.Logf("check map: %v", sslReadMap) + return sslReadMap + } + sslReadMap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap("ssl_read_args") + require.NotNil(t, sslReadMap) + t.Logf("check map: %v", sslReadMap) + return sslReadMap +} + func detachProbe(t *testing.T, m *manager.Manager, probe *manager.Probe) { id := manager.ProbeIdentificationPair{ UID: probe.UID, From e0aef6561478c53c3e67cb0eaba47b272b973a81 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Tue, 18 Feb 2025 11:02:00 -0500 Subject: [PATCH 11/22] [usm] adjust type SslReadExArgs --- pkg/network/protocols/http/types_linux.go | 4 +-- .../protocols/http/types_linux_test.go | 4 +++ pkg/network/usm/ebpf_ssl.go | 28 ++++--------------- 3 files changed, 12 insertions(+), 24 deletions(-) diff --git a/pkg/network/protocols/http/types_linux.go b/pkg/network/protocols/http/types_linux.go index a49141e9fea0ad..c2cd0c5a11297a 100644 --- a/pkg/network/protocols/http/types_linux.go +++ b/pkg/network/protocols/http/types_linux.go @@ -24,8 +24,8 @@ type SslReadArgs struct { Buf uint64 } type SslReadExArgs struct { - Ctx *byte - Buf *byte + Ctx uint64 + Buf uint64 Out_param *uint64 } diff --git a/pkg/network/protocols/http/types_linux_test.go b/pkg/network/protocols/http/types_linux_test.go index 7a5617dce5c046..9b417a12aca7eb 100644 --- a/pkg/network/protocols/http/types_linux_test.go +++ b/pkg/network/protocols/http/types_linux_test.go @@ -16,6 +16,10 @@ func TestCgoAlignment_SslReadArgs(t *testing.T) { ebpftest.TestCgoAlignment[SslReadArgs](t) } +func TestCgoAlignment_SslReadExArgs(t *testing.T) { + ebpftest.TestCgoAlignment[SslReadExArgs](t) +} + func TestCgoAlignment_EbpfEvent(t *testing.T) { ebpftest.TestCgoAlignment[EbpfEvent](t) } diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index b15821ff6d876d..64a9405b65c25d 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -535,34 +535,18 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map) io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.ssl_read_args_t'\n") iter := currentMap.Iterate() var key uint64 - // The wrapper struct prevents 'Fdump' from accessing content of pointers. - a := struct { - Ctx unsafe.Pointer - Buf unsafe.Pointer - }{ - Ctx: unsafe.Pointer(http.SslReadArgs{}.Ctx), - Buf: unsafe.Pointer(http.SslReadArgs{}.Buf), - } - for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&a)) { - spew.Fdump(w, key, a) + var value http.SslReadArgs + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { + spew.Fdump(w, key, value) } case "ssl_read_ex_args": // maps/ssl_read_ex_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_read_ex_args_t io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.ssl_read_ex_args_t'\n") iter := currentMap.Iterate() var key uint64 - // The wrapper struct prevents 'Fdump' from accessing content of pointers. - a := struct { - Ctx unsafe.Pointer - Buf unsafe.Pointer - SizeOutParam unsafe.Pointer - }{ - Ctx: unsafe.Pointer(http.SslReadExArgs{}.Ctx), - Buf: unsafe.Pointer(http.SslReadExArgs{}.Buf), - SizeOutParam: unsafe.Pointer(http.SslReadExArgs{}.Out_param), - } - for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&a)) { - spew.Fdump(w, key, a) + var value http.SslReadExArgs + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { + spew.Fdump(w, key, value) } case "bio_new_socket_args": // maps/bio_new_socket_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.__u32 From c576481c6570b17c65a8cda73a1af5f3a7b1d01a Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Tue, 18 Feb 2025 15:49:47 -0500 Subject: [PATCH 12/22] [usm] simplify in SSL maps cleaner test in tlsSuite. --- pkg/network/usm/monitor_tls_test.go | 222 +++++++++------------------- 1 file changed, 69 insertions(+), 153 deletions(-) diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index 9d670e17e29b7f..6da7aa66d51a4b 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -10,6 +10,7 @@ package usm import ( "bufio" "bytes" + "context" "crypto/tls" "fmt" "io" @@ -24,6 +25,7 @@ import ( "sync" "testing" "time" + "unsafe" "github.com/cilium/ebpf" krpretty "github.com/kr/pretty" @@ -48,7 +50,6 @@ import ( usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" "github.com/DataDog/datadog-agent/pkg/process/monitor" - "github.com/DataDog/datadog-agent/pkg/util/kernel" globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" manager "github.com/DataDog/ebpf-manager" @@ -996,192 +997,107 @@ func (s *tlsSuite) TestNodeJSTLS() { } } -// TestSSLReadArgsMaps verifies proper clearance of SSL-related kernel maps. -func (s *tlsSuite) TestSSLReadArgsMaps() { +// TestSSLMapsCleaner verifies that SSL-related kernel maps are cleared correctly. +// the map entry is deleted when the thread exits, and periodic map cleaner removes dead threads. +func (s *tlsSuite) TestSSLMapsCleaner() { t := s.T() // setup monitor cfg := utils.NewUSMEmptyConfig() cfg.EnableNativeTLSMonitoring = true - addressOfHTTPPythonServer := "127.0.0.1:4443" - cmd := testutil.HTTPPythonServer(t, addressOfHTTPPythonServer, testutil.Options{ - EnableTLS: true, - }) - monitor := setupUSMTLSMonitor(t, cfg, reInitEventConsumer) - // Giving the tracer time to install the hooks - utils.WaitForProgramsToBeTraced(t, consts.USMModuleName, "shared_libraries", cmd.Process.Pid, utils.ManualTracingFallbackEnabled) - if !utils.IsProgramTraced(consts.USMModuleName, "shared_libraries", cmd.Process.Pid) { - return - } - - // find probes for the programs we want to manipulate - sslReadProbe, useEx := getSSLReadProbe(t, monitor) - require.NotNil(t, sslReadProbe) - - procExitProbe := getProcExitProbe(t, monitor) - require.NotNil(t, procExitProbe) - - // find the map - sslReadMap := getSSLReadMap(t, monitor, useEx) - - // create client - client, requestFn := simpleGetRequestsGenerator(t, addressOfHTTPPythonServer) - require.NotNil(t, client) units := []struct { - name string - found int64 - expected int64 - preRun func(t *testing.T) - postRun func(t *testing.T) + mapName string }{ { - // disable both 'SSL_read_ex' and 'sched_process_exit', check the map is not empty - name: "eBPF programs disabled", - expected: 1, - preRun: func(t *testing.T) { - cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) - detachProbe(t, monitor.ebpfProgram.Manager.Manager, sslReadProbe) - err := procExitProbe.Pause() - assert.NoError(t, err) - }, - postRun: func(*testing.T) { - client.CloseIdleConnections() - }, - }, - { - // disable both 'SSL_read_ex' and 'sched_process_exit', ensure the cleaner properly clears the map - name: "periodic cleaner", - expected: 0, - preRun: func(t *testing.T) { - cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) - detachProbe(t, monitor.ebpfProgram.Manager.Manager, sslReadProbe) - err := procExitProbe.Pause() - assert.NoError(t, err) - }, - postRun: func(t *testing.T) { - // allow eBPF maps to sync with kernel - assert.Eventually(t, func() bool { - err := monitor.ebpfProgram.cleanDeadPidsInSslMaps() - return err == nil - }, 200*time.Millisecond, 50*time.Millisecond, "failed to clean up dead processes") - - client.CloseIdleConnections() - }, + mapName: "ssl_read_args", }, { - // check if only 'sched_process_exit' is present and process terminates then the map is empty. - // must be last test case, because it terminates the server - name: "check terminated server", - expected: 0, - preRun: func(t *testing.T) { - cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) - err := procExitProbe.Resume() - assert.NoError(t, err) - }, - postRun: func(*testing.T) { - cmd.Process.Kill() - cmd.Wait() - client.CloseIdleConnections() - }, + mapName: "ssl_read_ex_args", }, } for _, unit := range units { - t.Run(unit.name, func(t *testing.T) { - unit.preRun(t) - - // send requests to server - for i := 0; i < numberOfRequests; i++ { - requestFn() - } - unit.postRun(t) - - require.Eventually(t, func() bool { - num, err := ebpfcheck.HashMapNumberOfEntries(sslReadMap) - assert.NoError(t, err) - unit.found = num - - return num == unit.expected - }, 2*time.Second, 200*time.Millisecond, "unexpected map entries") + testName := fmt.Sprintf("verify %s", unit.mapName) + t.Run(testName, func(t *testing.T) { + cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) + // find map by name + emap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap(unit.mapName) + require.NotNil(t, emap) + + // add random pid to the map + addPidEntryToMap(t, emap, 100) + verifyMap(t, emap, 1) + + // call SSL map cleaner and verify that map is empty + cleanDeadPids(t, monitor, unit.mapName) + verifyMap(t, emap, 0) + + // start dummy program and add it's pid to the map + cmd, cancel := startDummyProgram(t) + addPidEntryToMap(t, emap, cmd.Process.Pid) + verifyMap(t, emap, 1) + + // verify exit of process cleans the map + cancel() + _ = cmd.Wait() + verifyMap(t, emap, 0) }) if t.Failed() { - t.Logf("'%s' expect number of entries '%d' got '%d' in the map '%v'", unit.name, unit.expected, unit.found, sslReadMap) - ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, "ssl_read_ex_args") - ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, "ssl_read_args") + t.Logf("unexpect number of entries in the map '%s'", unit.mapName) + ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, unit.mapName) } } } -// getProbeByName returns the probe of running or paused program that match the input name. -func getProbeByName(m *manager.Manager, name string) *manager.Probe { - for _, probe := range m.Probes { - if probe.EBPFFuncName == name && probe.IsRunning() { - return probe - } - } - return nil +// verifyMap checks if the number of entries in the map matches the expected number. +func verifyMap(t *testing.T, m *ebpf.Map, expected int64) { + require.Eventually(t, func() bool { + num, err := ebpfcheck.HashMapNumberOfEntries(m) + assert.NoError(t, err) + return num == expected + }, 1*time.Second, 100*time.Millisecond, "unexpected map entries") } -// getSSLReadProbe it searches an available 'SSL_read ex' or 'SSL_read' program. -func getSSLReadProbe(t *testing.T, monitor *Monitor) (*manager.Probe, bool) { - probe := getProbeByName(monitor.ebpfProgram.Manager.Manager, "uretprobe__SSL_read_ex") - if probe != nil && probe.IsRunning() { - t.Logf("found probe: %v", probe.GetEBPFFuncName()) - return probe, true - } +// addPidEntryToMap adds an entry to the map using the PID as a key. +func addPidEntryToMap(t *testing.T, m *ebpf.Map, pid int) { + require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") - probe = getProbeByName(monitor.ebpfProgram.Manager.Manager, "uretprobe__SSL_read") - t.Logf("found probe: %v", probe.GetEBPFFuncName()) - return probe, false -} + // make the key for single thread process when pid and tgid are the same + key := uint64(pid)<<32 | uint64(pid) + value := make([]byte, m.ValueSize()) -// getProcExitProbe returns the 'Probe' for the process exit program, either as a regular trace or a raw trace. -func getProcExitProbe(t *testing.T, monitor *Monitor) *manager.Probe { - probeRawProcExit := getProbeByName(monitor.ebpfProgram.Manager.Manager, "raw_tracepoint__sched_process_exit") - if probeRawProcExit != nil { - return probeRawProcExit - } - if kversion, err := kernel.HostVersion(); err == nil && kversion >= kernel.VersionCode(4, 17, 0) { - // raw tracepoints are supported on kernel>=4.17 - require.FailNow(t, "raw tracepoint missing despite kernel supports it") - } - probeProcExit := getProbeByName(monitor.ebpfProgram.Manager.Manager, "tracepoint__sched__sched_process_exit") - return probeProcExit + err := m.Put(&key, value) + require.NoError(t, err) } -// getSSLReadMap returns a map corresponding to the program under test. -func getSSLReadMap(t *testing.T, monitor *Monitor, useEx bool) *ebpf.Map { - if useEx { - sslReadMap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap("ssl_read_ex_args") - require.NotNil(t, sslReadMap) - t.Logf("check map: %v", sslReadMap) - return sslReadMap - } - sslReadMap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap("ssl_read_args") - require.NotNil(t, sslReadMap) - t.Logf("check map: %v", sslReadMap) - return sslReadMap +// startDummyProgram starts sleeping thread. +func startDummyProgram(t *testing.T) (*exec.Cmd, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(func() { cancel() }) + + cmd := exec.CommandContext(ctx, "sleep", "1000") + err := cmd.Start() + require.NoError(t, err) + + return cmd, cancel } -func detachProbe(t *testing.T, m *manager.Manager, probe *manager.Probe) { - id := manager.ProbeIdentificationPair{ - UID: probe.UID, - EBPFFuncName: probe.EBPFFuncName, - } - err := m.DetachHook(id) - assert.NoError(t, err) +// cleanDeadPids calls the map cleaner for map 'name', as the periodic job does. +func cleanDeadPids(t *testing.T, monitor *Monitor, name string) { + err := monitor.ebpfProgram.cleanDeadPidsInSslMap(name) + require.NoError(t, err) } -// cleanDeadPidsInSslMaps finds activated 'sslProgram' and calls map cleaner. -func (e *ebpfProgram) cleanDeadPidsInSslMaps() error { +// cleanDeadPidsInSslMap finds activated 'sslProgram' and calls map cleaner. +func (e *ebpfProgram) cleanDeadPidsInSslMap(name string) error { for _, prot := range e.enabledProtocols { if prot.Instance.Name() == "openssl" { switch prot.Instance.(type) { case *sslProgram: p, ok := prot.Instance.(*sslProgram) if ok { - return p.cleanDeadPidsInMaps(e.Manager.Manager) + return p.cleanDeadPidsInMap(e.Manager.Manager, name) } default: } @@ -1190,7 +1106,7 @@ func (e *ebpfProgram) cleanDeadPidsInSslMaps() error { return nil } -// cleanDeadPidsInMaps clears terminated processes from SSL-related kernel maps. -func (o *sslProgram) cleanDeadPidsInMaps(manager *manager.Manager) error { - return o.watcher.CleanDeadPidsInMaps(manager, []string{"ssl_read_args", "ssl_read_ex_args"}, nil) +// cleanDeadPidsInMap clears terminated processes from SSL-related kernel map. +func (o *sslProgram) cleanDeadPidsInMap(manager *manager.Manager, mapName string) error { + return o.watcher.CleanDeadPidsInMaps(manager, []string{mapName}, nil) } From f1a6d7ebfe98a9c84db14650484e1465e16c4fe4 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Tue, 18 Feb 2025 17:30:51 -0500 Subject: [PATCH 13/22] [usm] remove auto-generated TestCgoAlignment_SslReadExArgs --- pkg/network/protocols/http/types_linux_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/network/protocols/http/types_linux_test.go b/pkg/network/protocols/http/types_linux_test.go index 9b417a12aca7eb..7a5617dce5c046 100644 --- a/pkg/network/protocols/http/types_linux_test.go +++ b/pkg/network/protocols/http/types_linux_test.go @@ -16,10 +16,6 @@ func TestCgoAlignment_SslReadArgs(t *testing.T) { ebpftest.TestCgoAlignment[SslReadArgs](t) } -func TestCgoAlignment_SslReadExArgs(t *testing.T) { - ebpftest.TestCgoAlignment[SslReadExArgs](t) -} - func TestCgoAlignment_EbpfEvent(t *testing.T) { ebpftest.TestCgoAlignment[EbpfEvent](t) } From e4a5338bfa5d37f0a740affd104ba251a92fe1bd Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Wed, 19 Feb 2025 08:44:41 -0500 Subject: [PATCH 14/22] [usm] add auto-generated TestCgoAlignment_SslReadExArgs --- pkg/network/protocols/http/types_linux.go | 2 +- pkg/network/protocols/http/types_linux_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/network/protocols/http/types_linux.go b/pkg/network/protocols/http/types_linux.go index c2cd0c5a11297a..4c1fb05b219db5 100644 --- a/pkg/network/protocols/http/types_linux.go +++ b/pkg/network/protocols/http/types_linux.go @@ -26,7 +26,7 @@ type SslReadArgs struct { type SslReadExArgs struct { Ctx uint64 Buf uint64 - Out_param *uint64 + Out_param uint64 } type EbpfEvent struct { diff --git a/pkg/network/protocols/http/types_linux_test.go b/pkg/network/protocols/http/types_linux_test.go index 7a5617dce5c046..9b417a12aca7eb 100644 --- a/pkg/network/protocols/http/types_linux_test.go +++ b/pkg/network/protocols/http/types_linux_test.go @@ -16,6 +16,10 @@ func TestCgoAlignment_SslReadArgs(t *testing.T) { ebpftest.TestCgoAlignment[SslReadArgs](t) } +func TestCgoAlignment_SslReadExArgs(t *testing.T) { + ebpftest.TestCgoAlignment[SslReadExArgs](t) +} + func TestCgoAlignment_EbpfEvent(t *testing.T) { ebpftest.TestCgoAlignment[EbpfEvent](t) } From 1fffb430c15bcfcf01ac6940d6f2b898cb0ea8f7 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Wed, 19 Feb 2025 11:50:05 -0500 Subject: [PATCH 15/22] [usm] clean maps on thread exit: ssl_write_args, ssl_write_ex_args, ssl_ctx_by_pid_tgid, bio_new_socket_args --- pkg/network/ebpf/c/protocols/flush.h | 8 ++++++++ pkg/network/protocols/http/types.go | 2 ++ pkg/network/protocols/http/types_linux.go | 9 +++++++++ pkg/network/protocols/http/types_linux_test.go | 8 ++++++++ pkg/network/usm/ebpf_ssl.go | 18 ++++++++++++++++++ pkg/network/usm/monitor_tls_test.go | 13 +++++++++++++ pkg/network/usm/sharedlibraries/watcher.go | 10 +++++++++- 7 files changed, 67 insertions(+), 1 deletion(-) diff --git a/pkg/network/ebpf/c/protocols/flush.h b/pkg/network/ebpf/c/protocols/flush.h index 2c0744e5265092..87e94406f8052b 100644 --- a/pkg/network/ebpf/c/protocols/flush.h +++ b/pkg/network/ebpf/c/protocols/flush.h @@ -35,6 +35,10 @@ int tracepoint__sched__sched_process_exit(void *ctx) { bpf_map_delete_elem(&ssl_read_args, &pid_tgid); bpf_map_delete_elem(&ssl_read_ex_args, &pid_tgid); + bpf_map_delete_elem(&ssl_write_args, &pid_tgid); + bpf_map_delete_elem(&ssl_write_ex_args, &pid_tgid); + bpf_map_delete_elem(&ssl_ctx_by_pid_tgid, &pid_tgid); + bpf_map_delete_elem(&bio_new_socket_args, &pid_tgid); return 0; } @@ -46,6 +50,10 @@ int raw_tracepoint__sched_process_exit(void *ctx) { bpf_map_delete_elem(&ssl_read_args, &pid_tgid); bpf_map_delete_elem(&ssl_read_ex_args, &pid_tgid); + bpf_map_delete_elem(&ssl_write_args, &pid_tgid); + bpf_map_delete_elem(&ssl_write_ex_args, &pid_tgid); + bpf_map_delete_elem(&ssl_ctx_by_pid_tgid, &pid_tgid); + bpf_map_delete_elem(&bio_new_socket_args, &pid_tgid); return 0; } diff --git a/pkg/network/protocols/http/types.go b/pkg/network/protocols/http/types.go index 21bf1e91e8010f..05a97f01ea2ced 100644 --- a/pkg/network/protocols/http/types.go +++ b/pkg/network/protocols/http/types.go @@ -18,6 +18,8 @@ type ConnTuple = C.conn_tuple_t type SslSock C.ssl_sock_t type SslReadArgs C.ssl_read_args_t type SslReadExArgs C.ssl_read_ex_args_t +type SslWriteArgs C.ssl_write_args_t +type SslWriteExArgs C.ssl_write_ex_args_t type EbpfEvent C.http_event_t type EbpfTx C.http_transaction_t diff --git a/pkg/network/protocols/http/types_linux.go b/pkg/network/protocols/http/types_linux.go index 4c1fb05b219db5..19fa6e778dd1be 100644 --- a/pkg/network/protocols/http/types_linux.go +++ b/pkg/network/protocols/http/types_linux.go @@ -28,6 +28,15 @@ type SslReadExArgs struct { Buf uint64 Out_param uint64 } +type SslWriteArgs struct { + Ctx uint64 + Buf uint64 +} +type SslWriteExArgs struct { + Ctx uint64 + Buf uint64 + Out_param uint64 +} type EbpfEvent struct { Tuple ConnTuple diff --git a/pkg/network/protocols/http/types_linux_test.go b/pkg/network/protocols/http/types_linux_test.go index 9b417a12aca7eb..87250bf1fe5954 100644 --- a/pkg/network/protocols/http/types_linux_test.go +++ b/pkg/network/protocols/http/types_linux_test.go @@ -20,6 +20,14 @@ func TestCgoAlignment_SslReadExArgs(t *testing.T) { ebpftest.TestCgoAlignment[SslReadExArgs](t) } +func TestCgoAlignment_SslWriteArgs(t *testing.T) { + ebpftest.TestCgoAlignment[SslWriteArgs](t) +} + +func TestCgoAlignment_SslWriteExArgs(t *testing.T) { + ebpftest.TestCgoAlignment[SslWriteExArgs](t) +} + func TestCgoAlignment_EbpfEvent(t *testing.T) { ebpftest.TestCgoAlignment[EbpfEvent](t) } diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index 47741c57ebde00..d2c12d128dd78b 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -549,6 +549,24 @@ func (o *sslProgram) DumpMaps(w io.Writer, mapName string, currentMap *ebpf.Map) spew.Fdump(w, key, value) } + case "ssl_write_args": // maps/ssl_write_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_write_args_t + io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.ssl_write_args_t'\n") + iter := currentMap.Iterate() + var key uint64 + var value http.SslWriteArgs + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { + spew.Fdump(w, key, value) + } + + case "ssl_write_ex_args_t": // maps/ssl_write_ex_args_t (BPF_MAP_TYPE_HASH), key C.__u64, value C.ssl_write_args_t + io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.ssl_write_ex_args_t'\n") + iter := currentMap.Iterate() + var key uint64 + var value http.SslWriteExArgs + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { + spew.Fdump(w, key, value) + } + case "bio_new_socket_args": // maps/bio_new_socket_args (BPF_MAP_TYPE_HASH), key C.__u64, value C.__u32 io.WriteString(w, "Map: '"+mapName+"', key: 'C.__u64', value: 'C.__u32'\n") iter := currentMap.Iterate() diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index dfa14614f882de..62a873b6663138 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -1010,6 +1010,7 @@ func (s *tlsSuite) TestSSLMapsCleaner() { // setup monitor cfg := utils.NewUSMEmptyConfig() cfg.EnableNativeTLSMonitoring = true + cfg.EnableUSMEventStream = false monitor := setupUSMTLSMonitor(t, cfg, reInitEventConsumer) @@ -1022,6 +1023,18 @@ func (s *tlsSuite) TestSSLMapsCleaner() { { mapName: "ssl_read_ex_args", }, + { + mapName: "ssl_write_args", + }, + { + mapName: "ssl_write_ex_args", + }, + { + mapName: "ssl_ctx_by_pid_tgid", + }, + { + mapName: "bio_new_socket_args", + }, } for _, unit := range units { testName := fmt.Sprintf("verify %s", unit.mapName) diff --git a/pkg/network/usm/sharedlibraries/watcher.go b/pkg/network/usm/sharedlibraries/watcher.go index db0ef3ea25e900..e351f9b18a7d6d 100644 --- a/pkg/network/usm/sharedlibraries/watcher.go +++ b/pkg/network/usm/sharedlibraries/watcher.go @@ -353,7 +353,15 @@ func (w *Watcher) sync() { } if w.ebpfManager != nil && rawTracepointsNotSupported() { - err := w.CleanDeadPidsInMaps(w.ebpfManager, []string{"ssl_read_args", "ssl_read_ex_args"}, alivePIDs) + maps := []string{ + "ssl_read_args", + "ssl_read_ex_args", + "ssl_write_args", + "ssl_write_ex_args", + "ssl_ctx_by_pid_tgid", + "bio_new_socket_args", + } + err := w.CleanDeadPidsInMaps(w.ebpfManager, maps, alivePIDs) if err != nil { log.Debugf("clean 'ssl_read_args' map error: %v", err) } From 946fde86dab132ae5609ce606475834c8a6344c9 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Fri, 21 Feb 2025 09:49:26 -0500 Subject: [PATCH 16/22] [usm] moved exit tracepoint setup and ssl maps cleanup to 'sslProgram' --- .../corechecks/ebpf/probe/ebpfcheck/probe.go | 12 -- pkg/network/ebpf/c/protocols/flush.h | 19 +- pkg/network/usm/ebpf_main.go | 37 ---- pkg/network/usm/ebpf_ssl.go | 99 +++++++++- pkg/network/usm/monitor_tls_test.go | 176 ++++++++++-------- pkg/network/usm/sharedlibraries/watcher.go | 77 +------- .../usm/sharedlibraries/watcher_test.go | 18 +- 7 files changed, 216 insertions(+), 222 deletions(-) diff --git a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go index 3da5bcde4717ae..d09baa968c0a59 100644 --- a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go +++ b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go @@ -965,15 +965,3 @@ func hashMapNumberOfEntriesWithHelper(mp *ebpf.Map, mapid ebpf.MapID, mphCache * return int64(res), nil } - -// HashMapNumberOfEntries returns the number of entries in the map -func HashMapNumberOfEntries(mp *ebpf.Map) (int64, error) { - if isPerCPU(mp.Type()) { - return -1, fmt.Errorf("unsupported map type: %s", mp.String()) - } - buffers := entryCountBuffers{ - keysBufferSizeLimit: 0, // No limit - valuesBufferSizeLimit: 0, // No limit - } - return hashMapNumberOfEntriesWithIteration(mp, &buffers, 1) -} diff --git a/pkg/network/ebpf/c/protocols/flush.h b/pkg/network/ebpf/c/protocols/flush.h index 87e94406f8052b..cacf458035e591 100644 --- a/pkg/network/ebpf/c/protocols/flush.h +++ b/pkg/network/ebpf/c/protocols/flush.h @@ -28,9 +28,7 @@ int tracepoint__net__netif_receive_skb(void *ctx) { return 0; } -SEC("tracepoint/sched/sched_process_exit") -int tracepoint__sched__sched_process_exit(void *ctx) { - CHECK_BPF_PROGRAM_BYPASSED() +static __always_inline void delete_pid_in_maps() { u64 pid_tgid = bpf_get_current_pid_tgid(); bpf_map_delete_elem(&ssl_read_args, &pid_tgid); @@ -39,22 +37,19 @@ int tracepoint__sched__sched_process_exit(void *ctx) { bpf_map_delete_elem(&ssl_write_ex_args, &pid_tgid); bpf_map_delete_elem(&ssl_ctx_by_pid_tgid, &pid_tgid); bpf_map_delete_elem(&bio_new_socket_args, &pid_tgid); +} +SEC("tracepoint/sched/sched_process_exit") +int tracepoint__sched__sched_process_exit(void *ctx) { + CHECK_BPF_PROGRAM_BYPASSED() + delete_pid_in_maps(); return 0; } SEC("raw_tracepoint/sched_process_exit") int raw_tracepoint__sched_process_exit(void *ctx) { CHECK_BPF_PROGRAM_BYPASSED() - u64 pid_tgid = bpf_get_current_pid_tgid(); - - bpf_map_delete_elem(&ssl_read_args, &pid_tgid); - bpf_map_delete_elem(&ssl_read_ex_args, &pid_tgid); - bpf_map_delete_elem(&ssl_write_args, &pid_tgid); - bpf_map_delete_elem(&ssl_write_ex_args, &pid_tgid); - bpf_map_delete_elem(&ssl_ctx_by_pid_tgid, &pid_tgid); - bpf_map_delete_elem(&bio_new_socket_args, &pid_tgid); - + delete_pid_in_maps(); return 0; } diff --git a/pkg/network/usm/ebpf_main.go b/pkg/network/usm/ebpf_main.go index d811c0ae3db983..2da22a10775416 100644 --- a/pkg/network/usm/ebpf_main.go +++ b/pkg/network/usm/ebpf_main.go @@ -34,7 +34,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/tracer/offsetguess" "github.com/DataDog/datadog-agent/pkg/network/usm/buildmode" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" - "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" manager "github.com/DataDog/ebpf-manager" ) @@ -150,29 +149,6 @@ func newEBPFProgram(c *config.Config, connectionProtocolMap *ebpf.Map) (*ebpfPro } } - if rawTracepointsSupported() { - // use a raw tracepoint on a supported kernel to intercept terminated threads and clear the corresponding maps. - mgr.Probes = append(mgr.Probes, []*manager.Probe{ - { - ProbeIdentificationPair: manager.ProbeIdentificationPair{ - EBPFFuncName: "raw_tracepoint__sched_process_exit", - UID: probeUID, - }, - TracepointName: "sched_process_exit", - }, - }...) - } else { - // use a regular tracepoint to intercept terminated threads. - mgr.Probes = append(mgr.Probes, []*manager.Probe{ - { - ProbeIdentificationPair: manager.ProbeIdentificationPair{ - EBPFFuncName: "tracepoint__sched__sched_process_exit", - UID: probeUID, - }, - }, - }...) - } - program := &ebpfProgram{ Manager: ddebpf.NewManager(mgr, "usm", &ebpftelemetry.ErrorsTelemetryModifier{}), cfg: c, @@ -486,14 +462,6 @@ func (e *ebpfProgram) init(buf bytecode.AssetReader, options manager.Options) er } } - if rawTracepointsSupported() { - // exclude regular tracepoint if kernel supports raw tracepoint - options.ExcludedFunctions = append(options.ExcludedFunctions, "tracepoint__sched__sched_process_exit") - } else { - //exclude a raw tracepoint if kernel does not support it. - options.ExcludedFunctions = append(options.ExcludedFunctions, "raw_tracepoint__sched_process_exit") - } - err := e.InitWithOptions(buf, &options) if err != nil { cleanup() @@ -506,11 +474,6 @@ func (e *ebpfProgram) init(buf bytecode.AssetReader, options manager.Options) er return err } -func rawTracepointsSupported() bool { - kversion, err := kernel.HostVersion() - return err == nil && kversion >= kernel.VersionCode(4, 17, 0) -} - func getAssetName(module string, debug bool) string { if debug { return fmt.Sprintf("%s-debug.o", module) diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index d2c12d128dd78b..aefcf83e0a7a54 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -9,6 +9,7 @@ package usm import ( "bytes" + "errors" "fmt" "io" "os" @@ -20,8 +21,8 @@ import ( "time" "unsafe" - manager "github.com/DataDog/ebpf-manager" "github.com/cilium/ebpf" + "github.com/cilium/ebpf/features" "github.com/davecgh/go-spew/spew" ddebpf "github.com/DataDog/datadog-agent/pkg/ebpf" @@ -38,6 +39,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/util/safeelf" ddsync "github.com/DataDog/datadog-agent/pkg/util/sync" + manager "github.com/DataDog/ebpf-manager" ) const ( @@ -424,6 +426,7 @@ type sslProgram struct { watcher *sharedlibraries.Watcher istioMonitor *istioMonitor nodeJSMonitor *nodeJSMonitor + ebpfManager *manager.Manager } func newSSLProgramProtocolFactory(m *manager.Manager) protocols.ProtocolFactory { @@ -487,7 +490,7 @@ func (o *sslProgram) Name() string { } // ConfigureOptions changes map attributes to the given options. -func (o *sslProgram) ConfigureOptions(_ *manager.Manager, options *manager.Options) { +func (o *sslProgram) ConfigureOptions(m *manager.Manager, options *manager.Options) { options.MapSpecEditors[sslSockByCtxMap] = manager.MapSpecEditor{ MaxEntries: o.cfg.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries, @@ -496,12 +499,13 @@ func (o *sslProgram) ConfigureOptions(_ *manager.Manager, options *manager.Optio MaxEntries: o.cfg.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries, } + o.addProcessExitProbe(m, options) } // PreStart is called before the start of the provided eBPF manager. func (o *sslProgram) PreStart(m *manager.Manager) error { - o.watcher.Start() - o.watcher.SetEbpfManager(m) + o.ebpfManager = m + o.watcher.Start(o.cleanupDeadPids) o.istioMonitor.Start() o.nodeJSMonitor.Start() return nil @@ -806,3 +810,90 @@ func getUID(lib utils.PathIdentifier) string { func (*sslProgram) IsBuildModeSupported(buildmode.Type) bool { return true } + +// addProcessExitProbe adds a raw or regular tracepoint program depending on which is supported. +func (o *sslProgram) addProcessExitProbe(mgr *manager.Manager, options *manager.Options) { + const rawTracepointSchedProcessExit = "raw_tracepoint__sched_process_exit" + const oldTracepointSchedProcessExit = "tracepoint__sched__sched_process_exit" + + if features.HaveProgramType(ebpf.RawTracepoint) == nil { + // use a raw tracepoint on a supported kernel to intercept terminated threads and clear the corresponding maps + p := &manager.Probe{ + ProbeIdentificationPair: manager.ProbeIdentificationPair{ + EBPFFuncName: rawTracepointSchedProcessExit, + UID: probeUID, + }, + TracepointName: "sched_process_exit", + } + mgr.Probes = append(mgr.Probes, p) + options.ActivatedProbes = append(options.ActivatedProbes, &manager.ProbeSelector{ProbeIdentificationPair: p.ProbeIdentificationPair}) + // exclude regular tracepoint + options.ExcludedFunctions = append(options.ExcludedFunctions, oldTracepointSchedProcessExit) + } else { + // use a regular tracepoint to intercept terminated threads + p := &manager.Probe{ + ProbeIdentificationPair: manager.ProbeIdentificationPair{ + EBPFFuncName: oldTracepointSchedProcessExit, + UID: probeUID, + }, + } + mgr.Probes = append(mgr.Probes, p) + options.ActivatedProbes = append(options.ActivatedProbes, &manager.ProbeSelector{ProbeIdentificationPair: p.ProbeIdentificationPair}) + // exclude a raw tracepoint + options.ExcludedFunctions = append(options.ExcludedFunctions, rawTracepointSchedProcessExit) + } +} + +var sslPidKeyMaps = []string{ + "ssl_read_args", + "ssl_read_ex_args", + "ssl_write_args", + "ssl_write_ex_args", + "ssl_ctx_by_pid_tgid", + "bio_new_socket_args", +} + +// cleanupDeadPids clears maps of terminated processes. +func (o *sslProgram) cleanupDeadPids(alivePIDs map[uint32]struct{}) { + if o.ebpfManager != nil { + err := o.deleteDeadPidsInMaps(sslPidKeyMaps, alivePIDs) + if err != nil { + log.Debugf("SSL maps cleanup error: %v", err) + } + } +} + +// deleteDeadPidsInMaps deletes dead processes in maps with the key 'pid_tgid' +func (o *sslProgram) deleteDeadPidsInMaps(mapNames []string, alivePIDs map[uint32]struct{}) error { + var errs []error + for _, n := range mapNames { + err := deleteDeadPidsInMap(o.ebpfManager, n, alivePIDs) + if err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// deleteDeadPidsInMap finds a map by name and deletes dead processes. +func deleteDeadPidsInMap(manager *manager.Manager, mapName string, alivePIDs map[uint32]struct{}) error { + emap, _, err := manager.GetMap(mapName) + if err != nil { + return fmt.Errorf("dead process cleaner failed to get map: %q error: %s", mapName, err) + } + + var keysToDelete []uint64 + var key uint64 + value := make([]byte, emap.ValueSize()) + iter := emap.Iterate() + + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { + pid := uint32(key >> 32) + if _, exists := alivePIDs[pid]; !exists { + keysToDelete = append(keysToDelete, key) + } + } + _, err = emap.BatchDelete(keysToDelete, nil) + + return err +} diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index 62a873b6663138..b247396abb87fb 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -33,7 +33,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/DataDog/datadog-agent/pkg/collector/corechecks/ebpf/probe/ebpfcheck" "github.com/DataDog/datadog-agent/pkg/ebpf/ebpftest" "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers" consumerstestutil "github.com/DataDog/datadog-agent/pkg/eventmonitor/consumers/testutil" @@ -52,7 +51,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/process/monitor" globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" - manager "github.com/DataDog/ebpf-manager" ) type tlsSuite struct { @@ -1013,81 +1011,111 @@ func (s *tlsSuite) TestSSLMapsCleaner() { cfg.EnableUSMEventStream = false monitor := setupUSMTLSMonitor(t, cfg, reInitEventConsumer) + require.NotNil(t, monitor) + + t.Run("SSL maps cleaner", func(t *testing.T) { + cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) + cleanProtocolMaps(t, "bio_new_socket_args", monitor.ebpfProgram.Manager.Manager) + + // find maps by names + maps := getMaps(t, monitor, sslPidKeyMaps) + require.Equal(t, len(maps), 6) + + // add random pid to the maps + pid := 100 + addPidEntryToMaps(t, maps, pid) + checkPidExistsInMaps(t, monitor, maps, pid) + + // call SSL map cleaner and verify that map is empty + monitor.ebpfProgram.cleanDeadPidsInSslMaps() + checkPidNotFoundInMaps(t, monitor, maps, pid) + + // start dummy program and add its pid to the map + cmd, cancel := startDummyProgram(t) + addPidEntryToMaps(t, maps, cmd.Process.Pid) + checkPidExistsInMaps(t, monitor, maps, cmd.Process.Pid) + + // verify exit of process cleans the map + cancel() + _ = cmd.Wait() + checkPidNotFoundInMaps(t, monitor, maps, cmd.Process.Pid) + }) +} - units := []struct { - mapName string - }{ - { - mapName: "ssl_read_args", - }, - { - mapName: "ssl_read_ex_args", - }, - { - mapName: "ssl_write_args", - }, - { - mapName: "ssl_write_ex_args", - }, - { - mapName: "ssl_ctx_by_pid_tgid", - }, - { - mapName: "bio_new_socket_args", - }, - } - for _, unit := range units { - testName := fmt.Sprintf("verify %s", unit.mapName) - t.Run(testName, func(t *testing.T) { - cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) - // find map by name - emap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap(unit.mapName) - require.NotNil(t, emap) - - // add random pid to the map - addPidEntryToMap(t, emap, 100) - verifyMap(t, emap, 1) - - // call SSL map cleaner and verify that map is empty - cleanDeadPids(t, monitor, unit.mapName) - verifyMap(t, emap, 0) - - // start dummy program and add it's pid to the map - cmd, cancel := startDummyProgram(t) - addPidEntryToMap(t, emap, cmd.Process.Pid) - verifyMap(t, emap, 1) - - // verify exit of process cleans the map - cancel() - _ = cmd.Wait() - verifyMap(t, emap, 0) - }) - if t.Failed() { - t.Logf("unexpect number of entries in the map '%s'", unit.mapName) - ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, unit.mapName) - } +// getMaps returns eBPF maps searched by names. +func getMaps(t *testing.T, monitor *Monitor, mapNames []string) []*ebpf.Map { + var maps []*ebpf.Map + for _, mapName := range mapNames { + emap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap(mapName) + require.NotNil(t, emap) + maps = append(maps, emap) } + return maps } -// verifyMap checks if the number of entries in the map matches the expected number. -func verifyMap(t *testing.T, m *ebpf.Map, expected int64) { - require.Eventually(t, func() bool { - num, err := ebpfcheck.HashMapNumberOfEntries(m) - assert.NoError(t, err) - return num == expected - }, 1*time.Second, 100*time.Millisecond, "unexpected map entries") +// addPidEntryToMaps adds an entry to maps using the PID as a key. +func addPidEntryToMaps(t *testing.T, maps []*ebpf.Map, pid int) { + for _, m := range maps { + require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") + + // make the key for single thread process when pid and tgid are the same + key := uint64(pid)<<32 | uint64(pid) + value := make([]byte, m.ValueSize()) + + err := m.Put(&key, value) + require.NoError(t, err) + } } -// addPidEntryToMap adds an entry to the map using the PID as a key. -func addPidEntryToMap(t *testing.T, m *ebpf.Map, pid int) { - require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") +// checkPidExistsInMaps checks that pid exists in all provided maps. +func checkPidExistsInMaps(t *testing.T, monitor *Monitor, maps []*ebpf.Map, pid int) { + // make the key for single thread process when pid and tgid are the same + key := uint64(pid)<<32 | uint64(pid) + + for _, m := range maps { + require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") + mapInfo, err := m.Info() + require.NoError(t, err) + require.Eventually(t, func() bool { + return findKeyInMap(m, key) + }, 1*time.Second, 100*time.Millisecond) + if t.Failed() { + t.Logf("pid '%d' not found in the map '%s'", pid, mapInfo.Name) + ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, mapInfo.Name) + } + } +} + +// checkPidNotFoundInMaps checks that pid does not exist in all provided maps. +func checkPidNotFoundInMaps(t *testing.T, monitor *Monitor, maps []*ebpf.Map, pid int) { // make the key for single thread process when pid and tgid are the same key := uint64(pid)<<32 | uint64(pid) + + for _, m := range maps { + require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") + mapInfo, err := m.Info() + require.NoError(t, err) + + if findKeyInMap(m, key) == true { + t.Logf("pid '%d' was found in the map '%s'", pid, mapInfo.Name) + ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, mapInfo.Name) + } + } +} + +// findKeyInMap returns true if 'key' was found in the map, otherwise returns false. +func findKeyInMap(m *ebpf.Map, theKey uint64) bool { + var key uint64 value := make([]byte, m.ValueSize()) + iter := m.Iterate() - err := m.Put(&key, value) - require.NoError(t, err) + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { + if key == theKey { + return true + } + } + return false } // startDummyProgram starts sleeping thread. @@ -1102,30 +1130,18 @@ func startDummyProgram(t *testing.T) (*exec.Cmd, context.CancelFunc) { return cmd, cancel } -// cleanDeadPids calls the map cleaner for map 'name', as the periodic job does. -func cleanDeadPids(t *testing.T, monitor *Monitor, name string) { - err := monitor.ebpfProgram.cleanDeadPidsInSslMap(name) - require.NoError(t, err) -} - // cleanDeadPidsInSslMap finds activated 'sslProgram' and calls map cleaner. -func (e *ebpfProgram) cleanDeadPidsInSslMap(name string) error { +func (e *ebpfProgram) cleanDeadPidsInSslMaps() { for _, prot := range e.enabledProtocols { if prot.Instance.Name() == "openssl" { switch prot.Instance.(type) { case *sslProgram: p, ok := prot.Instance.(*sslProgram) if ok { - return p.cleanDeadPidsInMap(e.Manager.Manager, name) + p.cleanupDeadPids(nil) } default: } } } - return nil -} - -// cleanDeadPidsInMap clears terminated processes from SSL-related kernel map. -func (o *sslProgram) cleanDeadPidsInMap(manager *manager.Manager, mapName string) error { - return o.watcher.CleanDeadPidsInMaps(manager, []string{mapName}, nil) } diff --git a/pkg/network/usm/sharedlibraries/watcher.go b/pkg/network/usm/sharedlibraries/watcher.go index e351f9b18a7d6d..e1aa0e96298131 100644 --- a/pkg/network/usm/sharedlibraries/watcher.go +++ b/pkg/network/usm/sharedlibraries/watcher.go @@ -9,7 +9,6 @@ package sharedlibraries import ( "bufio" - "errors" "fmt" "os" "regexp" @@ -18,6 +17,9 @@ import ( "time" "unsafe" + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/features" + "github.com/DataDog/datadog-agent/pkg/network/config" "github.com/DataDog/datadog-agent/pkg/network/protocols/telemetry" "github.com/DataDog/datadog-agent/pkg/network/usm/consts" @@ -25,7 +27,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/process/monitor" "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" - manager "github.com/DataDog/ebpf-manager" ) var ( @@ -74,7 +75,7 @@ type Watcher struct { libHits *telemetry.Counter libMatches *telemetry.Counter - ebpfManager *manager.Manager + mapsCleaner func(map[uint32]struct{}) } // Validate that Watcher implements the Attacher interface. @@ -212,10 +213,11 @@ func (w *Watcher) handleLibraryOpen(lib LibPath) { } // Start consuming shared-library events -func (w *Watcher) Start() { +func (w *Watcher) Start(mapsCleaner func(map[uint32]struct{})) { if w == nil { return } + w.mapsCleaner = mapsCleaner var err error w.thisPID, err = kernel.RootNSPID() @@ -352,69 +354,8 @@ func (w *Watcher) sync() { _ = w.registry.Unregister(pid) } - if w.ebpfManager != nil && rawTracepointsNotSupported() { - maps := []string{ - "ssl_read_args", - "ssl_read_ex_args", - "ssl_write_args", - "ssl_write_ex_args", - "ssl_ctx_by_pid_tgid", - "bio_new_socket_args", - } - err := w.CleanDeadPidsInMaps(w.ebpfManager, maps, alivePIDs) - if err != nil { - log.Debugf("clean 'ssl_read_args' map error: %v", err) - } + if w.mapsCleaner != nil && features.HaveProgramType(ebpf.RawTracepoint) != nil { + // call maps cleaner if raw tracepoints are not supported + w.mapsCleaner(alivePIDs) } } - -// SetEbpfManager assigns eBPF manager -func (w *Watcher) SetEbpfManager(m *manager.Manager) { - if w == nil { - return - } - w.ebpfManager = m -} - -// CleanDeadPidsInMaps finds a map by name and deletes dead processes, used for maps with the key 'u64 pid_tgid' -func (w *Watcher) CleanDeadPidsInMaps(manager *manager.Manager, mapNames []string, alivePIDs map[uint32]struct{}) error { - var errs []error - for _, n := range mapNames { - err := cleanDeadPidsInMap(manager, n, alivePIDs) - if err != nil { - errs = append(errs, err) - } - } - return errors.Join(errs...) -} - -func cleanDeadPidsInMap(manager *manager.Manager, mapName string, alivePIDs map[uint32]struct{}) error { - emap, _, err := manager.GetMap(mapName) - if err != nil { - return fmt.Errorf("dead process cleaner failed to get map: %q error: %s", emap, err) - } - iter := emap.Iterate() - var keysToDelete []uint64 - var key uint64 - value := make([]byte, emap.ValueSize()) - - for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { - pid := uint32(key >> 32) - if _, exists := alivePIDs[pid]; !exists { - keysToDelete = append(keysToDelete, key) - } - } - var lastError error - for _, k := range keysToDelete { - if err := emap.Delete(&k); err != nil { - lastError = err - } - } - - return lastError -} - -func rawTracepointsNotSupported() bool { - kversion, err := kernel.HostVersion() - return err == nil && kversion < kernel.VersionCode(4, 17, 0) -} diff --git a/pkg/network/usm/sharedlibraries/watcher_test.go b/pkg/network/usm/sharedlibraries/watcher_test.go index 05b914ef5ec46f..ce552eebd68edd 100644 --- a/pkg/network/usm/sharedlibraries/watcher_test.go +++ b/pkg/network/usm/sharedlibraries/watcher_test.go @@ -84,7 +84,7 @@ func (s *SharedLibrarySuite) TestSharedLibraryDetection() { }, ) require.NoError(t, err) - watcher.Start() + watcher.Start(nil) t.Cleanup(watcher.Stop) // create files @@ -161,7 +161,7 @@ func (s *SharedLibrarySuite) TestSharedLibraryIgnoreWrite() { }, ) require.NoError(t, err) - watcher.Start() + watcher.Start(nil) t.Cleanup(watcher.Stop) // Overriding PID, to allow the watcher to watch the test process watcher.thisPID = 0 @@ -208,7 +208,7 @@ func (s *SharedLibrarySuite) TestLongPath() { }, ) require.NoError(t, err) - watcher.Start() + watcher.Start(nil) t.Cleanup(watcher.Stop) // create files @@ -276,7 +276,7 @@ func (s *SharedLibrarySuite) TestSharedLibraryDetectionPeriodic() { }, ) require.NoError(t, err) - watcher.Start() + watcher.Start(nil) t.Cleanup(watcher.Stop) // create files @@ -370,7 +370,7 @@ func (s *SharedLibrarySuite) TestSharedLibraryDetectionWithPIDAndRootNamespace() }, ) require.NoError(t, err) - watcher.Start() + watcher.Start(nil) t.Cleanup(watcher.Stop) time.Sleep(10 * time.Millisecond) @@ -419,7 +419,7 @@ func (s *SharedLibrarySuite) TestSameInodeRegression() { }, ) require.NoError(t, err) - watcher.Start() + watcher.Start(nil) t.Cleanup(watcher.Stop) command1, err := fileopener.OpenFromAnotherProcess(t, fooPath1, fooPath2) @@ -465,7 +465,7 @@ func (s *SharedLibrarySuite) TestSoWatcherLeaks() { }, ) require.NoError(t, err) - watcher.Start() + watcher.Start(nil) t.Cleanup(watcher.Stop) command1, err := fileopener.OpenFromAnotherProcess(t, fooPath1, fooPath2) @@ -537,7 +537,7 @@ func (s *SharedLibrarySuite) TestSoWatcherProcessAlreadyHoldingReferences() { command2, err := fileopener.OpenFromAnotherProcess(t, fooPath1) require.NoError(t, err) - watcher.Start() + watcher.Start(nil) t.Cleanup(watcher.Stop) require.Eventually(t, func() bool { @@ -664,7 +664,7 @@ func (s *SharedLibrarySuite) TestValidPathExistsInTheMemory() { }, ) require.NoError(t, err) - watcher.Start() + watcher.Start(nil) t.Cleanup(watcher.Stop) // Overriding PID, to allow the watcher to watch the test process watcher.thisPID = 0 From d541a5b0b0b029e7f303147754a07bef29624e05 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Fri, 21 Feb 2025 14:43:55 -0500 Subject: [PATCH 17/22] [usm] add probe 'sched_process_exit' to openSSLProbes. --- pkg/network/usm/ebpf_ssl.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index aefcf83e0a7a54..72e4b3d8ac1117 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -71,6 +71,9 @@ const ( gnutlsRecordSendRetprobe = "uretprobe__gnutls_record_send" gnutlsByeProbe = "uprobe__gnutls_bye" gnutlsDeinitProbe = "uprobe__gnutls_deinit" + + rawTracepointSchedProcessExit = "raw_tracepoint__sched_process_exit" + oldTracepointSchedProcessExit = "tracepoint__sched__sched_process_exit" ) var openSSLProbes = []manager.ProbesSelector{ @@ -418,6 +421,16 @@ var opensslSpec = &protocols.ProtocolSpec{ EBPFFuncName: gnutlsDeinitProbe, }, }, + { + ProbeIdentificationPair: manager.ProbeIdentificationPair{ + EBPFFuncName: rawTracepointSchedProcessExit, + }, + }, + { + ProbeIdentificationPair: manager.ProbeIdentificationPair{ + EBPFFuncName: oldTracepointSchedProcessExit, + }, + }, }, } @@ -480,6 +493,7 @@ func newSSLProgramProtocolFactory(m *manager.Manager) protocols.ProtocolFactory watcher: watcher, istioMonitor: istio, nodeJSMonitor: nodejs, + ebpfManager: m, }, nil } } @@ -503,8 +517,7 @@ func (o *sslProgram) ConfigureOptions(m *manager.Manager, options *manager.Optio } // PreStart is called before the start of the provided eBPF manager. -func (o *sslProgram) PreStart(m *manager.Manager) error { - o.ebpfManager = m +func (o *sslProgram) PreStart(*manager.Manager) error { o.watcher.Start(o.cleanupDeadPids) o.istioMonitor.Start() o.nodeJSMonitor.Start() @@ -813,9 +826,6 @@ func (*sslProgram) IsBuildModeSupported(buildmode.Type) bool { // addProcessExitProbe adds a raw or regular tracepoint program depending on which is supported. func (o *sslProgram) addProcessExitProbe(mgr *manager.Manager, options *manager.Options) { - const rawTracepointSchedProcessExit = "raw_tracepoint__sched_process_exit" - const oldTracepointSchedProcessExit = "tracepoint__sched__sched_process_exit" - if features.HaveProgramType(ebpf.RawTracepoint) == nil { // use a raw tracepoint on a supported kernel to intercept terminated threads and clear the corresponding maps p := &manager.Probe{ From ea736fabf0441378c7898ecc7f8b256c421341ad Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Mon, 24 Feb 2025 09:05:04 -0500 Subject: [PATCH 18/22] [usm] move TestSSLMapsCleaner() to ebpf_ssl_test.go --- pkg/network/usm/ebpf_ssl.go | 33 ++--- pkg/network/usm/ebpf_ssl_test.go | 148 +++++++++++++++++++++ pkg/network/usm/monitor_tls_test.go | 148 --------------------- pkg/network/usm/sharedlibraries/watcher.go | 6 +- 4 files changed, 162 insertions(+), 173 deletions(-) diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index 72e4b3d8ac1117..c93ee9efb1b4cf 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -9,7 +9,6 @@ package usm import ( "bytes" - "errors" "fmt" "io" "os" @@ -865,31 +864,23 @@ var sslPidKeyMaps = []string{ // cleanupDeadPids clears maps of terminated processes. func (o *sslProgram) cleanupDeadPids(alivePIDs map[uint32]struct{}) { - if o.ebpfManager != nil { - err := o.deleteDeadPidsInMaps(sslPidKeyMaps, alivePIDs) - if err != nil { - log.Debugf("SSL maps cleanup error: %v", err) - } - } -} - -// deleteDeadPidsInMaps deletes dead processes in maps with the key 'pid_tgid' -func (o *sslProgram) deleteDeadPidsInMaps(mapNames []string, alivePIDs map[uint32]struct{}) error { - var errs []error - for _, n := range mapNames { - err := deleteDeadPidsInMap(o.ebpfManager, n, alivePIDs) - if err != nil { - errs = append(errs, err) + if o.ebpfManager != nil && features.HaveProgramType(ebpf.RawTracepoint) != nil { + // call maps cleaner if raw tracepoints are not supported + for _, mapName := range sslPidKeyMaps { + err := deleteDeadPidsInMap(o.ebpfManager, mapName, alivePIDs) + if err != nil { + log.Debugf("SSL maps %q cleanup error: %v", mapName, err) + } } } - return errors.Join(errs...) } // deleteDeadPidsInMap finds a map by name and deletes dead processes. +// enters when raw tracepoint is not supported, kernel < 4.17 func deleteDeadPidsInMap(manager *manager.Manager, mapName string, alivePIDs map[uint32]struct{}) error { emap, _, err := manager.GetMap(mapName) if err != nil { - return fmt.Errorf("dead process cleaner failed to get map: %q error: %s", mapName, err) + return fmt.Errorf("dead process cleaner failed to get map: %q error: %w", mapName, err) } var keysToDelete []uint64 @@ -903,7 +894,9 @@ func deleteDeadPidsInMap(manager *manager.Manager, mapName string, alivePIDs map keysToDelete = append(keysToDelete, key) } } - _, err = emap.BatchDelete(keysToDelete, nil) + for _, k := range keysToDelete { + emap.Delete(&k) + } - return err + return nil } diff --git a/pkg/network/usm/ebpf_ssl_test.go b/pkg/network/usm/ebpf_ssl_test.go index e4b9576669cbf2..141ed0897c6e72 100644 --- a/pkg/network/usm/ebpf_ssl_test.go +++ b/pkg/network/usm/ebpf_ssl_test.go @@ -8,12 +8,18 @@ package usm import ( + "context" "fmt" "os" + "os/exec" "path/filepath" "runtime" "testing" + "time" + "unsafe" + "github.com/DataDog/datadog-agent/pkg/ebpf/ebpftest" + "github.com/cilium/ebpf" "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" @@ -65,3 +71,145 @@ func TestContainerdTmpErrEnvironment(t *testing.T) { err := hookFunction(path) require.ErrorIs(t, err, utils.ErrEnvironment) } + +// TestSSLMapsCleaner verifies that SSL-related kernel maps are cleared correctly. +// the map entry is deleted when the thread exits, also periodic map cleaner removes dead threads. +func TestSSLMapsCleaner(t *testing.T) { + // setup monitor + cfg := utils.NewUSMEmptyConfig() + cfg.EnableNativeTLSMonitoring = true + cfg.EnableUSMEventStream = false + + if !usmconfig.TLSSupported(cfg) { + t.Skip("SSL maps cleaner not supported for this platform") + } + + monitor := setupUSMTLSMonitor(t, cfg, reInitEventConsumer) + require.NotNil(t, monitor) + + t.Run("SSL maps cleaner", func(t *testing.T) { + cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) + cleanProtocolMaps(t, "bio_new_socket_args", monitor.ebpfProgram.Manager.Manager) + + // find maps by names + maps := getMaps(t, monitor, sslPidKeyMaps) + require.Equal(t, len(maps), 6) + + // add random pid to the maps + pid := 100 + addPidEntryToMaps(t, maps, pid) + checkPidExistsInMaps(t, monitor, maps, pid) + + // verify that map is empty after cleaning up terminated processes + cleanDeadPidsInSslMaps(t, monitor) + checkPidNotFoundInMaps(t, monitor, maps, pid) + + // start dummy program and add its pid to the map + cmd, cancel := startDummyProgram(t) + addPidEntryToMaps(t, maps, cmd.Process.Pid) + checkPidExistsInMaps(t, monitor, maps, cmd.Process.Pid) + + // verify exit of process cleans the map + cancel() + _ = cmd.Wait() + checkPidNotFoundInMaps(t, monitor, maps, cmd.Process.Pid) + }) +} + +// getMaps returns eBPF maps searched by names. +func getMaps(t *testing.T, monitor *Monitor, mapNames []string) []*ebpf.Map { + maps := make([]*ebpf.Map, 0, len(mapNames)) + for _, mapName := range mapNames { + emap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap(mapName) + require.NotNil(t, emap) + maps = append(maps, emap) + } + return maps +} + +// addPidEntryToMaps adds an entry to maps using the PID as a key. +func addPidEntryToMaps(t *testing.T, maps []*ebpf.Map, pid int) { + for _, m := range maps { + require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") + + // make the key for single thread process when pid and tgid are the same + key := uint64(pid)<<32 | uint64(pid) + value := make([]byte, m.ValueSize()) + + err := m.Put(&key, value) + require.NoError(t, err) + } +} + +// checkPidExistsInMaps checks that pid exists in all provided maps. +func checkPidExistsInMaps(t *testing.T, monitor *Monitor, maps []*ebpf.Map, pid int) { + // make the key for single thread process when pid and tgid are the same + key := uint64(pid)<<32 | uint64(pid) + + for _, m := range maps { + require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") + mapInfo, err := m.Info() + require.NoError(t, err) + + require.Eventually(t, func() bool { + return findKeyInMap(m, key) + }, 1*time.Second, 100*time.Millisecond) + if t.Failed() { + t.Logf("pid '%d' not found in the map '%s'", pid, mapInfo.Name) + ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, mapInfo.Name) + } + } +} + +// checkPidNotFoundInMaps checks that pid does not exist in all provided maps. +func checkPidNotFoundInMaps(t *testing.T, monitor *Monitor, maps []*ebpf.Map, pid int) { + // make the key for single thread process when pid and tgid are the same + key := uint64(pid)<<32 | uint64(pid) + + for _, m := range maps { + require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") + mapInfo, err := m.Info() + require.NoError(t, err) + + if findKeyInMap(m, key) == true { + t.Logf("pid '%d' was found in the map '%s'", pid, mapInfo.Name) + ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, mapInfo.Name) + } + } +} + +// findKeyInMap returns true if 'theKey' was found in the map, otherwise returns false. +func findKeyInMap(m *ebpf.Map, theKey uint64) bool { + var key uint64 + value := make([]byte, m.ValueSize()) + iter := m.Iterate() + + for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { + if key == theKey { + return true + } + } + return false +} + +// startDummyProgram starts sleeping thread. +func startDummyProgram(t *testing.T) (*exec.Cmd, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(func() { cancel() }) + + cmd := exec.CommandContext(ctx, "sleep", "1000") + err := cmd.Start() + require.NoError(t, err) + + return cmd, cancel +} + +// cleanDeadPidsInSslMap delete terminated pid entries in the SSL maps. +func cleanDeadPidsInSslMaps(t *testing.T, monitor *Monitor) { + for _, mapName := range sslPidKeyMaps { + err := deleteDeadPidsInMap(monitor.ebpfProgram.Manager.Manager, mapName, nil) + if err != nil { + require.NoError(t, err) + } + } +} diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index b247396abb87fb..df7a2692be4754 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -10,7 +10,6 @@ package usm import ( "bufio" "bytes" - "context" "crypto/tls" "fmt" "io" @@ -25,9 +24,7 @@ import ( "sync" "testing" "time" - "unsafe" - "github.com/cilium/ebpf" krpretty "github.com/kr/pretty" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -1000,148 +997,3 @@ func (s *tlsSuite) TestNodeJSTLS() { } } } - -// TestSSLMapsCleaner verifies that SSL-related kernel maps are cleared correctly. -// the map entry is deleted when the thread exits, and periodic map cleaner removes dead threads. -func (s *tlsSuite) TestSSLMapsCleaner() { - t := s.T() - // setup monitor - cfg := utils.NewUSMEmptyConfig() - cfg.EnableNativeTLSMonitoring = true - cfg.EnableUSMEventStream = false - - monitor := setupUSMTLSMonitor(t, cfg, reInitEventConsumer) - require.NotNil(t, monitor) - - t.Run("SSL maps cleaner", func(t *testing.T) { - cleanProtocolMaps(t, "ssl", monitor.ebpfProgram.Manager.Manager) - cleanProtocolMaps(t, "bio_new_socket_args", monitor.ebpfProgram.Manager.Manager) - - // find maps by names - maps := getMaps(t, monitor, sslPidKeyMaps) - require.Equal(t, len(maps), 6) - - // add random pid to the maps - pid := 100 - addPidEntryToMaps(t, maps, pid) - checkPidExistsInMaps(t, monitor, maps, pid) - - // call SSL map cleaner and verify that map is empty - monitor.ebpfProgram.cleanDeadPidsInSslMaps() - checkPidNotFoundInMaps(t, monitor, maps, pid) - - // start dummy program and add its pid to the map - cmd, cancel := startDummyProgram(t) - addPidEntryToMaps(t, maps, cmd.Process.Pid) - checkPidExistsInMaps(t, monitor, maps, cmd.Process.Pid) - - // verify exit of process cleans the map - cancel() - _ = cmd.Wait() - checkPidNotFoundInMaps(t, monitor, maps, cmd.Process.Pid) - }) -} - -// getMaps returns eBPF maps searched by names. -func getMaps(t *testing.T, monitor *Monitor, mapNames []string) []*ebpf.Map { - var maps []*ebpf.Map - for _, mapName := range mapNames { - emap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap(mapName) - require.NotNil(t, emap) - maps = append(maps, emap) - } - return maps -} - -// addPidEntryToMaps adds an entry to maps using the PID as a key. -func addPidEntryToMaps(t *testing.T, maps []*ebpf.Map, pid int) { - for _, m := range maps { - require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") - - // make the key for single thread process when pid and tgid are the same - key := uint64(pid)<<32 | uint64(pid) - value := make([]byte, m.ValueSize()) - - err := m.Put(&key, value) - require.NoError(t, err) - } -} - -// checkPidExistsInMaps checks that pid exists in all provided maps. -func checkPidExistsInMaps(t *testing.T, monitor *Monitor, maps []*ebpf.Map, pid int) { - // make the key for single thread process when pid and tgid are the same - key := uint64(pid)<<32 | uint64(pid) - - for _, m := range maps { - require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") - mapInfo, err := m.Info() - require.NoError(t, err) - - require.Eventually(t, func() bool { - return findKeyInMap(m, key) - }, 1*time.Second, 100*time.Millisecond) - if t.Failed() { - t.Logf("pid '%d' not found in the map '%s'", pid, mapInfo.Name) - ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, mapInfo.Name) - } - } -} - -// checkPidNotFoundInMaps checks that pid does not exist in all provided maps. -func checkPidNotFoundInMaps(t *testing.T, monitor *Monitor, maps []*ebpf.Map, pid int) { - // make the key for single thread process when pid and tgid are the same - key := uint64(pid)<<32 | uint64(pid) - - for _, m := range maps { - require.Equal(t, m.KeySize(), uint32(unsafe.Sizeof(uint64(0))), "wrong key size") - mapInfo, err := m.Info() - require.NoError(t, err) - - if findKeyInMap(m, key) == true { - t.Logf("pid '%d' was found in the map '%s'", pid, mapInfo.Name) - ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, mapInfo.Name) - } - } -} - -// findKeyInMap returns true if 'key' was found in the map, otherwise returns false. -func findKeyInMap(m *ebpf.Map, theKey uint64) bool { - var key uint64 - value := make([]byte, m.ValueSize()) - iter := m.Iterate() - - for iter.Next(unsafe.Pointer(&key), unsafe.Pointer(&value)) { - if key == theKey { - return true - } - } - return false -} - -// startDummyProgram starts sleeping thread. -func startDummyProgram(t *testing.T) (*exec.Cmd, context.CancelFunc) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(func() { cancel() }) - - cmd := exec.CommandContext(ctx, "sleep", "1000") - err := cmd.Start() - require.NoError(t, err) - - return cmd, cancel -} - -// cleanDeadPidsInSslMap finds activated 'sslProgram' and calls map cleaner. -func (e *ebpfProgram) cleanDeadPidsInSslMaps() { - for _, prot := range e.enabledProtocols { - if prot.Instance.Name() == "openssl" { - switch prot.Instance.(type) { - case *sslProgram: - p, ok := prot.Instance.(*sslProgram) - if ok { - p.cleanupDeadPids(nil) - } - default: - } - } - } -} diff --git a/pkg/network/usm/sharedlibraries/watcher.go b/pkg/network/usm/sharedlibraries/watcher.go index e1aa0e96298131..802e1d8affde88 100644 --- a/pkg/network/usm/sharedlibraries/watcher.go +++ b/pkg/network/usm/sharedlibraries/watcher.go @@ -17,9 +17,6 @@ import ( "time" "unsafe" - "github.com/cilium/ebpf" - "github.com/cilium/ebpf/features" - "github.com/DataDog/datadog-agent/pkg/network/config" "github.com/DataDog/datadog-agent/pkg/network/protocols/telemetry" "github.com/DataDog/datadog-agent/pkg/network/usm/consts" @@ -354,8 +351,7 @@ func (w *Watcher) sync() { _ = w.registry.Unregister(pid) } - if w.mapsCleaner != nil && features.HaveProgramType(ebpf.RawTracepoint) != nil { - // call maps cleaner if raw tracepoints are not supported + if w.mapsCleaner != nil { w.mapsCleaner(alivePIDs) } } From 37b688c55685606083956a159733b3b46b8ff626 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Mon, 24 Feb 2025 10:41:54 -0500 Subject: [PATCH 19/22] [usm] fix linter error in ebpf_ssl.go --- pkg/network/usm/ebpf_ssl.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index c93ee9efb1b4cf..ea527ac6b4c934 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -517,7 +517,12 @@ func (o *sslProgram) ConfigureOptions(m *manager.Manager, options *manager.Optio // PreStart is called before the start of the provided eBPF manager. func (o *sslProgram) PreStart(*manager.Manager) error { - o.watcher.Start(o.cleanupDeadPids) + if features.HaveProgramType(ebpf.RawTracepoint) != nil { + o.watcher.Start(o.cleanupDeadPids) + } else { + o.watcher.Start(nil) + } + o.istioMonitor.Start() o.nodeJSMonitor.Start() return nil @@ -895,7 +900,7 @@ func deleteDeadPidsInMap(manager *manager.Manager, mapName string, alivePIDs map } } for _, k := range keysToDelete { - emap.Delete(&k) + _ = emap.Delete(&k) } return nil From 51cf027047d316203c11281eb66736c14bacf207 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Tue, 25 Feb 2025 13:11:04 -0500 Subject: [PATCH 20/22] [usm] enhanced UT TestSSLMapsCleaner() --- pkg/network/usm/ebpf_main.go | 2 +- pkg/network/usm/ebpf_ssl.go | 23 +++++++++--------- pkg/network/usm/ebpf_ssl_test.go | 40 +++++++++++++++++--------------- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/pkg/network/usm/ebpf_main.go b/pkg/network/usm/ebpf_main.go index 2da22a10775416..58df837dd62754 100644 --- a/pkg/network/usm/ebpf_main.go +++ b/pkg/network/usm/ebpf_main.go @@ -14,6 +14,7 @@ import ( "slices" "unsafe" + manager "github.com/DataDog/ebpf-manager" "github.com/cilium/ebpf" "github.com/davecgh/go-spew/spew" @@ -35,7 +36,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/usm/buildmode" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" "github.com/DataDog/datadog-agent/pkg/util/log" - manager "github.com/DataDog/ebpf-manager" ) var ( diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index ea527ac6b4c934..d85c2f92b72e95 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -20,6 +20,7 @@ import ( "time" "unsafe" + manager "github.com/DataDog/ebpf-manager" "github.com/cilium/ebpf" "github.com/cilium/ebpf/features" "github.com/davecgh/go-spew/spew" @@ -38,7 +39,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/util/safeelf" ddsync "github.com/DataDog/datadog-agent/pkg/util/sync" - manager "github.com/DataDog/ebpf-manager" ) const ( @@ -503,7 +503,7 @@ func (o *sslProgram) Name() string { } // ConfigureOptions changes map attributes to the given options. -func (o *sslProgram) ConfigureOptions(m *manager.Manager, options *manager.Options) { +func (o *sslProgram) ConfigureOptions(_ *manager.Manager, options *manager.Options) { options.MapSpecEditors[sslSockByCtxMap] = manager.MapSpecEditor{ MaxEntries: o.cfg.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries, @@ -512,17 +512,16 @@ func (o *sslProgram) ConfigureOptions(m *manager.Manager, options *manager.Optio MaxEntries: o.cfg.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries, } - o.addProcessExitProbe(m, options) + o.addProcessExitProbe(options) } // PreStart is called before the start of the provided eBPF manager. func (o *sslProgram) PreStart(*manager.Manager) error { + var cleanerCB func(map[uint32]struct{}) = nil if features.HaveProgramType(ebpf.RawTracepoint) != nil { - o.watcher.Start(o.cleanupDeadPids) - } else { - o.watcher.Start(nil) + cleanerCB = o.cleanupDeadPids } - + o.watcher.Start(cleanerCB) o.istioMonitor.Start() o.nodeJSMonitor.Start() return nil @@ -829,7 +828,7 @@ func (*sslProgram) IsBuildModeSupported(buildmode.Type) bool { } // addProcessExitProbe adds a raw or regular tracepoint program depending on which is supported. -func (o *sslProgram) addProcessExitProbe(mgr *manager.Manager, options *manager.Options) { +func (o *sslProgram) addProcessExitProbe(options *manager.Options) { if features.HaveProgramType(ebpf.RawTracepoint) == nil { // use a raw tracepoint on a supported kernel to intercept terminated threads and clear the corresponding maps p := &manager.Probe{ @@ -839,7 +838,7 @@ func (o *sslProgram) addProcessExitProbe(mgr *manager.Manager, options *manager. }, TracepointName: "sched_process_exit", } - mgr.Probes = append(mgr.Probes, p) + o.ebpfManager.Probes = append(o.ebpfManager.Probes, p) options.ActivatedProbes = append(options.ActivatedProbes, &manager.ProbeSelector{ProbeIdentificationPair: p.ProbeIdentificationPair}) // exclude regular tracepoint options.ExcludedFunctions = append(options.ExcludedFunctions, oldTracepointSchedProcessExit) @@ -851,7 +850,7 @@ func (o *sslProgram) addProcessExitProbe(mgr *manager.Manager, options *manager. UID: probeUID, }, } - mgr.Probes = append(mgr.Probes, p) + o.ebpfManager.Probes = append(o.ebpfManager.Probes, p) options.ActivatedProbes = append(options.ActivatedProbes, &manager.ProbeSelector{ProbeIdentificationPair: p.ProbeIdentificationPair}) // exclude a raw tracepoint options.ExcludedFunctions = append(options.ExcludedFunctions, rawTracepointSchedProcessExit) @@ -869,12 +868,12 @@ var sslPidKeyMaps = []string{ // cleanupDeadPids clears maps of terminated processes. func (o *sslProgram) cleanupDeadPids(alivePIDs map[uint32]struct{}) { - if o.ebpfManager != nil && features.HaveProgramType(ebpf.RawTracepoint) != nil { + if features.HaveProgramType(ebpf.RawTracepoint) != nil { // call maps cleaner if raw tracepoints are not supported for _, mapName := range sslPidKeyMaps { err := deleteDeadPidsInMap(o.ebpfManager, mapName, alivePIDs) if err != nil { - log.Debugf("SSL maps %q cleanup error: %v", mapName, err) + log.Debugf("SSL map %q cleanup error: %v", mapName, err) } } } diff --git a/pkg/network/usm/ebpf_ssl_test.go b/pkg/network/usm/ebpf_ssl_test.go index 141ed0897c6e72..b045be447a51cc 100644 --- a/pkg/network/usm/ebpf_ssl_test.go +++ b/pkg/network/usm/ebpf_ssl_test.go @@ -18,10 +18,12 @@ import ( "time" "unsafe" - "github.com/DataDog/datadog-agent/pkg/ebpf/ebpftest" + manager "github.com/DataDog/ebpf-manager" "github.com/cilium/ebpf" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/pkg/ebpf/ebpftest" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" usmconfig "github.com/DataDog/datadog-agent/pkg/network/usm/config" "github.com/DataDog/datadog-agent/pkg/network/usm/consts" @@ -92,35 +94,35 @@ func TestSSLMapsCleaner(t *testing.T) { cleanProtocolMaps(t, "bio_new_socket_args", monitor.ebpfProgram.Manager.Manager) // find maps by names - maps := getMaps(t, monitor, sslPidKeyMaps) + maps := getMaps(t, monitor.ebpfProgram.Manager.Manager, sslPidKeyMaps) require.Equal(t, len(maps), 6) // add random pid to the maps pid := 100 addPidEntryToMaps(t, maps, pid) - checkPidExistsInMaps(t, monitor, maps, pid) + checkPidExistsInMaps(t, monitor.ebpfProgram.Manager.Manager, maps, pid) // verify that map is empty after cleaning up terminated processes - cleanDeadPidsInSslMaps(t, monitor) - checkPidNotFoundInMaps(t, monitor, maps, pid) + cleanDeadPidsInSslMaps(t, monitor.ebpfProgram.Manager.Manager) + checkPidNotFoundInMaps(t, monitor.ebpfProgram.Manager.Manager, maps, pid) // start dummy program and add its pid to the map cmd, cancel := startDummyProgram(t) addPidEntryToMaps(t, maps, cmd.Process.Pid) - checkPidExistsInMaps(t, monitor, maps, cmd.Process.Pid) + checkPidExistsInMaps(t, monitor.ebpfProgram.Manager.Manager, maps, cmd.Process.Pid) // verify exit of process cleans the map cancel() _ = cmd.Wait() - checkPidNotFoundInMaps(t, monitor, maps, cmd.Process.Pid) + checkPidNotFoundInMaps(t, monitor.ebpfProgram.Manager.Manager, maps, cmd.Process.Pid) }) } // getMaps returns eBPF maps searched by names. -func getMaps(t *testing.T, monitor *Monitor, mapNames []string) []*ebpf.Map { +func getMaps(t *testing.T, manager *manager.Manager, mapNames []string) []*ebpf.Map { maps := make([]*ebpf.Map, 0, len(mapNames)) for _, mapName := range mapNames { - emap, _, _ := monitor.ebpfProgram.Manager.Manager.GetMap(mapName) + emap, _, _ := manager.GetMap(mapName) require.NotNil(t, emap) maps = append(maps, emap) } @@ -142,7 +144,7 @@ func addPidEntryToMaps(t *testing.T, maps []*ebpf.Map, pid int) { } // checkPidExistsInMaps checks that pid exists in all provided maps. -func checkPidExistsInMaps(t *testing.T, monitor *Monitor, maps []*ebpf.Map, pid int) { +func checkPidExistsInMaps(t *testing.T, manager *manager.Manager, maps []*ebpf.Map, pid int) { // make the key for single thread process when pid and tgid are the same key := uint64(pid)<<32 | uint64(pid) @@ -151,18 +153,19 @@ func checkPidExistsInMaps(t *testing.T, monitor *Monitor, maps []*ebpf.Map, pid mapInfo, err := m.Info() require.NoError(t, err) - require.Eventually(t, func() bool { + assert.Eventually(t, func() bool { return findKeyInMap(m, key) }, 1*time.Second, 100*time.Millisecond) if t.Failed() { t.Logf("pid '%d' not found in the map '%s'", pid, mapInfo.Name) - ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, mapInfo.Name) + ebpftest.DumpMapsTestHelper(t, manager.DumpMaps, mapInfo.Name) + t.FailNow() } } } // checkPidNotFoundInMaps checks that pid does not exist in all provided maps. -func checkPidNotFoundInMaps(t *testing.T, monitor *Monitor, maps []*ebpf.Map, pid int) { +func checkPidNotFoundInMaps(t *testing.T, manager *manager.Manager, maps []*ebpf.Map, pid int) { // make the key for single thread process when pid and tgid are the same key := uint64(pid)<<32 | uint64(pid) @@ -173,7 +176,8 @@ func checkPidNotFoundInMaps(t *testing.T, monitor *Monitor, maps []*ebpf.Map, pi if findKeyInMap(m, key) == true { t.Logf("pid '%d' was found in the map '%s'", pid, mapInfo.Name) - ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, mapInfo.Name) + ebpftest.DumpMapsTestHelper(t, manager.DumpMaps, mapInfo.Name) + t.FailNow() } } } @@ -205,11 +209,9 @@ func startDummyProgram(t *testing.T) (*exec.Cmd, context.CancelFunc) { } // cleanDeadPidsInSslMap delete terminated pid entries in the SSL maps. -func cleanDeadPidsInSslMaps(t *testing.T, monitor *Monitor) { +func cleanDeadPidsInSslMaps(t *testing.T, manager *manager.Manager) { for _, mapName := range sslPidKeyMaps { - err := deleteDeadPidsInMap(monitor.ebpfProgram.Manager.Manager, mapName, nil) - if err != nil { - require.NoError(t, err) - } + err := deleteDeadPidsInMap(manager, mapName, nil) + require.NoError(t, err) } } From f666e460e80f073f989a159b989380254ae760b3 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Tue, 25 Feb 2025 13:21:48 -0500 Subject: [PATCH 21/22] [usm] call m.Put(unsafe.Pointer(&key), unsafe.Pointer(&value)) in TestSSLMapsCleaner() --- pkg/network/usm/ebpf_ssl_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/network/usm/ebpf_ssl_test.go b/pkg/network/usm/ebpf_ssl_test.go index b045be447a51cc..8a49db3daacdeb 100644 --- a/pkg/network/usm/ebpf_ssl_test.go +++ b/pkg/network/usm/ebpf_ssl_test.go @@ -138,7 +138,7 @@ func addPidEntryToMaps(t *testing.T, maps []*ebpf.Map, pid int) { key := uint64(pid)<<32 | uint64(pid) value := make([]byte, m.ValueSize()) - err := m.Put(&key, value) + err := m.Put(unsafe.Pointer(&key), unsafe.Pointer(&value)) require.NoError(t, err) } } From df55c29fb908b9fd97d4d6cf6b5e955fcf4b77aa Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" Date: Tue, 25 Feb 2025 14:02:18 -0500 Subject: [PATCH 22/22] [usm] drop = nil from declaration var cleanerCB func(). --- pkg/network/usm/ebpf_ssl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/network/usm/ebpf_ssl.go b/pkg/network/usm/ebpf_ssl.go index d85c2f92b72e95..3a00ea5a2df7df 100644 --- a/pkg/network/usm/ebpf_ssl.go +++ b/pkg/network/usm/ebpf_ssl.go @@ -517,7 +517,7 @@ func (o *sslProgram) ConfigureOptions(_ *manager.Manager, options *manager.Optio // PreStart is called before the start of the provided eBPF manager. func (o *sslProgram) PreStart(*manager.Manager) error { - var cleanerCB func(map[uint32]struct{}) = nil + var cleanerCB func(map[uint32]struct{}) if features.HaveProgramType(ebpf.RawTracepoint) != nil { cleanerCB = o.cleanupDeadPids }