Mostly a wip

Author: i3149

pkg/inputs/snmp/disco.go

@@ -87,20 +87,46 @@ func Discover(ctx context.Context, log logger.ContextL, pollDuration time.Durati
 	}
 	defer mdb.Close()
 
+	ignoreMap := map[string]bool{}
+	for _, ip := range conf.Disco.IgnoreList {
+		ignoreMap[ip] = true
+	}
+
+	foundDevices := map[string]*kt.SnmpDeviceConfig{}
 	if conf.Disco.NetboxAPIHost != "" {
-		err = getDevicesFromNetbox(ctx, conf, log)
+		log.Infof("Discovering devices from Netbox with tag %s", conf.Disco.NetboxTag)
+		err = getDevicesFromNetbox(ctx, ctl, foundDevices, mdb, conf, kentikDevices, log, ignoreMap)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		log.Infof("Discovering devices from network scan")
+		err := runScanCheckDisco(ctx, ctl, foundDevices, mdb, conf, kentikDevices, log, ignoreMap)
 		if err != nil {
 			return nil, err
 		}
-		return nil, nil
 	}
 
-	ignoreMap := map[string]bool{}
-	for _, ip := range conf.Disco.IgnoreList {
-		ignoreMap[ip] = true
+	var stats *SnmpDiscoDeviceStat
+	if conf.Disco.AddDevices {
+		stats, err = addDevices(ctx, foundDevices, snmpFile, conf, false, log, pollDuration)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	foundDevices := map[string]*kt.SnmpDeviceConfig{}
+	// Phone home if we have one of these set up.
+	if confMgr != nil && (stats.added > 0 || stats.replaced > 0) {
+		confMgr.DeviceDiscovery(conf.Devices)
+	}
+
+	time.Sleep(2 * time.Second) // Give logs time to get sent back.
+
+	return stats, nil
+}
+
+func runScanCheckDisco(ctx context.Context, ctl chan bool, foundDevices map[string]*kt.SnmpDeviceConfig,
+	mdb *mibs.MibDB, conf *kt.SnmpConfig, kentikDevices map[string]string, log logger.ContextL, ignoreMap map[string]bool) error {
 	for _, ipr := range conf.Disco.Cidrs {
 		_, _, err := net.ParseCIDR(ipr)
 		if err != nil {
@@ -121,11 +147,11 @@ func Discover(ctx context.Context, log logger.ContextL, pollDuration time.Durati
 		timeout := time.Millisecond * time.Duration(conf.Global.TimeoutMS)
 		scanner := scan.NewDeviceScanner(targetIterator, timeout)
 		if err := scanner.Start(); err != nil {
-			return nil, err
+			return err
 		}
 		results, err := scanner.Scan(ctx, conf.Disco.Ports)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		var wg sync.WaitGroup
 		var mux sync.RWMutex
@@ -145,22 +171,7 @@ func Discover(ctx context.Context, log logger.ContextL, pollDuration time.Durati
 		log.Infof("Checked %d ips in %v (from start: %v)", len(results), time.Now().Sub(st), time.Now().Sub(stb))
 	}
 
-	var stats *SnmpDiscoDeviceStat
-	if conf.Disco.AddDevices {
-		stats, err = addDevices(ctx, foundDevices, snmpFile, conf, false, log, pollDuration)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Phone home if we have one of these set up.
-	if confMgr != nil && (stats.added > 0 || stats.replaced > 0) {
-		confMgr.DeviceDiscovery(conf.Devices)
-	}
-
-	time.Sleep(2 * time.Second) // Give logs time to get sent back.
-
-	return stats, nil
+	return nil
 }
 
 func RunDiscoOnTimer(ctx context.Context, c chan os.Signal, log logger.ContextL, pollTimeMin int, checkNow bool, cfg *ktranslate.SNMPInputConfig, apic *api.KentikApi, confMgr config.ConfigManager) {

pkg/inputs/snmp/netbox.go

@@ -5,28 +5,73 @@ Use netbox API to pull down list of devices to target. Get all devices matching
 */
 import (
 	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"sync"
+	"time"
 
 	"github.com/kentik/ktranslate/pkg/eggs/logger"
+	"github.com/kentik/ktranslate/pkg/inputs/snmp/mibs"
 	"github.com/kentik/ktranslate/pkg/kt"
 
+	"github.com/liamg/furious/scan"
 	"github.com/netbox-community/go-netbox/v4"
 )
 
-func getDevicesFromNetbox(ctx context.Context, conf *kt.SnmpConfig, log logger.ContextL) error {
+func getDevicesFromNetbox(ctx context.Context, ctl chan bool, foundDevices map[string]*kt.SnmpDeviceConfig,
+	mdb *mibs.MibDB, conf *kt.SnmpConfig, kentikDevices map[string]string, log logger.ContextL, ignoreMap map[string]bool) error {
 	c := netbox.NewAPIClientFor(conf.Disco.NetboxAPIHost, conf.Disco.NetboxAPIToken)
 
+	limit := int32(0) // This implies we see all. Do we need to implement paging?
 	res, _, err := c.DcimAPI.
 		DcimDevicesList(ctx).
-		Status([]string{"active"}).
-		Limit(10).
+		Status([]string{"active"}). // Only look at active devices.
+		Limit(limit).
 		Execute()
 
 	if err != nil {
 		return err
 	}
 
-	log.Infof("%v", res.Results)
+	timeout := time.Millisecond * time.Duration(conf.Global.TimeoutMS)
+	stb := time.Now()
+	results := make([]scan.Result, 0)
+	for _, res := range res.Results {
+		hasTag := false
+		for _, tag := range res.Tags {
+			if tag.Display == conf.Disco.NetboxTag {
+				hasTag = true
+				continue
+			}
+		}
 
-	return nil
+		if hasTag && res.PrimaryIp.IsSet() {
+			addr := res.PrimaryIp.Get().GetAddress()
+			ipv, err := netip.ParsePrefix(addr)
+			if err != nil {
+				log.Infof("Skipping %s with bad PrimaryIP: %v", res.Display, addr)
+			} else {
+				results = append(results, scan.Result{Name: res.Display, Manufacturer: res.DeviceType.GetDisplay(), Host: net.ParseIP(ipv.Addr().String())})
+			}
+		}
+
+	}
 
+	var wg sync.WaitGroup
+	var mux sync.RWMutex
+	st := time.Now()
+	log.Infof("Starting to check %d ips from netbox", len(results))
+	for i, result := range results {
+		if ignoreMap[result.Host.String()] { // If we have marked this ip as to be ignored, don't do anything more with it.
+			continue
+		}
+		wg.Add(1)
+		posit := fmt.Sprintf("%d/%d)", i+1, len(results))
+		go doubleCheckHost(result, timeout, ctl, &mux, &wg, foundDevices, mdb, conf, posit, kentikDevices, log)
+	}
+	wg.Wait()
+	log.Infof("Checked %d ips in %v (from start: %v)", len(results), time.Now().Sub(st), time.Now().Sub(stb))
+
+	return nil
 }

pkg/kt/snmp.go

@@ -260,6 +260,7 @@ type SnmpDiscoConfig struct {
 	NoUseBulkWalkAll   bool            `yaml:"no_use_bulkwalkall"`
 	NetboxAPIHost      string          `yaml:"netbox_host"`
 	NetboxAPIToken     string          `yaml:"netbox_token"`
+	NetboxTag          string          `yaml:"netbox_tag"`
 }
 
 type ProviderMap struct {