feat: Add rate limiting, configurable workers and network optimizations

.gitignore

@@ -1 +1,4 @@
+.env
 dist/
+output/
+prometheus-net-discovery

.goreleaser.yaml

@@ -1,3 +1,5 @@
+version: 2
+
 project_name: prometheus-net-discovery
 
 # setups builds for linux and darwin on amd64 and arm64
@@ -53,6 +55,7 @@ signs:
       - sign-blob
       - "--output-certificate=${certificate}"
       - "--output-signature=${signature}"
+      - "--bundle=${artifact}.bundle"
       - "${artifact}"
       - "--yes" # needed on cosign 2.0.0+
     artifacts: checksum

Makefile

@@ -5,9 +5,8 @@ MAKEFLAGS += --no-print-directory
 IMAGE = quay.fnox.se/fortnox/prometheus-net-discovery
 VERSION?=0.0.1-local
 
-# Sets DOCKER_HOST to the Podman socket if it exists, otherwise defaults to the Docker socket
-export DOCKER_HOST := unix://$(or $(wildcard /run/user/$(shell id -u)/podman/podman.sock),/var/run/docker.sock)
-
+-include .env
+export
 
 build:
 	CGO_ENABLED=0 GOOS=linux go build
@@ -19,19 +18,7 @@ push: docker
 	docker push $(IMAGE):$(VERSION)
 
 localrun:
-	go run main.go
+	go run . 
 
 test:
 	go test ./... -count=1 -cover
-
-test-coverage:
-	go test ./... -count=1 -coverprofile=coverage.out
-	grep -vE "/mocks/|main.go" coverage.out > coverage-filtered.out
-	go tool cover -func=coverage-filtered.out
-
-lint:
-	@if command -v golangci-lint >/dev/null 2>&1; then \
-		golangci-lint run ./...; \
-	else \
-		echo "golangci-lint not installed, skipping lint"; \
-	fi

README.md

@@ -13,11 +13,41 @@ go get -u github.com/fortnoxab/prometheus-net-discovery
 prometheus-net-discovery -networks "192.168.1.0/24" --filesdpath /tmp/
 ```
 
+### Configuration options
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `-workers` | 16 | Number of concurrent workers |
+| `-scanratelimit` | 50 | Maximum hosts to probe per second |
+| `-skipnetworkbroadcast` | true | Skip network and broadcast addresses |
+| `-exporterexporterport` | 9999 | Port where exporter_exporter is listening |
+| `-port` | 8080 | Port for the internal webserver |
+| `-interval` | 60m | How often to scan |
+
+#### Example configurations
+
+Default scanning:
+```
+prometheus-net-discovery -networks "192.168.1.0/24" -filesdpath /tmp/
+```
+
+Gentle scanning for sensitive networks:
+```
+prometheus-net-discovery -networks "192.168.1.0/24" -filesdpath /tmp/ -workers 8 -scanratelimit 25
+```
+
+Fast scanning for trusted networks:
+```
+prometheus-net-discovery -networks "10.0.0.0/16" -filesdpath /tmp/ -workers 32 -scanratelimit 100
+```
+
 ### usage
 
 ```
 $ prometheus-net-discovery --help
 Usage of prometheus-net-discovery:
+  -exporterexporterport
+    	Change value of ExporterExporterPort. (default 9999)
   -filesdpath
     	Change value of FileSdPath.
   -interval
@@ -30,13 +60,26 @@ Usage of prometheus-net-discovery:
     	Change value of Log-Level.
   -networks
     	Change value of Networks.
+  -port
+    	Change value of Port. (default 8080)
+  -workers
+    	Change value of Workers. (default 16)
+  -scanratelimit
+    	Change value of ScanRateLimit. (default 50)
+  -skipnetworkbroadcast
+    	Change value of SkipNetworkBroadcast. (default true)
 
 Generated environment variables:
+   CONFIG_EXPORTEREXPORTERPORT
    CONFIG_FILESDPATH
    CONFIG_INTERVAL
    CONFIG_LOG_FORMAT
    CONFIG_LOG_FORMATTER
    CONFIG_LOG_LEVEL
    CONFIG_NETWORKS
+   CONFIG_PORT
+   CONFIG_WORKERS
+   CONFIG_SCANRATELIMIT
+   CONFIG_SKIPNETWORKBROADCAST
 
 ```

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/prometheus/client_golang v1.23.2
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.11.1
+	golang.org/x/time v0.5.0
 )
 
 require (

go.sum

@@ -55,6 +55,8 @@ go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
 golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/fortnoxab/fnxlogrus"
 	"github.com/koding/multiconfig"
 	"github.com/sirupsen/logrus"
+	"golang.org/x/time/rate"
 )
 
 // ExporterConfig configures ports to scan to what filename to save it to.
@@ -43,7 +44,7 @@ var mutex sync.Mutex
 func main() {
 	config := &Config{}
 	multiconfig.MustLoad(&config)
-	exporterConfig[0].port = config.ExpoterExporterPort
+	exporterConfig[0].port = config.ExporterExporterPort
 
 	fnxlogrus.Init(config.Log, logrus.StandardLogger())
 
@@ -85,8 +86,13 @@ func runDiscovery(parentCtx context.Context, config *Config, networks []string)
 	exporter := make(chan *Address)
 	ctx, cancel := context.WithCancel(parentCtx)
 	defer cancel()
+
+	// Rate limiter for HTTP probes (still useful to spread load)
+	limiter := rate.NewLimiter(rate.Limit(config.ScanRateLimit), 1)
+
 	var wg sync.WaitGroup
-	for i := 0; i < 128; i++ {
+	workerCount := config.Workers
+	for i := 0; i < workerCount; i++ {
 		wg.Add(1)
 		i := i
 		go func() {
@@ -98,6 +104,10 @@ func runDiscovery(parentCtx context.Context, config *Config, networks []string)
 						logrus.Debugf("worker %d finished", i)
 						return
 					}
+					// Wait for rate limiter before processing
+					if err := limiter.Wait(ctx); err != nil {
+						return
+					}
 					fn(ctx)
 				case <-ctx.Done():
 					return
@@ -115,7 +125,7 @@ func runDiscovery(parentCtx context.Context, config *Config, networks []string)
 			if network == "" {
 				continue
 			}
-			discoverNetwork(network, job, exporter)
+			discoverNetwork(network, job, exporter, config)
 		}
 		close(job)
 	}()
@@ -139,6 +149,7 @@ func runDiscovery(parentCtx context.Context, config *Config, networks []string)
 	wg.Wait()
 
 	saveConfigs(ctx, config, exporters)
+
 	logrus.Info("discovery done")
 }
 
@@ -163,12 +174,46 @@ func isVip(name string) bool {
 	return vipRegexp.MatchString(name)
 }
 
-func discoverNetwork(network string, queue chan func(context.Context), exporter chan *Address) {
+// isNetworkOrBroadcast checks if the IP is the network address or broadcast address of the given subnet.
+// Note: IPv6 addresses always return false as IPv6 doesn't have broadcast addresses.
+// For IPv6, this function only filters the all-zeros network address.
+func isNetworkOrBroadcast(ip net.IP, ipnet *net.IPNet) bool {
+	// Try IPv4 first
+	ipv4 := ip.To4()
+	if ipv4 != nil {
+		// Get network address
+		networkIP := ipv4.Mask(ipnet.Mask)
+
+		// Calculate broadcast address
+		broadcast := make(net.IP, len(networkIP))
+		for i := range networkIP {
+			broadcast[i] = networkIP[i] | ^ipnet.Mask[i]
+		}
+
+		return ipv4.Equal(networkIP) || ipv4.Equal(broadcast)
+	}
+
+	// For IPv6, check if it's the network address (IPv6 doesn't have broadcast)
+	networkIP := ip.Mask(ipnet.Mask)
+	return ip.Equal(networkIP)
+}
+
+func discoverNetwork(network string, queue chan func(context.Context), exporter chan *Address, config *Config) {
 	networkip, ipnet, err := net.ParseCIDR(network)
 	if err != nil {
 		log.Fatal("network CIDR could not be parsed:", err)
 	}
+
+	skippedCount := 0
 	for ip := networkip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
+		// Skip network and broadcast addresses if configured
+		currentIP := make(net.IP, len(ip))
+		copy(currentIP, ip)
+		if config.SkipNetworkBroadcast && isNetworkOrBroadcast(currentIP, ipnet) {
+			skippedCount++
+			continue
+		}
+
 		network := network
 		ip := ip.String()
 		queue <- func(ctx context.Context) {
@@ -189,10 +234,10 @@ func discoverNetwork(network string, queue chan func(context.Context), exporter
 				}
 
 				logrus.Info(net.JoinHostPort(ip, port), " is alive")
-				addr, _ := net.LookupAddr(ip) // #nosec
+				addr, _ := net.DefaultResolver.LookupAddr(ctx, ip) // #nosec
 				hostname := strings.TrimRight(getFirst(addr), ".")
 				if hostname == "" {
-					logrus.Error("missing reverse record for ", ip)
+					logrus.Debugf("skipping %s: missing reverse record", ip)
 					continue
 				}
 				if isVip(hostname) && !strings.HasPrefix(hostname, "k8s-") {
@@ -215,6 +260,9 @@ func discoverNetwork(network string, queue chan func(context.Context), exporter
 			}
 		}
 	}
+	if skippedCount > 0 {
+		logrus.Debugf("Skipped %d network/broadcast addresses in %s", skippedCount, network)
+	}
 }
 
 func getOldGroups(path string) ([]Group, error) {
@@ -254,7 +302,7 @@ func writeFileSDConfig(config *Config, exporterName string, addresses []Address)
 				"host":   v.Hostname,
 			},
 		}
-		if v.Port == config.ExpoterExporterPort {
+		if v.Port == config.ExporterExporterPort {
 			group.Labels["__metrics_path__"] = "/proxy"
 			group.Labels["__param_module"] = exporterName
 		}

models.go

@@ -1,6 +1,8 @@
 package main
 
-import "github.com/fortnoxab/fnxlogrus"
+import (
+	"github.com/fortnoxab/fnxlogrus"
+)
 
 // Config is main application configuration.
 type Config struct {
@@ -12,7 +14,13 @@ type Config struct {
 	FileSdPath          string
 	Log                 fnxlogrus.Config
 	Port                string `default:"8080"`
-	ExpoterExporterPort string `default:"9999"`
+	ExporterExporterPort string `default:"9999"`
+	// Workers is the number of concurrent workers for scanning. Default 16
+	Workers int `default:"16"`
+	// ScanRateLimit is the maximum number of hosts to scan per second. Default 50 to reduce network load
+	ScanRateLimit float64 `default:"50"`
+	// SkipNetworkBroadcast skips network and broadcast addresses. Default true
+	SkipNetworkBroadcast bool `default:"true"`
 }
 
 // Exporters is a list of addresses grouped by exporter name.