From b6405d86f93c8886a3b268cf2c9beec37bb20632 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 16 Jul 2025 22:58:59 +0200 Subject: [PATCH 1/5] optimizations --- pkg/massdns/massdns.go | 2 ++ pkg/massdns/process.go | 47 ++++++++++++++++++++++------- pkg/runner/options.go | 2 ++ pkg/runner/runner.go | 1 + pkg/store/store.go | 68 +++++++++++++++++++++++++++++++++++------- 5 files changed, 99 insertions(+), 21 deletions(-) diff --git a/pkg/massdns/massdns.go b/pkg/massdns/massdns.go index 4c05b8b..e07896c 100644 --- a/pkg/massdns/massdns.go +++ b/pkg/massdns/massdns.go @@ -46,6 +46,8 @@ type Options struct { WildcardOutputFile string // MassDnsCmd supports massdns flags MassDnsCmd string + // KeepStderr controls whether to capture and store massdns stderr output + KeepStderr bool OnResult func(*retryabledns.DNSData) } diff --git a/pkg/massdns/process.go b/pkg/massdns/process.go index 0edb1ae..8c42230 100644 --- a/pkg/massdns/process.go +++ b/pkg/massdns/process.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "log" "os" "os/exec" "strconv" @@ -30,32 +31,54 @@ import ( func (instance *Instance) RunWithContext(ctx context.Context) (stdout, stderr string, took time.Duration, err error) { start := time.Now() + // Create temporary file for massdns output stdoutFile, err := os.CreateTemp(instance.options.TempDir, "massdns-stdout-") if err != nil { - return "", "", 0, fmt.Errorf("could not create temp file for massdns stdout: %w", err) + return "", "", 0, fmt.Errorf("could not create temp file for massdns output: %w", err) } defer func() { _ = stdoutFile.Close() }() - stderrFile, err := os.CreateTemp(instance.options.TempDir, "massdns-stderr-") - if err != nil { - return "", "", 0, fmt.Errorf("could not create temp file for massdns stdout: %w", err) + // Handle stderr based on KeepStderr option + var stderrFile *os.File + if instance.options.KeepStderr { + stderrFile, err = os.CreateTemp(instance.options.TempDir, "massdns-stderr-") + if err != nil { + return "", "", 0, fmt.Errorf("could not create temp file for massdns stderr: %w", err) + } + defer func() { + _ = stderrFile.Close() + }() } - defer func() { - _ = stderrFile.Close() - }() // Run the command on a temp file and wait for the output args := []string{"-r", instance.options.ResolversFile, "-o", "Snl", "--retry", "REFUSED", "--retry", "SERVFAIL", "-t", "A", instance.options.InputFile, "-s", strconv.Itoa(instance.options.Threads)} if instance.options.MassDnsCmd != "" { args = append(args, strings.Split(instance.options.MassDnsCmd, " ")...) } + + log.Fatalf("flag: %s %s", instance.options.MassdnsPath, strings.Join(args, " ")) cmd := exec.CommandContext(ctx, instance.options.MassdnsPath, args...) cmd.Stdout = stdoutFile - cmd.Stderr = stderrFile + + // Set stderr based on KeepStderr option + if instance.options.KeepStderr { + cmd.Stderr = stderrFile + } else { + // Discard stderr by sending it to /dev/null + cmd.Stderr = nil + } + err = cmd.Run() - return stdoutFile.Name(), stderrFile.Name(), time.Since(start), err + + // Return stderr filename only if it was captured + stderrFilename := "" + if instance.options.KeepStderr { + stderrFilename = stderrFile.Name() + } + + return stdoutFile.Name(), stderrFilename, time.Since(start), err } func (instance *Instance) Run(ctx context.Context) error { @@ -96,7 +119,11 @@ func (instance *Instance) Run(ctx context.Context) error { gologger.Info().Msgf("using massdns output directory: %s\n", tmpDir) stdoutFile, stderrFile, took, err := instance.RunWithContext(ctx) gologger.Info().Msgf("massdns output file: %s\n", stdoutFile) - gologger.Info().Msgf("massdns error file: %s\n", stderrFile) + if stderrFile != "" { + gologger.Info().Msgf("massdns error file: %s\n", stderrFile) + } else { + gologger.Info().Msgf("massdns stderr discarded (KeepStderr=false)\n") + } if err != nil { return fmt.Errorf("could not execute massdns: %s", err) } diff --git a/pkg/runner/options.go b/pkg/runner/options.go index e7a78df..777e253 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -35,6 +35,7 @@ type Options struct { MassDnsCmd string // Supports massdns flags(example -i) DisableUpdateCheck bool // DisableUpdateCheck disable automatic update check Mode string + KeepStderr bool // KeepStderr controls whether to capture and store massdns stderr output OnResult func(*retryabledns.DNSData) } @@ -88,6 +89,7 @@ func ParseOptions() *Options { flagSet.IntVar(&options.Retries, "retries", 5, "Number of retries for dns enumeration"), flagSet.BoolVarP(&options.StrictWildcard, "strict-wildcard", "sw", false, "Perform wildcard check on all found subdomains"), flagSet.IntVar(&options.WildcardThreads, "wt", 250, "Number of concurrent wildcard checks"), + flagSet.BoolVar(&options.KeepStderr, "retain-stderr", false, "Capture and store massdns stderr output (default: discard)"), ) flagSet.CreateGroup("debug", "Debug", diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index ac70d80..3e63769 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -185,6 +185,7 @@ func (r *Runner) runMassdns(inputFile string) { StrictWildcard: r.options.StrictWildcard, WildcardOutputFile: r.options.WildcardOutputFile, MassDnsCmd: r.options.MassDnsCmd, + KeepStderr: r.options.KeepStderr, OnResult: r.options.OnResult, }) if err != nil { diff --git a/pkg/store/store.go b/pkg/store/store.go index 2632b44..c6c1aed 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -1,10 +1,11 @@ package store import ( + "encoding/json" "os" "strings" - sliceutil "github.com/projectdiscovery/utils/slice" + mapsutil "github.com/projectdiscovery/utils/maps" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/opt" ) @@ -23,7 +24,13 @@ func New(dbPath string) (*Store, error) { return nil, err } db, err := leveldb.OpenFile(storeDb, &opt.Options{ - CompactionTableSize: 256 * Megabyte, + // Optimize for disk space reduction + CompactionTableSize: 64 * Megabyte, // Reduced from 256MB for more frequent compaction + WriteBuffer: 2 * Megabyte, // Reduced from 4MB for more frequent flushing + WriteL0SlowdownTrigger: 4, // Trigger slowdown earlier + WriteL0PauseTrigger: 8, // Trigger pause earlier + BlockSize: 2 * 1024, // Reduced from 4KB for better compression of small records + BlockCacheCapacity: 4 * Megabyte, // Reduced from 8MB to lower memory usage }) if err != nil { return nil, err @@ -33,7 +40,12 @@ func New(dbPath string) (*Store, error) { // New creates a new ip-hostname pair in the map func (s *Store) New(ip, hostname string) error { - return s.DB.Put([]byte(ip), []byte(hostname), nil) + hostnameMap := map[string]struct{}{hostname: {}} + jsonData, err := json.Marshal(hostnameMap) + if err != nil { + return err + } + return s.DB.Put([]byte(ip), jsonData, nil) } // Exists indicates if an IP exists in the map @@ -44,20 +56,44 @@ func (s *Store) Exists(ip string) bool { // Get gets the meta-information for an IP address from the map. func (s *Store) GetHostnames(ip string) string { - hostname, err := s.DB.Get([]byte(ip), nil) + data, err := s.DB.Get([]byte(ip), nil) if err != nil { return "" } - return string(hostname) + + var hostnameMap map[string]struct{} + if err := json.Unmarshal(data, &hostnameMap); err != nil { + return "" + } + + return strings.Join(mapsutil.GetKeys(hostnameMap), ",") } func (s *Store) Append(ip string, hostnames ...string) error { - existingHostnames, _ := s.DB.Get([]byte(ip), nil) - if len(existingHostnames) > 0 { - hostnames = append(hostnames, string(existingHostnames)) + // Get existing hostnames + var hostnameMap map[string]struct{} + existingData, err := s.DB.Get([]byte(ip), nil) + if err == nil && len(existingData) > 0 { + if err := json.Unmarshal(existingData, &hostnameMap); err != nil { + // If unmarshaling fails, start with empty map + hostnameMap = make(map[string]struct{}) + } + } else { + hostnameMap = make(map[string]struct{}) + } + + // Add new hostnames to map (automatic deduplication) + for _, hostname := range hostnames { + hostnameMap[hostname] = struct{}{} + } + + // Marshal and store + jsonData, err := json.Marshal(hostnameMap) + if err != nil { + return err } - return s.DB.Put([]byte(ip), []byte(strings.Join(hostnames, ",")), nil) + return s.DB.Put([]byte(ip), jsonData, nil) } // Delete deletes the records for an IP from store. @@ -75,8 +111,18 @@ func (s *Store) Iterate(f func(ip string, hostnames []string, counter int)) { for iter.Next() { ip := string(iter.Key()) - hostnames := strings.Split(string(iter.Value()), ",") - hostnames = sliceutil.Dedupe(hostnames) + + var hostnameMap map[string]struct{} + if err := json.Unmarshal(iter.Value(), &hostnameMap); err != nil { + continue + } + + // Convert map keys to slice + hostnames := make([]string, 0, len(hostnameMap)) + for hostname := range hostnameMap { + hostnames = append(hostnames, hostname) + } + counter := len(hostnames) f(ip, hostnames, counter) } From 222cabc0af5d365508967b65f096182ee4f61b23 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 16 Jul 2025 23:10:36 +0200 Subject: [PATCH 2/5] adding incremental batching --- pkg/massdns/massdns.go | 2 + pkg/massdns/process.go | 246 +++++++++++++++++++++++++++++++++++++---- pkg/runner/options.go | 3 + pkg/runner/runner.go | 1 + 4 files changed, 229 insertions(+), 23 deletions(-) diff --git a/pkg/massdns/massdns.go b/pkg/massdns/massdns.go index e07896c..2b6bb8e 100644 --- a/pkg/massdns/massdns.go +++ b/pkg/massdns/massdns.go @@ -48,6 +48,8 @@ type Options struct { MassDnsCmd string // KeepStderr controls whether to capture and store massdns stderr output KeepStderr bool + // BatchSize controls the number of lines per chunk for incremental processing + BatchSize int OnResult func(*retryabledns.DNSData) } diff --git a/pkg/massdns/process.go b/pkg/massdns/process.go index 8c42230..b47386b 100644 --- a/pkg/massdns/process.go +++ b/pkg/massdns/process.go @@ -104,9 +104,6 @@ func (instance *Instance) Run(ctx context.Context) error { } defer shstore.Close() - // Set the correct target file - tmpDir := instance.options.TempDir - // Check if we need to run massdns if instance.options.MassdnsRaw == "" { if len(instance.options.Domains) > 0 { @@ -115,31 +112,19 @@ func (instance *Instance) Run(ctx context.Context) error { gologger.Info().Msgf("Executing massdns\n") } - // Create a temporary file for the massdns output - gologger.Info().Msgf("using massdns output directory: %s\n", tmpDir) - stdoutFile, stderrFile, took, err := instance.RunWithContext(ctx) - gologger.Info().Msgf("massdns output file: %s\n", stdoutFile) - if stderrFile != "" { - gologger.Info().Msgf("massdns error file: %s\n", stderrFile) + // Use incremental processing for large inputs + if instance.options.BatchSize > 0 { + gologger.Info().Msgf("Using incremental processing with batch size: %d\n", instance.options.BatchSize) + err = instance.processIncremental(ctx, inputFile, shstore) } else { - gologger.Info().Msgf("massdns stderr discarded (KeepStderr=false)\n") - } - if err != nil { - return fmt.Errorf("could not execute massdns: %s", err) + // Fallback to original single massdns run + gologger.Info().Msgf("Using single massdns run\n") + err = instance.processSingle(ctx, shstore) } - gologger.Info().Msgf("Massdns execution took %s\n", took) - - gologger.Info().Msgf("Started parsing massdns output\n") - - now := time.Now() - - err = instance.parseMassDNSOutputDir(tmpDir, shstore) if err != nil { - return fmt.Errorf("could not parse massdns output: %w", err) + return fmt.Errorf("could not execute massdns: %w", err) } - - gologger.Info().Msgf("Massdns output parsing completed in %s\n", time.Since(now)) } else { // parse the input file gologger.Info().Msgf("Started parsing massdns input\n") now := time.Now() @@ -183,6 +168,221 @@ func (instance *Instance) Run(ctx context.Context) error { return nil } +// processSingle runs massdns on the entire input file (original behavior) +func (instance *Instance) processSingle(ctx context.Context, shstore *store.Store) error { + // Create a temporary file for the massdns output + gologger.Info().Msgf("using massdns output directory: %s\n", instance.options.TempDir) + stdoutFile, stderrFile, took, err := instance.RunWithContext(ctx) + gologger.Info().Msgf("massdns output file: %s\n", stdoutFile) + if stderrFile != "" { + gologger.Info().Msgf("massdns error file: %s\n", stderrFile) + } else { + gologger.Info().Msgf("massdns stderr discarded (KeepStderr=false)\n") + } + if err != nil { + return fmt.Errorf("could not execute massdns: %s", err) + } + + gologger.Info().Msgf("Massdns execution took %s\n", took) + + gologger.Info().Msgf("Started parsing massdns output\n") + + now := time.Now() + + err = instance.parseMassDNSOutputDir(instance.options.TempDir, shstore) + if err != nil { + return fmt.Errorf("could not parse massdns output: %w", err) + } + + gologger.Info().Msgf("Massdns output parsing completed in %s\n", time.Since(now)) + return nil +} + +// processIncremental processes input in chunks for better memory and disk usage +func (instance *Instance) processIncremental(ctx context.Context, inputFile string, shstore *store.Store) error { + // Count total lines to estimate progress + totalLines, err := instance.countLines(inputFile) + if err != nil { + return fmt.Errorf("could not count input lines: %w", err) + } + + gologger.Info().Msgf("Total input lines: %d\n", totalLines) + + // Create chunks and process them sequentially + chunkNum := 0 + processedLines := 0 + + for { + chunkNum++ + chunkFile, linesInChunk, err := instance.createChunk(inputFile, chunkNum, processedLines) + if err != nil { + return fmt.Errorf("could not create chunk %d: %w", chunkNum, err) + } + + // If no lines in chunk, we're done + if linesInChunk == 0 { + break + } + + gologger.Info().Msgf("Processing chunk %d (%d lines, %.1f%% complete)\n", + chunkNum, linesInChunk, float64(processedLines+linesInChunk)/float64(totalLines)*100) + + // Run massdns on this chunk + chunkStart := time.Now() + stdoutFile, stderrFile, took, err := instance.runChunk(ctx, chunkFile) + if err != nil { + // Clean up chunk file even on error + _ = os.Remove(chunkFile) + return fmt.Errorf("could not execute massdns on chunk %d: %w", chunkNum, err) + } + + gologger.Info().Msgf("Chunk %d massdns execution took %s\n", chunkNum, took) + + // Parse the chunk output immediately + parseStart := time.Now() + err = instance.parseMassDNSOutputFile(stdoutFile, shstore) + if err != nil { + // Clean up files even on error + _ = os.Remove(chunkFile) + _ = os.Remove(stdoutFile) + if stderrFile != "" { + _ = os.Remove(stderrFile) + } + return fmt.Errorf("could not parse massdns output for chunk %d: %w", chunkNum, err) + } + + gologger.Info().Msgf("Chunk %d parsing completed in %s\n", chunkNum, time.Since(parseStart)) + + // Clean up chunk files immediately + _ = os.Remove(chunkFile) + _ = os.Remove(stdoutFile) + if stderrFile != "" { + _ = os.Remove(stderrFile) + } + + processedLines += linesInChunk + gologger.Info().Msgf("Chunk %d completed in %s\n", chunkNum, time.Since(chunkStart)) + } + + gologger.Info().Msgf("All chunks processed successfully (%d total chunks)\n", chunkNum-1) + return nil +} + +// countLines counts the number of lines in a file +func (instance *Instance) countLines(filename string) (int, error) { + file, err := os.Open(filename) + if err != nil { + return 0, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + count := 0 + for scanner.Scan() { + count++ + } + return count, scanner.Err() +} + +// createChunk creates a chunk file with the specified number of lines +func (instance *Instance) createChunk(inputFile string, chunkNum, startLine int) (string, int, error) { + // Create chunk file + chunkFile, err := os.CreateTemp(instance.options.TempDir, fmt.Sprintf("chunk-%d-", chunkNum)) + if err != nil { + return "", 0, fmt.Errorf("could not create chunk file: %w", err) + } + defer chunkFile.Close() + + // Open input file + input, err := os.Open(inputFile) + if err != nil { + _ = os.Remove(chunkFile.Name()) + return "", 0, fmt.Errorf("could not open input file: %w", err) + } + defer input.Close() + + scanner := bufio.NewScanner(input) + writer := bufio.NewWriter(chunkFile) + + // Skip to start line + for i := 0; i < startLine; i++ { + if !scanner.Scan() { + break + } + } + + // Write chunk lines + linesInChunk := 0 + for i := 0; i < instance.options.BatchSize && scanner.Scan(); i++ { + _, err := writer.WriteString(scanner.Text() + "\n") + if err != nil { + _ = os.Remove(chunkFile.Name()) + return "", 0, fmt.Errorf("could not write to chunk file: %w", err) + } + linesInChunk++ + } + + if err := writer.Flush(); err != nil { + _ = os.Remove(chunkFile.Name()) + return "", 0, fmt.Errorf("could not flush chunk file: %w", err) + } + + return chunkFile.Name(), linesInChunk, nil +} + +// runChunk runs massdns on a specific chunk file +func (instance *Instance) runChunk(ctx context.Context, chunkFile string) (stdout, stderr string, took time.Duration, err error) { + start := time.Now() + + // Create temporary file for massdns output + stdoutFile, err := os.CreateTemp(instance.options.TempDir, "massdns-chunk-stdout-") + if err != nil { + return "", "", 0, fmt.Errorf("could not create temp file for massdns output: %w", err) + } + defer func() { + _ = stdoutFile.Close() + }() + + // Handle stderr based on KeepStderr option + var stderrFile *os.File + if instance.options.KeepStderr { + stderrFile, err = os.CreateTemp(instance.options.TempDir, "massdns-chunk-stderr-") + if err != nil { + return "", "", 0, fmt.Errorf("could not create temp file for massdns stderr: %w", err) + } + defer func() { + _ = stderrFile.Close() + }() + } + + // Run the command on the chunk file + args := []string{"-r", instance.options.ResolversFile, "-o", "Snl", "--retry", "REFUSED", "--retry", "SERVFAIL", "-t", "A", chunkFile, "-s", strconv.Itoa(instance.options.Threads)} + if instance.options.MassDnsCmd != "" { + args = append(args, strings.Split(instance.options.MassDnsCmd, " ")...) + } + + cmd := exec.CommandContext(ctx, instance.options.MassdnsPath, args...) + cmd.Stdout = stdoutFile + + // Set stderr based on KeepStderr option + if instance.options.KeepStderr { + cmd.Stderr = stderrFile + } else { + // Discard stderr by sending it to /dev/null + cmd.Stderr = nil + } + + err = cmd.Run() + + // Return stderr filename only if it was captured + stderrFilename := "" + if instance.options.KeepStderr { + stderrFilename = stderrFile.Name() + } + + return stdoutFile.Name(), stderrFilename, time.Since(start), err +} + type item struct { ip string domain string diff --git a/pkg/runner/options.go b/pkg/runner/options.go index 777e253..7bfca71 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -36,6 +36,7 @@ type Options struct { DisableUpdateCheck bool // DisableUpdateCheck disable automatic update check Mode string KeepStderr bool // KeepStderr controls whether to capture and store massdns stderr output + BatchSize int // BatchSize controls the number of lines per chunk for incremental processing OnResult func(*retryabledns.DNSData) } @@ -44,6 +45,7 @@ var DefaultOptions = Options{ Threads: 10000, Retries: 5, WildcardThreads: 250, + BatchSize: 50000, // Default batch size for incremental processing } // ParseOptions parses the command line flags provided by a user @@ -90,6 +92,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.StrictWildcard, "strict-wildcard", "sw", false, "Perform wildcard check on all found subdomains"), flagSet.IntVar(&options.WildcardThreads, "wt", 250, "Number of concurrent wildcard checks"), flagSet.BoolVar(&options.KeepStderr, "retain-stderr", false, "Capture and store massdns stderr output (default: discard)"), + flagSet.IntVar(&options.BatchSize, "batch-size", 50000, "Number of lines per chunk for incremental processing"), ) flagSet.CreateGroup("debug", "Debug", diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 3e63769..6ed7c62 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -186,6 +186,7 @@ func (r *Runner) runMassdns(inputFile string) { WildcardOutputFile: r.options.WildcardOutputFile, MassDnsCmd: r.options.MassDnsCmd, KeepStderr: r.options.KeepStderr, + BatchSize: r.options.BatchSize, OnResult: r.options.OnResult, }) if err != nil { From cbe619574583a9ca6ef179e50d9e8078ccf9f986 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 16 Jul 2025 23:18:14 +0200 Subject: [PATCH 3/5] lint --- pkg/massdns/process.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/massdns/process.go b/pkg/massdns/process.go index b47386b..2d6ff20 100644 --- a/pkg/massdns/process.go +++ b/pkg/massdns/process.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "log" "os" "os/exec" "strconv" @@ -58,7 +57,6 @@ func (instance *Instance) RunWithContext(ctx context.Context) (stdout, stderr st args = append(args, strings.Split(instance.options.MassDnsCmd, " ")...) } - log.Fatalf("flag: %s %s", instance.options.MassdnsPath, strings.Join(args, " ")) cmd := exec.CommandContext(ctx, instance.options.MassdnsPath, args...) cmd.Stdout = stdoutFile @@ -274,7 +272,9 @@ func (instance *Instance) countLines(filename string) (int, error) { if err != nil { return 0, err } - defer file.Close() + defer func() { + _ = file.Close() + }() scanner := bufio.NewScanner(file) count := 0 @@ -291,7 +291,9 @@ func (instance *Instance) createChunk(inputFile string, chunkNum, startLine int) if err != nil { return "", 0, fmt.Errorf("could not create chunk file: %w", err) } - defer chunkFile.Close() + defer func() { + _ = chunkFile.Close() + }() // Open input file input, err := os.Open(inputFile) @@ -299,7 +301,9 @@ func (instance *Instance) createChunk(inputFile string, chunkNum, startLine int) _ = os.Remove(chunkFile.Name()) return "", 0, fmt.Errorf("could not open input file: %w", err) } - defer input.Close() + defer func() { + _ = input.Close() + }() scanner := bufio.NewScanner(input) writer := bufio.NewWriter(chunkFile) From e8148c242a8a9ceb4e62b3be36cda5471d47248b Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 17 Jul 2025 00:16:12 +0200 Subject: [PATCH 4/5] runtime chunking with batcher --- pkg/massdns/process.go | 503 ++++++++++++++++++++++++----------------- pkg/runner/runner.go | 149 +++++++----- 2 files changed, 391 insertions(+), 261 deletions(-) diff --git a/pkg/massdns/process.go b/pkg/massdns/process.go index 2d6ff20..ef9a9ea 100644 --- a/pkg/massdns/process.go +++ b/pkg/massdns/process.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "os/exec" "strconv" @@ -19,9 +20,7 @@ import ( "github.com/projectdiscovery/shuffledns/pkg/wildcards" "github.com/projectdiscovery/utils/batcher" fileutil "github.com/projectdiscovery/utils/file" - folderutil "github.com/projectdiscovery/utils/folder" ioutil "github.com/projectdiscovery/utils/io" - stringsutil "github.com/projectdiscovery/utils/strings" "github.com/remeh/sizedwaitgroup" "github.com/weppos/publicsuffix-go/publicsuffix" ) @@ -104,25 +103,9 @@ func (instance *Instance) Run(ctx context.Context) error { // Check if we need to run massdns if instance.options.MassdnsRaw == "" { - if len(instance.options.Domains) > 0 { - gologger.Info().Msgf("Executing massdns on %s\n", strings.Join(instance.options.Domains, ", ")) - } else { - gologger.Info().Msgf("Executing massdns\n") - } - - // Use incremental processing for large inputs - if instance.options.BatchSize > 0 { - gologger.Info().Msgf("Using incremental processing with batch size: %d\n", instance.options.BatchSize) - err = instance.processIncremental(ctx, inputFile, shstore) - } else { - // Fallback to original single massdns run - gologger.Info().Msgf("Using single massdns run\n") - err = instance.processSingle(ctx, shstore) - } - - if err != nil { - return fmt.Errorf("could not execute massdns: %w", err) - } + // This case is now handled by the streaming methods in the runner + // The Run method is only called for raw massdns output processing + return errors.New("streaming processing should be used for new massdns runs") } else { // parse the input file gologger.Info().Msgf("Started parsing massdns input\n") now := time.Now() @@ -166,174 +149,6 @@ func (instance *Instance) Run(ctx context.Context) error { return nil } -// processSingle runs massdns on the entire input file (original behavior) -func (instance *Instance) processSingle(ctx context.Context, shstore *store.Store) error { - // Create a temporary file for the massdns output - gologger.Info().Msgf("using massdns output directory: %s\n", instance.options.TempDir) - stdoutFile, stderrFile, took, err := instance.RunWithContext(ctx) - gologger.Info().Msgf("massdns output file: %s\n", stdoutFile) - if stderrFile != "" { - gologger.Info().Msgf("massdns error file: %s\n", stderrFile) - } else { - gologger.Info().Msgf("massdns stderr discarded (KeepStderr=false)\n") - } - if err != nil { - return fmt.Errorf("could not execute massdns: %s", err) - } - - gologger.Info().Msgf("Massdns execution took %s\n", took) - - gologger.Info().Msgf("Started parsing massdns output\n") - - now := time.Now() - - err = instance.parseMassDNSOutputDir(instance.options.TempDir, shstore) - if err != nil { - return fmt.Errorf("could not parse massdns output: %w", err) - } - - gologger.Info().Msgf("Massdns output parsing completed in %s\n", time.Since(now)) - return nil -} - -// processIncremental processes input in chunks for better memory and disk usage -func (instance *Instance) processIncremental(ctx context.Context, inputFile string, shstore *store.Store) error { - // Count total lines to estimate progress - totalLines, err := instance.countLines(inputFile) - if err != nil { - return fmt.Errorf("could not count input lines: %w", err) - } - - gologger.Info().Msgf("Total input lines: %d\n", totalLines) - - // Create chunks and process them sequentially - chunkNum := 0 - processedLines := 0 - - for { - chunkNum++ - chunkFile, linesInChunk, err := instance.createChunk(inputFile, chunkNum, processedLines) - if err != nil { - return fmt.Errorf("could not create chunk %d: %w", chunkNum, err) - } - - // If no lines in chunk, we're done - if linesInChunk == 0 { - break - } - - gologger.Info().Msgf("Processing chunk %d (%d lines, %.1f%% complete)\n", - chunkNum, linesInChunk, float64(processedLines+linesInChunk)/float64(totalLines)*100) - - // Run massdns on this chunk - chunkStart := time.Now() - stdoutFile, stderrFile, took, err := instance.runChunk(ctx, chunkFile) - if err != nil { - // Clean up chunk file even on error - _ = os.Remove(chunkFile) - return fmt.Errorf("could not execute massdns on chunk %d: %w", chunkNum, err) - } - - gologger.Info().Msgf("Chunk %d massdns execution took %s\n", chunkNum, took) - - // Parse the chunk output immediately - parseStart := time.Now() - err = instance.parseMassDNSOutputFile(stdoutFile, shstore) - if err != nil { - // Clean up files even on error - _ = os.Remove(chunkFile) - _ = os.Remove(stdoutFile) - if stderrFile != "" { - _ = os.Remove(stderrFile) - } - return fmt.Errorf("could not parse massdns output for chunk %d: %w", chunkNum, err) - } - - gologger.Info().Msgf("Chunk %d parsing completed in %s\n", chunkNum, time.Since(parseStart)) - - // Clean up chunk files immediately - _ = os.Remove(chunkFile) - _ = os.Remove(stdoutFile) - if stderrFile != "" { - _ = os.Remove(stderrFile) - } - - processedLines += linesInChunk - gologger.Info().Msgf("Chunk %d completed in %s\n", chunkNum, time.Since(chunkStart)) - } - - gologger.Info().Msgf("All chunks processed successfully (%d total chunks)\n", chunkNum-1) - return nil -} - -// countLines counts the number of lines in a file -func (instance *Instance) countLines(filename string) (int, error) { - file, err := os.Open(filename) - if err != nil { - return 0, err - } - defer func() { - _ = file.Close() - }() - - scanner := bufio.NewScanner(file) - count := 0 - for scanner.Scan() { - count++ - } - return count, scanner.Err() -} - -// createChunk creates a chunk file with the specified number of lines -func (instance *Instance) createChunk(inputFile string, chunkNum, startLine int) (string, int, error) { - // Create chunk file - chunkFile, err := os.CreateTemp(instance.options.TempDir, fmt.Sprintf("chunk-%d-", chunkNum)) - if err != nil { - return "", 0, fmt.Errorf("could not create chunk file: %w", err) - } - defer func() { - _ = chunkFile.Close() - }() - - // Open input file - input, err := os.Open(inputFile) - if err != nil { - _ = os.Remove(chunkFile.Name()) - return "", 0, fmt.Errorf("could not open input file: %w", err) - } - defer func() { - _ = input.Close() - }() - - scanner := bufio.NewScanner(input) - writer := bufio.NewWriter(chunkFile) - - // Skip to start line - for i := 0; i < startLine; i++ { - if !scanner.Scan() { - break - } - } - - // Write chunk lines - linesInChunk := 0 - for i := 0; i < instance.options.BatchSize && scanner.Scan(); i++ { - _, err := writer.WriteString(scanner.Text() + "\n") - if err != nil { - _ = os.Remove(chunkFile.Name()) - return "", 0, fmt.Errorf("could not write to chunk file: %w", err) - } - linesInChunk++ - } - - if err := writer.Flush(); err != nil { - _ = os.Remove(chunkFile.Name()) - return "", 0, fmt.Errorf("could not flush chunk file: %w", err) - } - - return chunkFile.Name(), linesInChunk, nil -} - // runChunk runs massdns on a specific chunk file func (instance *Instance) runChunk(ctx context.Context, chunkFile string) (stdout, stderr string, took time.Duration, err error) { start := time.Now() @@ -436,26 +251,6 @@ func (instance *Instance) parseMassDNSOutputFile(tmpFile string, store *store.St return nil } -func (instance *Instance) parseMassDNSOutputDir(tmpDir string, store *store.Store) error { - tmpFiles, err := folderutil.GetFiles(tmpDir) - if err != nil { - return fmt.Errorf("could not open massdns output directory: %w", err) - } - - for _, tmpFile := range tmpFiles { - // just process stdout files - if !stringsutil.ContainsAnyI(tmpFile, "stdout") { - continue - } - err = instance.parseMassDNSOutputFile(tmpFile, store) - if err != nil { - return fmt.Errorf("could not parse massdns output: %w", err) - } - } - - return nil -} - func (instance *Instance) autoExtractRootDomains(store *store.Store) error { candidateRootDomains := make(map[string]struct{}) store.Iterate(func(ip string, hostnames []string, counter int) { @@ -667,3 +462,293 @@ func (instance *Instance) writeOutput(store *store.Store) error { } return nil } + +// ProcessDomainStreaming processes domain bruteforce using streaming with batcher +func (instance *Instance) ProcessDomainStreaming(ctx context.Context, wordlistFile *os.File) error { + // Create a store for storing ip metadata + shstore, err := store.New(instance.options.TempDir) + if err != nil { + return fmt.Errorf("could not create store: %w", err) + } + defer shstore.Close() + + // Create batcher for streaming permutations + chunkNum := 0 + permutationCount := 0 + + bulkWriter := batcher.New[string]( + batcher.WithMaxCapacity[string](instance.options.BatchSize), + batcher.WithFlushInterval[string](10*time.Second), + batcher.WithFlushCallback[string](func(permutations []string) { + chunkNum++ + if len(permutations) == 0 { + return + } + + gologger.Info().Msgf("Processing chunk %d (%d permutations, total: %d)\n", + chunkNum, len(permutations), permutationCount) + + // Create temporary chunk file + chunkFile, err := os.CreateTemp(instance.options.TempDir, fmt.Sprintf("chunk-%d-", chunkNum)) + if err != nil { + gologger.Error().Msgf("Could not create chunk file: %s\n", err) + return + } + + // Write permutations to chunk file + writer := bufio.NewWriter(chunkFile) + for _, permutation := range permutations { + _, err := writer.WriteString(permutation + "\n") + if err != nil { + gologger.Error().Msgf("Could not write to chunk file: %s\n", err) + _ = chunkFile.Close() + _ = os.Remove(chunkFile.Name()) + return + } + } + _ = writer.Flush() + _ = chunkFile.Close() + + // Run massdns on this chunk + chunkStart := time.Now() + stdoutFile, stderrFile, took, err := instance.runChunk(ctx, chunkFile.Name()) + if err != nil { + gologger.Error().Msgf("Could not execute massdns on chunk %d: %s\n", chunkNum, err) + _ = os.Remove(chunkFile.Name()) + return + } + + gologger.Info().Msgf("Chunk %d massdns execution took %s\n", chunkNum, took) + + // Parse the chunk output immediately + parseStart := time.Now() + err = instance.parseMassDNSOutputFile(stdoutFile, shstore) + if err != nil { + gologger.Error().Msgf("Could not parse massdns output for chunk %d: %s\n", chunkNum, err) + _ = os.Remove(chunkFile.Name()) + _ = os.Remove(stdoutFile) + if stderrFile != "" { + _ = os.Remove(stderrFile) + } + return + } + + gologger.Info().Msgf("Chunk %d parsing completed in %s\n", chunkNum, time.Since(parseStart)) + + // Clean up chunk files immediately + _ = os.Remove(chunkFile.Name()) + _ = os.Remove(stdoutFile) + if stderrFile != "" { + _ = os.Remove(stderrFile) + } + + gologger.Info().Msgf("Chunk %d completed in %s\n", chunkNum, time.Since(chunkStart)) + }), + ) + + bulkWriter.Run() + + // Read wordlist and generate permutations on-the-fly + scanner := bufio.NewScanner(wordlistFile) + for scanner.Scan() { + // RFC4343 - case insensitive domain + text := strings.ToLower(scanner.Text()) + if text == "" { + continue + } + + // Generate permutations for each domain + for _, domain := range instance.options.Domains { + permutation := text + "." + domain + bulkWriter.Append(permutation) + permutationCount++ + } + } + + // Stop the batcher and wait for completion + bulkWriter.Stop() + bulkWriter.WaitDone() + + if err := scanner.Err(); err != nil { + return fmt.Errorf("error reading wordlist: %w", err) + } + + gologger.Info().Msgf("Total permutations generated: %d\n", permutationCount) + + // Perform post-processing steps + if instance.options.AutoExtractRootDomains { + gologger.Info().Msgf("Started extracting root domains\n") + now := time.Now() + err = instance.autoExtractRootDomains(shstore) + if err != nil { + return fmt.Errorf("could not extract root domains: %w", err) + } + gologger.Info().Msgf("Root domain extraction completed in %s\n", time.Since(now)) + } + + // Perform wildcard filtering only if domain name has been specified + if len(instance.options.Domains) > 0 { + gologger.Info().Msgf("Started removing wildcards records\n") + now := time.Now() + err = instance.filterWildcards(shstore) + if err != nil { + return fmt.Errorf("could not filter wildcards: %w", err) + } + gologger.Info().Msgf("Wildcard removal completed in %s\n", time.Since(now)) + } + + gologger.Info().Msgf("Finished enumeration, started writing output\n") + + // Write the final elaborated list out + now := time.Now() + err = instance.writeOutput(shstore) + if err != nil { + return fmt.Errorf("could not write output: %w", err) + } + gologger.Info().Msgf("Output written in %s\n", time.Since(now)) + + return nil +} + +// ProcessSubdomainsStreaming processes subdomain list using streaming with batcher +func (instance *Instance) ProcessSubdomainsStreaming(ctx context.Context, subdomainReader io.Reader) error { + // Create a store for storing ip metadata + shstore, err := store.New(instance.options.TempDir) + if err != nil { + return fmt.Errorf("could not create store: %w", err) + } + defer shstore.Close() + + // Create batcher for streaming subdomains + chunkNum := 0 + subdomainCount := 0 + + bulkWriter := batcher.New[string]( + batcher.WithMaxCapacity[string](instance.options.BatchSize), + batcher.WithFlushInterval[string](10*time.Second), + batcher.WithFlushCallback[string](func(subdomains []string) { + chunkNum++ + if len(subdomains) == 0 { + return + } + + gologger.Info().Msgf("Processing chunk %d (%d subdomains, total: %d)\n", + chunkNum, len(subdomains), subdomainCount) + + // Create temporary chunk file + chunkFile, err := os.CreateTemp(instance.options.TempDir, fmt.Sprintf("chunk-%d-", chunkNum)) + if err != nil { + gologger.Error().Msgf("Could not create chunk file: %s\n", err) + return + } + + // Write subdomains to chunk file + writer := bufio.NewWriter(chunkFile) + for _, subdomain := range subdomains { + _, err := writer.WriteString(subdomain + "\n") + if err != nil { + gologger.Error().Msgf("Could not write to chunk file: %s\n", err) + _ = chunkFile.Close() + _ = os.Remove(chunkFile.Name()) + return + } + } + _ = writer.Flush() + _ = chunkFile.Close() + + // Run massdns on this chunk + chunkStart := time.Now() + stdoutFile, stderrFile, took, err := instance.runChunk(ctx, chunkFile.Name()) + if err != nil { + gologger.Error().Msgf("Could not execute massdns on chunk %d: %s\n", chunkNum, err) + _ = os.Remove(chunkFile.Name()) + return + } + + gologger.Info().Msgf("Chunk %d massdns execution took %s\n", chunkNum, took) + + // Parse the chunk output immediately + parseStart := time.Now() + err = instance.parseMassDNSOutputFile(stdoutFile, shstore) + if err != nil { + gologger.Error().Msgf("Could not parse massdns output for chunk %d: %s\n", chunkNum, err) + _ = os.Remove(chunkFile.Name()) + _ = os.Remove(stdoutFile) + if stderrFile != "" { + _ = os.Remove(stderrFile) + } + return + } + + gologger.Info().Msgf("Chunk %d parsing completed in %s\n", chunkNum, time.Since(parseStart)) + + // Clean up chunk files immediately + _ = os.Remove(chunkFile.Name()) + _ = os.Remove(stdoutFile) + if stderrFile != "" { + _ = os.Remove(stderrFile) + } + + gologger.Info().Msgf("Chunk %d completed in %s\n", chunkNum, time.Since(chunkStart)) + }), + ) + + bulkWriter.Run() + + // Read subdomains and stream them to batcher + scanner := bufio.NewScanner(subdomainReader) + for scanner.Scan() { + // RFC4343 - case insensitive domain + subdomain := strings.ToLower(strings.TrimSpace(scanner.Text())) + if subdomain == "" { + continue + } + + bulkWriter.Append(subdomain) + subdomainCount++ + } + + // Stop the batcher and wait for completion + bulkWriter.Stop() + bulkWriter.WaitDone() + + if err := scanner.Err(); err != nil { + return fmt.Errorf("error reading subdomains: %w", err) + } + + gologger.Info().Msgf("Total subdomains processed: %d\n", subdomainCount) + + // Perform post-processing steps + if instance.options.AutoExtractRootDomains { + gologger.Info().Msgf("Started extracting root domains\n") + now := time.Now() + err = instance.autoExtractRootDomains(shstore) + if err != nil { + return fmt.Errorf("could not extract root domains: %w", err) + } + gologger.Info().Msgf("Root domain extraction completed in %s\n", time.Since(now)) + } + + // Perform wildcard filtering only if domain name has been specified + if len(instance.options.Domains) > 0 { + gologger.Info().Msgf("Started removing wildcards records\n") + now := time.Now() + err = instance.filterWildcards(shstore) + if err != nil { + return fmt.Errorf("could not filter wildcards: %w", err) + } + gologger.Info().Msgf("Wildcard removal completed in %s\n", time.Since(now)) + } + + gologger.Info().Msgf("Finished enumeration, started writing output\n") + + // Write the final elaborated list out + now := time.Now() + err = instance.writeOutput(shstore) + if err != nil { + return fmt.Errorf("could not write output: %w", err) + } + gologger.Info().Msgf("Output written in %s\n", time.Since(now)) + + return nil +} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 6ed7c62..6389e39 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -1,20 +1,14 @@ package runner import ( - "bufio" "context" "errors" - "io" "os" "os/exec" - "path/filepath" - "strings" - "time" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/shuffledns/pkg/massdns" fileutil "github.com/projectdiscovery/utils/file" - "github.com/rs/xid" ) // Runner is a client for running the enumeration process. @@ -81,9 +75,9 @@ func (r *Runner) findBinary() string { // RunEnumeration sets up the input layer for giving input to massdns // binary and runs the actual enumeration func (r *Runner) RunEnumeration() { - // Handle only wildcard filtering + // Handle only wildcard filtering on existing massdns output if r.options.MassdnsRaw != "" { - r.processSubdomains() + r.processExistingOutput() return } @@ -102,72 +96,123 @@ func (r *Runner) RunEnumeration() { // processDomain processes the bruteforce for a domain using a wordlist func (r *Runner) processDomain() { - resolveFile := filepath.Join(r.tempDir, xid.New().String()) - file, err := os.Create(resolveFile) - if err != nil { - gologger.Error().Msgf("Could not create bruteforce list (%s): %s\n", r.tempDir, err) - return - } - writer := bufio.NewWriter(file) - // Read the input wordlist for bruteforce generation inputFile, err := os.Open(r.options.Wordlist) if err != nil { gologger.Error().Msgf("Could not read bruteforce wordlist (%s): %s\n", r.options.Wordlist, err) - _ = file.Close() return } + defer func() { + _ = inputFile.Close() + }() - gologger.Info().Msgf("Started generating bruteforce permutation\n") + gologger.Info().Msgf("Started generating bruteforce permutation with streaming processing\n") - now := time.Now() - // Create permutation for domain with wordlist - scanner := bufio.NewScanner(inputFile) - for scanner.Scan() { - // RFC4343 - case insensitive domain - text := strings.ToLower(scanner.Text()) - if text == "" { - continue - } - for _, domain := range r.options.Domains { - _, _ = writer.WriteString(text + "." + domain + "\n") - } + // Create massdns instance for processing chunks + massdns, err := massdns.New(massdns.Options{ + Domains: r.options.Domains, + AutoExtractRootDomains: r.options.AutoExtractRootDomains, + Retries: r.options.Retries, + MassdnsPath: r.options.MassdnsPath, + Threads: r.options.Threads, + WildcardsThreads: r.options.WildcardThreads, + ResolversFile: r.options.ResolversFile, + TrustedResolvers: r.options.TrustedResolvers, + TempDir: r.tempDir, + OutputFile: r.options.Output, + Json: r.options.Json, + MassdnsRaw: r.options.MassdnsRaw, + StrictWildcard: r.options.StrictWildcard, + WildcardOutputFile: r.options.WildcardOutputFile, + MassDnsCmd: r.options.MassDnsCmd, + KeepStderr: r.options.KeepStderr, + BatchSize: r.options.BatchSize, + OnResult: r.options.OnResult, + }) + if err != nil { + gologger.Error().Msgf("Could not create massdns client: %s\n", err) + return } - _ = writer.Flush() - _ = inputFile.Close() - _ = file.Close() - gologger.Info().Msgf("Generating permutations took %s at %s\n", time.Since(now), resolveFile) + // Use streaming processing with batcher + err = massdns.ProcessDomainStreaming(context.Background(), inputFile) + if err != nil { + gologger.Error().Msgf("Could not process domain with streaming: %s\n", err) + return + } - // Run the actual massdns enumeration process - r.runMassdns(resolveFile) + if r.options.WildcardOutputFile != "" { + _ = massdns.DumpWildcardsToFile(r.options.WildcardOutputFile) + } + + gologger.Info().Msgf("Finished resolving.\n") } // processSubdomain processes the resolving for a list of subdomains func (r *Runner) processSubdomains() { - var resolveFile string + // Create massdns instance for processing chunks + massdns, err := massdns.New(massdns.Options{ + Domains: r.options.Domains, + AutoExtractRootDomains: r.options.AutoExtractRootDomains, + Retries: r.options.Retries, + MassdnsPath: r.options.MassdnsPath, + Threads: r.options.Threads, + WildcardsThreads: r.options.WildcardThreads, + ResolversFile: r.options.ResolversFile, + TrustedResolvers: r.options.TrustedResolvers, + TempDir: r.tempDir, + OutputFile: r.options.Output, + Json: r.options.Json, + MassdnsRaw: r.options.MassdnsRaw, + StrictWildcard: r.options.StrictWildcard, + WildcardOutputFile: r.options.WildcardOutputFile, + MassDnsCmd: r.options.MassDnsCmd, + KeepStderr: r.options.KeepStderr, + BatchSize: r.options.BatchSize, + OnResult: r.options.OnResult, + }) + if err != nil { + gologger.Error().Msgf("Could not create massdns client: %s\n", err) + return + } - // If there is stdin, write the resolution list to the file + // Handle stdin or file input if fileutil.HasStdin() && r.options.SubdomainsList == "" { - file, err := os.CreateTemp(r.tempDir, "massdns-stdin-") + // Use streaming processing for stdin + gologger.Info().Msgf("Processing subdomains from stdin with streaming\n") + err = massdns.ProcessSubdomainsStreaming(context.Background(), os.Stdin) if err != nil { - gologger.Error().Msgf("Could not create resolution list (%s): %s\n", r.tempDir, err) + gologger.Error().Msgf("Could not process subdomains with streaming: %s\n", err) return } - _, _ = io.Copy(file, os.Stdin) - _ = file.Close() - resolveFile = file.Name() } else { - // Use the file if user has provided one - resolveFile = r.options.SubdomainsList + // Use streaming processing for file + subdomainFile, err := os.Open(r.options.SubdomainsList) + if err != nil { + gologger.Error().Msgf("Could not open subdomain list (%s): %s\n", r.options.SubdomainsList, err) + return + } + defer func() { + _ = subdomainFile.Close() + }() + + gologger.Info().Msgf("Processing subdomains from file with streaming\n") + err = massdns.ProcessSubdomainsStreaming(context.Background(), subdomainFile) + if err != nil { + gologger.Error().Msgf("Could not process subdomains with streaming from file: %s\n", err) + return + } } - // Run the actual massdns enumeration process - r.runMassdns(resolveFile) + if r.options.WildcardOutputFile != "" { + _ = massdns.DumpWildcardsToFile(r.options.WildcardOutputFile) + } + + gologger.Info().Msgf("Finished resolving.\n") } -// runMassdns runs the massdns tool on the list of inputs -func (r *Runner) runMassdns(inputFile string) { +// processExistingOutput processes existing massdns output for wildcard filtering +func (r *Runner) processExistingOutput() { massdns, err := massdns.New(massdns.Options{ Domains: r.options.Domains, AutoExtractRootDomains: r.options.AutoExtractRootDomains, @@ -175,7 +220,6 @@ func (r *Runner) runMassdns(inputFile string) { MassdnsPath: r.options.MassdnsPath, Threads: r.options.Threads, WildcardsThreads: r.options.WildcardThreads, - InputFile: inputFile, ResolversFile: r.options.ResolversFile, TrustedResolvers: r.options.TrustedResolvers, TempDir: r.tempDir, @@ -196,12 +240,13 @@ func (r *Runner) runMassdns(inputFile string) { err = massdns.Run(context.Background()) if err != nil { - gologger.Error().Msgf("Could not run massdns: %s\n", err) + gologger.Error().Msgf("Could not process existing massdns output: %s\n", err) + return } if r.options.WildcardOutputFile != "" { _ = massdns.DumpWildcardsToFile(r.options.WildcardOutputFile) } - gologger.Info().Msgf("Finished resolving.\n") + gologger.Info().Msgf("Finished processing existing output.\n") } From 7b5a5c9eb70520f3b93c0e94b0cd7ff13470ce45 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 17 Jul 2025 13:19:38 +0200 Subject: [PATCH 5/5] increase batch size --- pkg/runner/options.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/runner/options.go b/pkg/runner/options.go index 7bfca71..6a18bdb 100644 --- a/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -9,6 +9,11 @@ import ( updateutils "github.com/projectdiscovery/utils/update" ) +const ( + // DefaultBatchSize is the default number of lines per chunk for incremental processing + DefaultBatchSize = 500000 +) + // Options contains the configuration options for tuning // the active dns resolving process. type Options struct { @@ -45,7 +50,7 @@ var DefaultOptions = Options{ Threads: 10000, Retries: 5, WildcardThreads: 250, - BatchSize: 50000, // Default batch size for incremental processing + BatchSize: DefaultBatchSize, // Default batch size for incremental processing } // ParseOptions parses the command line flags provided by a user @@ -92,7 +97,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.StrictWildcard, "strict-wildcard", "sw", false, "Perform wildcard check on all found subdomains"), flagSet.IntVar(&options.WildcardThreads, "wt", 250, "Number of concurrent wildcard checks"), flagSet.BoolVar(&options.KeepStderr, "retain-stderr", false, "Capture and store massdns stderr output (default: discard)"), - flagSet.IntVar(&options.BatchSize, "batch-size", 50000, "Number of lines per chunk for incremental processing"), + flagSet.IntVar(&options.BatchSize, "batch-size", DefaultBatchSize, "Number of lines per chunk for incremental processing"), ) flagSet.CreateGroup("debug", "Debug",