From b768158ff8b65f29f5339c709f955f0074da02fe Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 26 Jun 2026 15:40:18 +1200 Subject: [PATCH 1/4] Remove scrapeCollectors() method from scraper startup This method scraped every RIB and UPDATE for every known collector on startup, which is a) unnecessary, b) takes a long time and therefore prevents scraping newer files in the meantime. The regular scraping routines should suffice and also respect the last_completed_crawl times in the database so won't waste time crawling files that we already know about. --- scrape_collectors.go | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/scrape_collectors.go b/scrape_collectors.go index 2efb10f..e05fb20 100644 --- a/scrape_collectors.go +++ b/scrape_collectors.go @@ -73,43 +73,5 @@ func scrapeProject(ctx context.Context, logger *logging.Logger, db *pgxpool.Pool return fmt.Errorf("failed to upsert collectors for project %s: %w", project.Name, err) } - for _, collector := range collectors { - if err := scrapeCollector(ctx, logger, db, finder, collector); err != nil { - logger.Error().Err(err).Str("collector", collector.Name).Msg("Failed to scrape collector") - continue - } - } - return nil -} - -func scrapeCollector(ctx context.Context, logger *logging.Logger, db *pgxpool.Pool, finder Finder, collector Collector) error { - logger.Info().Str("collector", collector.Name).Msg("Starting to scrape collector data") - - // Use a sensible default interval for periodic scraping (e.g., last 24 hours) - // But for the very first scrape, we might want more. - // For now, sticking to the existing "all time" logic but cleaned up. - query := Query{ - Collectors: []Collector{collector}, - DumpType: DumpTypeAny, - Intervals: []Interval{{ - From: time.Unix(0, 0), - Until: time.Now().Add(time.Hour * 24), - }}, - } - - dumps, err := finder.Find(query) - if err != nil { - return fmt.Errorf("finder.Find failed for collector %s: %w", collector.Name, err) - } - - logger.Info(). - Str("collector", collector.Name). - Int("dumps_found", len(dumps)). - Msg("Found BGP dumps for collector") - - if err := UpsertBGPDumps(ctx, logger, db, dumps); err != nil { - return fmt.Errorf("failed to upsert dumps for collector %s: %w", collector.Name, err) - } - return nil } From e50961cb007b52754a47f3fff2e88a34238f7ae8 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 26 Jun 2026 15:43:10 +1200 Subject: [PATCH 2/4] Fix outdated arguments used by the query CLI in main.go --- cmd/bgpf/main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/bgpf/main.go b/cmd/bgpf/main.go index 0410a59..9e19016 100644 --- a/cmd/bgpf/main.go +++ b/cmd/bgpf/main.go @@ -140,8 +140,12 @@ func (f *FilesCmd) Run(parentLogger *logging.Logger, cli BgpfCLI) error { query := bgpfinder.Query{ Collectors: collectors, - From: fromTime, - Until: untilTime, + Intervals: []bgpfinder.Interval{ + { + From: fromTime, + Until: untilTime, + }, + }, DumpType: f.Type, } From 7afbcb47d3084bae22066b5289878690d9abb5cf Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 26 Jun 2026 15:43:55 +1200 Subject: [PATCH 3/4] Fix bug in main_test.go --- cmd/bgpfinder-server/main_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/bgpfinder-server/main_test.go b/cmd/bgpfinder-server/main_test.go index 680bfad..ad015a6 100644 --- a/cmd/bgpfinder-server/main_test.go +++ b/cmd/bgpfinder-server/main_test.go @@ -50,12 +50,12 @@ func TestParseDataRequest(t *testing.T) { expectedFrom := time.Unix(startTimeInt64, 0) expectedUntil := time.Unix(endTimeInt64, 0) - if !query.From.Equal(expectedFrom) { - t.Errorf("Expected From: %v, got %v", expectedFrom, query.From) + if !query.FirstInterval().From.Equal(expectedFrom) { + t.Errorf("Expected From: %v, got %v", expectedFrom, query.FirstInterval().From) } - if !query.Until.Equal(expectedUntil) { - t.Errorf("Expected Until: %v, got %v", expectedUntil, query.Until) + if !query.FirstInterval().Until.Equal(expectedUntil) { + t.Errorf("Expected Until: %v, got %v", expectedUntil, query.FirstInterval().Until) } // Verify Collectors From 28e7df54903e49bf7bbbd80225be7abe299b447f Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 14 Jul 2026 11:35:40 +1200 Subject: [PATCH 4/4] scraping: add 48 hour grace period to prevRunTime This resolves issues when updates appear on the collector site in non-chronological order. When this happens, the last completed crawl time is set to a timestamp after the missing updates and therefore they are never subsequently scraped once they arrive. Now, the scraper will look back up to 48 hours from the last completed crawl time to ensure that it sees those missing files on a subsequent crawl. Note: currently our scraper attempts to upsert each file that it scrapes so we are now going to be doing a lot of redundant updates in the general case. Might be worth re-thinking how we want to handle conflicts in db.go? --- periodicscraper/periodic_scraper_per_collector.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/periodicscraper/periodic_scraper_per_collector.go b/periodicscraper/periodic_scraper_per_collector.go index f529106..ea87985 100644 --- a/periodicscraper/periodic_scraper_per_collector.go +++ b/periodicscraper/periodic_scraper_per_collector.go @@ -116,14 +116,19 @@ func getDumps(ctx context.Context, dumpType := getDumpTypeFromBool(isRibsData) untilNextDay := time.Now().AddDate(0, 0, 1) + queryFrom := prevRunTimeEnd + if queryFrom.After(time.Unix(0, 0)) { + queryFrom = queryFrom.Add(-48 * time.Hour) + } + query := bgpfinder.Query{ Collectors: []bgpfinder.Collector{collector}, DumpType: dumpType, Intervals: []bgpfinder.Interval{ - { - From: prevRunTimeEnd, // Start from prevRuntime - Until: untilNextDay, // Until tomorrow (to ensure we get today's data) - }, + { + From: queryFrom, + Until: untilNextDay, // Until tomorrow (to ensure we get today's data) + }, }, }