diff --git a/cmd/bgpf/main.go b/cmd/bgpf/main.go index 0410a59..9e19016 100644 --- a/cmd/bgpf/main.go +++ b/cmd/bgpf/main.go @@ -140,8 +140,12 @@ func (f *FilesCmd) Run(parentLogger *logging.Logger, cli BgpfCLI) error { query := bgpfinder.Query{ Collectors: collectors, - From: fromTime, - Until: untilTime, + Intervals: []bgpfinder.Interval{ + { + From: fromTime, + Until: untilTime, + }, + }, DumpType: f.Type, } diff --git a/cmd/bgpfinder-server/main_test.go b/cmd/bgpfinder-server/main_test.go index 680bfad..ad015a6 100644 --- a/cmd/bgpfinder-server/main_test.go +++ b/cmd/bgpfinder-server/main_test.go @@ -50,12 +50,12 @@ func TestParseDataRequest(t *testing.T) { expectedFrom := time.Unix(startTimeInt64, 0) expectedUntil := time.Unix(endTimeInt64, 0) - if !query.From.Equal(expectedFrom) { - t.Errorf("Expected From: %v, got %v", expectedFrom, query.From) + if !query.FirstInterval().From.Equal(expectedFrom) { + t.Errorf("Expected From: %v, got %v", expectedFrom, query.FirstInterval().From) } - if !query.Until.Equal(expectedUntil) { - t.Errorf("Expected Until: %v, got %v", expectedUntil, query.Until) + if !query.FirstInterval().Until.Equal(expectedUntil) { + t.Errorf("Expected Until: %v, got %v", expectedUntil, query.FirstInterval().Until) } // Verify Collectors diff --git a/periodicscraper/periodic_scraper_per_collector.go b/periodicscraper/periodic_scraper_per_collector.go index f529106..ea87985 100644 --- a/periodicscraper/periodic_scraper_per_collector.go +++ b/periodicscraper/periodic_scraper_per_collector.go @@ -116,14 +116,19 @@ func getDumps(ctx context.Context, dumpType := getDumpTypeFromBool(isRibsData) untilNextDay := time.Now().AddDate(0, 0, 1) + queryFrom := prevRunTimeEnd + if queryFrom.After(time.Unix(0, 0)) { + queryFrom = queryFrom.Add(-48 * time.Hour) + } + query := bgpfinder.Query{ Collectors: []bgpfinder.Collector{collector}, DumpType: dumpType, Intervals: []bgpfinder.Interval{ - { - From: prevRunTimeEnd, // Start from prevRuntime - Until: untilNextDay, // Until tomorrow (to ensure we get today's data) - }, + { + From: queryFrom, + Until: untilNextDay, // Until tomorrow (to ensure we get today's data) + }, }, } diff --git a/scrape_collectors.go b/scrape_collectors.go index 2efb10f..e05fb20 100644 --- a/scrape_collectors.go +++ b/scrape_collectors.go @@ -73,43 +73,5 @@ func scrapeProject(ctx context.Context, logger *logging.Logger, db *pgxpool.Pool return fmt.Errorf("failed to upsert collectors for project %s: %w", project.Name, err) } - for _, collector := range collectors { - if err := scrapeCollector(ctx, logger, db, finder, collector); err != nil { - logger.Error().Err(err).Str("collector", collector.Name).Msg("Failed to scrape collector") - continue - } - } - return nil -} - -func scrapeCollector(ctx context.Context, logger *logging.Logger, db *pgxpool.Pool, finder Finder, collector Collector) error { - logger.Info().Str("collector", collector.Name).Msg("Starting to scrape collector data") - - // Use a sensible default interval for periodic scraping (e.g., last 24 hours) - // But for the very first scrape, we might want more. - // For now, sticking to the existing "all time" logic but cleaned up. - query := Query{ - Collectors: []Collector{collector}, - DumpType: DumpTypeAny, - Intervals: []Interval{{ - From: time.Unix(0, 0), - Until: time.Now().Add(time.Hour * 24), - }}, - } - - dumps, err := finder.Find(query) - if err != nil { - return fmt.Errorf("finder.Find failed for collector %s: %w", collector.Name, err) - } - - logger.Info(). - Str("collector", collector.Name). - Int("dumps_found", len(dumps)). - Msg("Found BGP dumps for collector") - - if err := UpsertBGPDumps(ctx, logger, db, dumps); err != nil { - return fmt.Errorf("failed to upsert dumps for collector %s: %w", collector.Name, err) - } - return nil }