Resolve some annoying issues in the periodic scraper

cmd/bgpf/main.go

@@ -140,8 +140,12 @@ func (f *FilesCmd) Run(parentLogger *logging.Logger, cli BgpfCLI) error {
 
 	query := bgpfinder.Query{
 		Collectors: collectors,
-		From:       fromTime,
-		Until:      untilTime,
+		Intervals: []bgpfinder.Interval{
+			{
+				From:  fromTime,
+				Until: untilTime,
+			},
+		},
 		DumpType:   f.Type,
 	}
 

cmd/bgpfinder-server/main_test.go

@@ -50,12 +50,12 @@ func TestParseDataRequest(t *testing.T) {
 	expectedFrom := time.Unix(startTimeInt64, 0)
 	expectedUntil := time.Unix(endTimeInt64, 0)
 
-	if !query.From.Equal(expectedFrom) {
-		t.Errorf("Expected From: %v, got %v", expectedFrom, query.From)
+	if !query.FirstInterval().From.Equal(expectedFrom) {
+		t.Errorf("Expected From: %v, got %v", expectedFrom, query.FirstInterval().From)
 	}
 
-	if !query.Until.Equal(expectedUntil) {
-		t.Errorf("Expected Until: %v, got %v", expectedUntil, query.Until)
+	if !query.FirstInterval().Until.Equal(expectedUntil) {
+		t.Errorf("Expected Until: %v, got %v", expectedUntil, query.FirstInterval().Until)
 	}
 
 	// Verify Collectors

periodicscraper/periodic_scraper_per_collector.go

@@ -116,14 +116,19 @@ func getDumps(ctx context.Context,
 	dumpType := getDumpTypeFromBool(isRibsData)
 	untilNextDay := time.Now().AddDate(0, 0, 1)
 
+	queryFrom := prevRunTimeEnd
+	if queryFrom.After(time.Unix(0, 0)) {
+		queryFrom = queryFrom.Add(-48 * time.Hour)
+	}
+
 	query := bgpfinder.Query{
 		Collectors: []bgpfinder.Collector{collector},
 		DumpType:   dumpType,
 		Intervals: []bgpfinder.Interval{
-		    {
-			From:       prevRunTimeEnd,   // Start from prevRuntime
-			Until:      untilNextDay, // Until tomorrow (to ensure we get today's data) 
-		    },
+			{
+				From:  queryFrom,
+				Until: untilNextDay, // Until tomorrow (to ensure we get today's data)
+			},
 		},
 	}
 

scrape_collectors.go

@@ -73,43 +73,5 @@ func scrapeProject(ctx context.Context, logger *logging.Logger, db *pgxpool.Pool
 		return fmt.Errorf("failed to upsert collectors for project %s: %w", project.Name, err)
 	}
 
-	for _, collector := range collectors {
-		if err := scrapeCollector(ctx, logger, db, finder, collector); err != nil {
-			logger.Error().Err(err).Str("collector", collector.Name).Msg("Failed to scrape collector")
-			continue
-		}
-	}
-	return nil
-}
-
-func scrapeCollector(ctx context.Context, logger *logging.Logger, db *pgxpool.Pool, finder Finder, collector Collector) error {
-	logger.Info().Str("collector", collector.Name).Msg("Starting to scrape collector data")
-
-	// Use a sensible default interval for periodic scraping (e.g., last 24 hours)
-	// But for the very first scrape, we might want more.
-	// For now, sticking to the existing "all time" logic but cleaned up.
-	query := Query{
-		Collectors: []Collector{collector},
-		DumpType:   DumpTypeAny,
-		Intervals: []Interval{{
-			From:  time.Unix(0, 0),
-			Until: time.Now().Add(time.Hour * 24),
-		}},
-	}
-
-	dumps, err := finder.Find(query)
-	if err != nil {
-		return fmt.Errorf("finder.Find failed for collector %s: %w", collector.Name, err)
-	}
-
-	logger.Info().
-		Str("collector", collector.Name).
-		Int("dumps_found", len(dumps)).
-		Msg("Found BGP dumps for collector")
-
-	if err := UpsertBGPDumps(ctx, logger, db, dumps); err != nil {
-		return fmt.Errorf("failed to upsert dumps for collector %s: %w", collector.Name, err)
-	}
-
 	return nil
 }