feat: support API-level label filtering during sync

.github/workflows/ci.yaml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Build image
         run: docker build --tag=tp .
@@ -35,9 +35,9 @@ jobs:
     name: Static Checks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2 # required
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5 # required
+      - uses: actions/setup-go@v5
         with:
-          go-version: 1.15.1
-      - uses: pre-commit/action@v2.0.0
+          go-version: '1.25'
+      - uses: pre-commit/action@v3.0.1

Makefile

@@ -2,8 +2,8 @@ REGISTRY ?= gcr.io/k8s-minikube
 TAG ?= v0.0.13
 
 .PHONY: push-latest-dev-image
-push-latest-dev-image: 
+push-latest-dev-image:
 	docker login gcr.io/k8s-minikube
 	docker buildx create --name multiarch --bootstrap
 	docker buildx build --push --builder multiarch --platform linux/amd64,linux/arm64 -t $(REGISTRY)/triage-party:$(TAG) -t $(REGISTRY)/triage-party:latest .
-	docker buildx rm multiarch
+	docker buildx rm multiarch

docs/config.md

@@ -28,7 +28,7 @@ There are only a handful of site-wide settings worth mentioning:
 
 * `name`: Name of the your Triage Party site
 * `min_similarity`: On a scale from 0-1, how similar do two titles need to be before they are labelled as similar. The default is 0 (disabled), but a useful setting is 0.75
-* `repos`: A list of repositories to query by default
+* `repos`: A list of repositories to query by default. You can append query parameters to the repository URL to filter issues at the API level (e.g., `https://github.com/org/repo?labels=foo,bar`). This is useful for large repositories to reduce the number of synced issues and avoid hitting GitHub API limits.
 * `member-roles`: Which GitHub roles to consider as project members
 * `members`: A list of people to hard-code as members of the project
 

go.mod

@@ -1,8 +1,6 @@
 module github.com/google/triage-party
 
-go 1.23.0
-
-toolchain go1.24.6
+go 1.25.0
 
 require (
 	github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20200501161113-5e9e23d7cb91

pkg/hubbub/analyze.go

@@ -81,11 +81,11 @@ func (h *Engine) analyzeIssue(ctx context.Context, i *provider.Issue, sp provide
 	}
 
 	if !preFetchMatch(i, labels, sp.Filters) {
-		klog.V(1).Infof("#%d - %q did not match item filter: %s", i.GetNumber(), i.GetTitle(), sp.Filters)
+		klog.V(1).Infof("#%d - %q did not match item filter: %v", i.GetNumber(), i.GetTitle(), sp.Filters)
 		return nil
 	}
 
-	klog.V(1).Infof("#%d - %q made it past pre-fetch: %s", i.GetNumber(), i.GetTitle(), sp.Filters)
+	klog.V(1).Infof("#%d - %q made it past pre-fetch: %v", i.GetNumber(), i.GetTitle(), sp.Filters)
 
 	fetchComments := false
 	if needComments(i, sp.Filters) && i.GetComments() > 0 {
@@ -111,10 +111,10 @@ func (h *Engine) analyzeIssue(ctx context.Context, i *provider.Issue, sp provide
 	}
 
 	if !postFetchMatch(co, sp.Filters) {
-		klog.V(1).Infof("#%d - %q did not match post-fetch filter: %s", i.GetNumber(), i.GetTitle(), sp.Filters)
+		klog.V(1).Infof("#%d - %q did not match post-fetch filter: %v", i.GetNumber(), i.GetTitle(), sp.Filters)
 		return nil
 	}
-	klog.V(1).Infof("#%d - %q made it past post-fetch: %s", i.GetNumber(), i.GetTitle(), sp.Filters)
+	klog.V(1).Infof("#%d - %q made it past post-fetch: %v", i.GetNumber(), i.GetTitle(), sp.Filters)
 
 	updatedAt := h.mtime(i)
 	var timeline []*provider.Timeline
@@ -144,11 +144,11 @@ func (h *Engine) analyzeIssue(ctx context.Context, i *provider.Issue, sp provide
 	co.PullRequestRefs = h.updateLinkedPRs(ctx, sp, co)
 
 	if !postEventsMatch(co, sp.Filters) {
-		klog.V(1).Infof("#%d - %q did not match post-events filter: %s", i.GetNumber(), i.GetTitle(), sp.Filters)
+		klog.V(1).Infof("#%d - %q did not match post-events filter: %v", i.GetNumber(), i.GetTitle(), sp.Filters)
 		return nil
 	}
 
-	klog.V(1).Infof("#%d - %q made it past post-events: %s", i.GetNumber(), i.GetTitle(), sp.Filters)
+	klog.V(1).Infof("#%d - %q made it past post-events: %v", i.GetNumber(), i.GetTitle(), sp.Filters)
 	return co
 }
 
@@ -269,7 +269,7 @@ func (h *Engine) analyzePR(ctx context.Context, pr *provider.PullRequest, sp pro
 	}
 
 	if !postEventsMatch(co, sp.Filters) {
-		klog.V(1).Infof("#%d - %q did not match post-events filter: %s", pr.GetNumber(), pr.GetTitle(), sp.Filters)
+		klog.V(1).Infof("#%d - %q did not match post-events filter: %v", pr.GetNumber(), pr.GetTitle(), sp.Filters)
 		return nil
 	}
 

pkg/hubbub/cache.go

@@ -16,22 +16,31 @@ package hubbub
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/google/triage-party/pkg/provider"
 )
 
 // issueSearchKey is the cache key used for issues
 func issueSearchKey(sp provider.SearchParams) string {
+	labelSuffix := ""
+	if len(sp.Repo.Labels) > 0 {
+		labelSuffix = "-" + strings.Join(sp.Repo.Labels, "-")
+	}
 	if sp.UpdateAge > 0 {
-		return fmt.Sprintf("%s-%s-%s-issues-within-%.1fh", sp.Repo.Organization, sp.Repo.Project, sp.State, sp.UpdateAge.Hours())
+		return fmt.Sprintf("%s-%s%s-%s-issues-within-%.1fh", sp.Repo.Organization, sp.Repo.Project, labelSuffix, sp.State, sp.UpdateAge.Hours())
 	}
-	return fmt.Sprintf("%s-%s-%s-issues", sp.Repo.Organization, sp.Repo.Project, sp.State)
+	return fmt.Sprintf("%s-%s%s-%s-issues", sp.Repo.Organization, sp.Repo.Project, labelSuffix, sp.State)
 }
 
 // prSearchKey is the cache key used for prs
 func prSearchKey(sp provider.SearchParams) string {
+	labelSuffix := ""
+	if len(sp.Repo.Labels) > 0 {
+		labelSuffix = "-" + strings.Join(sp.Repo.Labels, "-")
+	}
 	if sp.UpdateAge > 0 {
-		return fmt.Sprintf("%s-%s-%s-prs-within-%.1fh", sp.Repo.Organization, sp.Repo.Project, sp.State, sp.UpdateAge.Hours())
+		return fmt.Sprintf("%s-%s%s-%s-prs-within-%.1fh", sp.Repo.Organization, sp.Repo.Project, labelSuffix, sp.State, sp.UpdateAge.Hours())
 	}
-	return fmt.Sprintf("%s-%s-%s-prs", sp.Repo.Organization, sp.Repo.Project, sp.State)
+	return fmt.Sprintf("%s-%s%s-%s-prs", sp.Repo.Organization, sp.Repo.Project, labelSuffix, sp.State)
 }

pkg/hubbub/item.go

@@ -244,7 +244,7 @@ func (h *Engine) isMember(user string, role string) bool {
 		return true
 	}
 
-	klog.V(1).Infof("%s (%s) is not considered a member: members=%s memberRoles=%s", user, role, h.members, h.memberRoles)
+	klog.V(1).Infof("%s (%s) is not considered a member: members=%v memberRoles=%v", user, role, h.members, h.memberRoles)
 	return false
 }
 
@@ -324,7 +324,7 @@ func (h *Engine) parseRefs(text string, co *Conversation, t time.Time) {
 		project := m[2]
 		i, err := strconv.Atoi(m[3])
 		if err != nil {
-			klog.Errorf("unable to parse int from %s: %v", err)
+			klog.Errorf("unable to parse int from %s: %v", m[3], err)
 			continue
 		}
 

pkg/hubbub/match.go

@@ -106,7 +106,7 @@ func preFetchMatch(i provider.IItem, labels []*provider.Label, fs []provider.Fil
 
 		if f.Reactions != "" || f.ReactionsPerMonth != "" || f.Commenters != "" || f.Comments != "" {
 			if !i.GetUpdatedAt().After(i.GetCreatedAt()) {
-				klog.V(1).Infof("#%d has no updates, but need one for: %s", i.GetNumber(), f)
+				klog.V(1).Infof("#%d has no updates, but need one for: %v", i.GetNumber(), f)
 				return false
 			}
 		}
@@ -182,7 +182,7 @@ func postEventsMatch(co *Conversation, fs []provider.Filter) bool {
 	for _, f := range fs {
 		if f.TagRegex() != nil {
 			if ok, _ := matchTag(co.Tags, f.TagRegex(), f.TagNegate()); !ok {
-				klog.V(4).Infof("#%d did not pass matchTag: %s vs %s %v", co.ID, co.Tags, f.TagRegex(), f.TagNegate())
+				klog.V(4).Infof("#%d did not pass matchTag: %v vs %s %v", co.ID, co.Tags, f.TagRegex(), f.TagNegate())
 				return false
 			}
 		}

pkg/hubbub/search.go

@@ -70,7 +70,7 @@ func (h *Engine) SearchAny(ctx context.Context, sp provider.SearchParams) ([]*Co
 func (h *Engine) SearchIssues(ctx context.Context, sp provider.SearchParams) ([]*Conversation, time.Time, error) {
 	sp.Filters = openByDefault(sp)
 	klog.V(1).Infof(
-		"Gathering raw data for %s/%s issues %s - newer than %s",
+		"Gathering raw data for %s/%s issues %v - newer than %s",
 		sp.Repo.Organization,
 		sp.Repo.Project,
 		sp.Filters,
@@ -143,14 +143,14 @@ func (h *Engine) SearchIssues(ctx context.Context, sp provider.SearchParams) ([]
 		}
 
 		if seen[i.GetURL()] {
-			klog.Errorf("unusual: I already saw #%d", i.GetURL())
+			klog.Errorf("unusual: I already saw #%d", i.GetNumber())
 			continue
 		}
 		seen[i.GetURL()] = true
 		is = append(is, i)
 	}
 
-	klog.V(1).Infof("%s/%s aggregate issue count: %d, filtering for:\n%s", sp.Repo.Organization, sp.Repo.Project, len(is), sp.Filters)
+	klog.V(1).Infof("%s/%s aggregate issue count: %d, filtering for:\n%v", sp.Repo.Organization, sp.Repo.Project, len(is), sp.Filters)
 
 	// Avoids updating PR references on a quiet repository
 	latestIssueUpdate := time.Time{}
@@ -188,7 +188,7 @@ func NeedsClosed(fs []provider.Filter) bool {
 func (h *Engine) SearchPullRequests(ctx context.Context, sp provider.SearchParams) ([]*Conversation, time.Time, error) {
 	sp.Filters = openByDefault(sp)
 
-	klog.V(1).Infof("Gathering raw data for %s/%s PR's matching: %s - newer than %s",
+	klog.V(1).Infof("Gathering raw data for %s/%s PR's matching: %v - newer than %s",
 		sp.Repo.Organization, sp.Repo.Project, sp.Filters, logu.STime(sp.NewerThan))
 
 	var wg sync.WaitGroup

pkg/persist/mysql.go

@@ -94,7 +94,7 @@ func (m *MySQL) Set(key string, th *Blob) error {
 		ge := gob.NewEncoder(b)
 
 		if err := ge.Encode(th); err != nil {
-			klog.Errorf("encode: %w", err)
+			klog.Errorf("encode: %v", err)
 		}
 
 		_, err := m.db.Exec(`
@@ -130,7 +130,7 @@ func (m *MySQL) Get(key string, t time.Time) *Blob {
 	}
 
 	if err != nil {
-		klog.Errorf("query: %w", err)
+		klog.Errorf("query: %v", err)
 		return nil
 	}
 

pkg/persist/postgres.go

@@ -116,7 +116,7 @@ func (m *Postgres) Get(key string, t time.Time) *Blob {
 	}
 
 	if err != nil {
-		klog.Errorf("query: %w", err)
+		klog.Errorf("query: %v", err)
 		return nil
 	}
 

pkg/provider/github.go

@@ -80,11 +80,15 @@ func (p *GitHubProvider) getResponse(i *github.Response) *Response {
 }
 
 func (p *GitHubProvider) getIssueListByRepoOptions(sp SearchParams) *github.IssueListByRepoOptions {
-	return &github.IssueListByRepoOptions{
+	opt := &github.IssueListByRepoOptions{
 		ListOptions: p.getListOptions(sp.IssueListByRepoOptions.ListOptions),
 		State:       sp.IssueListByRepoOptions.State,
 		Since:       sp.IssueListByRepoOptions.Since,
 	}
+	if len(sp.Repo.Labels) > 0 {
+		opt.Labels = sp.Repo.Labels
+	}
+	return opt
 }
 
 func (p *GitHubProvider) IssuesListByRepo(ctx context.Context, sp SearchParams) (i []*Issue, r *Response, err error) {

pkg/provider/github_test.go

@@ -16,6 +16,9 @@ package provider
 
 import (
 	"testing"
+
+	"github.com/google/go-github/v33/github"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestGitHub_GetResponse(t *testing.T) {
@@ -57,3 +60,52 @@ func TestGitHub_GetPullRequestsListReviews(t *testing.T) {
 	p := GitHubProvider{}
 	p.getPullRequestsListReviews(nil)
 }
+
+func TestGitHub_GetIssueListByRepoOptions(t *testing.T) {
+	p := GitHubProvider{}
+
+	tests := []struct {
+		name string
+		sp   SearchParams
+		want *github.IssueListByRepoOptions
+	}{
+		{
+			name: "no labels",
+			sp:   SearchParams{},
+			want: &github.IssueListByRepoOptions{},
+		},
+		{
+			name: "with labels",
+			sp: SearchParams{
+				Repo: Repo{
+					Labels: []string{"bug", "p0"},
+				},
+			},
+			want: &github.IssueListByRepoOptions{
+				Labels: []string{"bug", "p0"},
+			},
+		},
+		{
+			name: "with other options and labels",
+			sp: SearchParams{
+				Repo: Repo{
+					Labels: []string{"feature"},
+				},
+				IssueListByRepoOptions: IssueListByRepoOptions{
+					State: "closed",
+				},
+			},
+			want: &github.IssueListByRepoOptions{
+				State:  "closed",
+				Labels: []string{"feature"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := p.getIssueListByRepoOptions(tt.sp)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

pkg/provider/models.go

@@ -55,6 +55,7 @@ type Repo struct {
 	Project      string
 	Host         string
 	Group        string
+	Labels       []string
 }
 
 type SearchParams struct {

pkg/provider/provider.go

@@ -47,15 +47,15 @@ func ReadToken(path string, envVar string) string {
 			klog.Exitf("unable to read token file: %v", err)
 		}
 		token := strings.TrimSpace(string(t))
-		klog.Infof("loaded %d byte %s token from %s", len(token), path)
+		klog.Infof("loaded %d byte token from %s", len(token), path)
 		return token
 	}
 
 	token := strings.TrimSpace(os.Getenv(envVar))
 	if token == "" {
 		klog.Warningf("No token found in environment variable %s (empty)", envVar)
 	} else {
-		klog.Infof("loaded %d byte %s token from %s", len(token), envVar)
+		klog.Infof("loaded %d byte token from %s", len(token), envVar)
 	}
 	return token
 }

pkg/triage/rule.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"sort"
 	"strings"
 	"time"
 
@@ -220,18 +221,36 @@ func parseRepo(rawURL string) (r provider.Repo, err error) {
 		err = fmt.Errorf("expected 2/3 repository parts, got %d: %v", len(parts), parts)
 		return
 	}
+
+	var labelList []string
+	seen := make(map[string]bool)
+	for _, labels := range u.Query()["labels"] {
+		for _, l := range strings.Split(labels, ",") {
+			l = strings.TrimSpace(l)
+			if l != "" && !seen[l] {
+				seen[l] = true
+				labelList = append(labelList, l)
+			}
+		}
+	}
+	if len(labelList) > 0 {
+		sort.Strings(labelList)
+	}
+
 	if len(parts) == 3 {
 		r = provider.Repo{
 			Host:         u.Host,
 			Organization: parts[1],
 			Project:      parts[2],
+			Labels:       labelList,
 		}
 	} else {
 		r = provider.Repo{
 			Host:         u.Host,
 			Organization: parts[1],
 			Group:        parts[2],
 			Project:      parts[3],
+			Labels:       labelList,
 		}
 	}