Skip to content

Commit 7e55ae2

Browse files
andrewandrew
authored
Restrict inferred repository URLs to known forges (#56)
* Restrict inferred repository URLs to known forges * Fix PyPI repository label matching * Fix repository URL fallback edge cases
1 parent fe89084 commit 7e55ae2

20 files changed

Lines changed: 287 additions & 27 deletions

‎internal/cran/cran.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ func parseDescription(content string) descriptionInfo {
178178
func extractRepository(urlField string) string {
179179
urls := strings.Split(urlField, ",")
180180
for _, u := range urls {
181-
if parsed := urlparser.Parse(strings.TrimSpace(u)); parsed != "" {
181+
if parsed := urlparser.CanonicalURL(strings.TrimSpace(u)); parsed != "" {
182182
return parsed
183183
}
184184
}

‎internal/cran/cran_test.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,34 @@ func TestFetchPackage(t *testing.T) {
6565
}
6666
}
6767

68+
func TestExtractRepositoryUsesOnlyKnownForges(t *testing.T) {
69+
tests := []struct {
70+
name string
71+
urlField string
72+
want string
73+
}{
74+
{
75+
name: "skip project website",
76+
urlField: "https://example.com/projects/ggplot2, https://github.com/tidyverse/ggplot2",
77+
want: "https://github.com/tidyverse/ggplot2",
78+
},
79+
{
80+
name: "reject project website",
81+
urlField: "https://example.com/projects/ggplot2",
82+
want: "",
83+
},
84+
}
85+
86+
for _, tt := range tests {
87+
t.Run(tt.name, func(t *testing.T) {
88+
got := extractRepository(tt.urlField)
89+
if got != tt.want {
90+
t.Errorf("extractRepository() = %q, want %q", got, tt.want)
91+
}
92+
})
93+
}
94+
}
95+
6896
func TestFetchVersions(t *testing.T) {
6997
mux := http.NewServeMux()
7098

‎internal/dub/dub.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package
8787
// Extract repository URL
8888
repository := urlparser.Parse(resp.Repository)
8989
if repository == "" {
90-
repository = urlparser.Parse(resp.Homepage)
90+
repository = urlparser.CanonicalURL(resp.Homepage)
9191
}
9292

9393
return &core.Package{

‎internal/dub/dub_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,27 @@ func TestFetchPackage(t *testing.T) {
5656
}
5757
}
5858

59+
func TestFetchPackageDoesNotUseNonForgeHomepageAsRepository(t *testing.T) {
60+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
61+
resp := packageResponse{
62+
Name: "example",
63+
Homepage: "https://example.com/packages/example",
64+
}
65+
_ = json.NewEncoder(w).Encode(resp)
66+
}))
67+
defer server.Close()
68+
69+
reg := New(server.URL, core.DefaultClient())
70+
pkg, err := reg.FetchPackage(context.Background(), "example")
71+
if err != nil {
72+
t.Fatalf("FetchPackage failed: %v", err)
73+
}
74+
75+
if pkg.Repository != "" {
76+
t.Errorf("expected no repository, got %q", pkg.Repository)
77+
}
78+
}
79+
5980
func TestFetchVersions(t *testing.T) {
6081
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
6182
resp := packageResponse{

‎internal/golang/golang.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ func (r *Registry) fetchPackagePkgsite(ctx context.Context, name string) (*core.
130130
repoURL = mod.RepoURL
131131
}
132132
if repoURL == "" {
133-
repoURL = urlparser.Parse(deriveRepoURL(name))
133+
repoURL = urlparser.CanonicalURL(deriveRepoURL(name))
134134
}
135135

136136
var types []string
@@ -164,7 +164,7 @@ func (r *Registry) fetchPackageProxy(ctx context.Context, name string) (*core.Pa
164164
return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name}
165165
}
166166

167-
repoURL := urlparser.Parse(deriveRepoURL(name))
167+
repoURL := urlparser.CanonicalURL(deriveRepoURL(name))
168168

169169
return &core.Package{
170170
Name: name,

‎internal/golang/golang_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,27 @@ func TestFetchPackage(t *testing.T) {
3838
}
3939
}
4040

41+
func TestFetchPackageDoesNotDeriveRepositoryFromVanityPath(t *testing.T) {
42+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
43+
if r.URL.Path == "/golang.org/x/sync/@v/list" {
44+
_, _ = w.Write([]byte("v0.7.0\n"))
45+
return
46+
}
47+
w.WriteHeader(404)
48+
}))
49+
defer server.Close()
50+
51+
reg := New(server.URL, core.DefaultClient())
52+
pkg, err := reg.FetchPackage(context.Background(), "golang.org/x/sync")
53+
if err != nil {
54+
t.Fatalf("FetchPackage failed: %v", err)
55+
}
56+
57+
if pkg.Repository != "" {
58+
t.Errorf("expected no repository, got %q", pkg.Repository)
59+
}
60+
}
61+
4162
func TestFetchPackagePkgsite(t *testing.T) {
4263
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
4364
if r.URL.Path != "/module/golang.org/x/sync" {

‎internal/haxelib/haxelib.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package
7979
}
8080

8181
// Extract repository URL from website
82-
repository := urlparser.Parse(resp.Website)
82+
repository := urlparser.CanonicalURL(resp.Website)
8383

8484
return &core.Package{
8585
Name: resp.Name,

‎internal/haxelib/haxelib_test.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,27 @@ func TestFetchPackage(t *testing.T) {
5959
}
6060
}
6161

62+
func TestFetchPackageDoesNotUseNonForgeWebsiteAsRepository(t *testing.T) {
63+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
64+
resp := packageResponse{
65+
Name: "example",
66+
Website: "https://example.com/packages/example",
67+
}
68+
_ = json.NewEncoder(w).Encode(resp)
69+
}))
70+
defer server.Close()
71+
72+
reg := New(server.URL, core.DefaultClient())
73+
pkg, err := reg.FetchPackage(context.Background(), "example")
74+
if err != nil {
75+
t.Fatalf("FetchPackage failed: %v", err)
76+
}
77+
78+
if pkg.Repository != "" {
79+
t.Errorf("expected no repository, got %q", pkg.Repository)
80+
}
81+
}
82+
6283
func TestFetchVersions(t *testing.T) {
6384
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
6485
resp := packageResponse{

‎internal/hex/hex.go‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,16 +107,26 @@ func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package
107107
}
108108

109109
var homepage, repository string
110-
if gh, ok := links["github"]; ok {
111-
repository = urlparser.Parse(gh)
110+
for _, key := range []string{"github", "repository", "source", "source code", "code"} {
111+
if link := links[key]; link != "" {
112+
repository = urlparser.Parse(link)
113+
if repository != "" {
114+
break
115+
}
116+
}
112117
}
113118
for k, v := range links {
114119
if k != "github" && homepage == "" {
115120
homepage = v
116121
}
117122
}
118123
if repository == "" {
119-
repository = urlparser.Parse(homepage)
124+
for _, link := range links {
125+
if parsed := urlparser.CanonicalURL(link); parsed != "" {
126+
repository = parsed
127+
break
128+
}
129+
}
120130
}
121131

122132
return &core.Package{

‎internal/hex/hex_test.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,56 @@ func TestFetchPackage(t *testing.T) {
5656
}
5757
}
5858

59+
func TestFetchPackageDoesNotUseNonForgeHomepageAsRepository(t *testing.T) {
60+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
61+
resp := packageResponse{
62+
Name: "example",
63+
Meta: metaInfo{
64+
Links: map[string]string{
65+
"Website": "https://example.com/packages/example",
66+
},
67+
},
68+
}
69+
_ = json.NewEncoder(w).Encode(resp)
70+
}))
71+
defer server.Close()
72+
73+
reg := New(server.URL, core.DefaultClient())
74+
pkg, err := reg.FetchPackage(context.Background(), "example")
75+
if err != nil {
76+
t.Fatalf("FetchPackage failed: %v", err)
77+
}
78+
79+
if pkg.Repository != "" {
80+
t.Errorf("expected no repository, got %q", pkg.Repository)
81+
}
82+
}
83+
84+
func TestFetchPackageUsesExplicitSelfHostedRepositoryLink(t *testing.T) {
85+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
86+
resp := packageResponse{
87+
Name: "example",
88+
Meta: metaInfo{
89+
Links: map[string]string{
90+
"Repository": "https://git.example.com/example/example",
91+
},
92+
},
93+
}
94+
_ = json.NewEncoder(w).Encode(resp)
95+
}))
96+
defer server.Close()
97+
98+
reg := New(server.URL, core.DefaultClient())
99+
pkg, err := reg.FetchPackage(context.Background(), "example")
100+
if err != nil {
101+
t.Fatalf("FetchPackage failed: %v", err)
102+
}
103+
104+
if pkg.Repository != "https://git.example.com/example/example" {
105+
t.Errorf("unexpected repository: %q", pkg.Repository)
106+
}
107+
}
108+
59109
func TestFetchVersions(t *testing.T) {
60110
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
61111
switch r.URL.Path {

0 commit comments

Comments
 (0)