Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ Every request requires `Authorization: Bearer <ADMIN_TOKEN>`. Responses are enve
| ------ | ---------------------------------------------- | ----------- | ---------------------------------------------------------------------- |
| `GET` | `/repositories/{id}/contents?ref=&path=` | — | List one directory level of the tree at a ref. |
| `GET` | `/repositories/{id}/blob?ref=&path=&lfs=` | — | Stream one file's raw content. |
| `GET` | `/repositories/{id}/archive?ref=&format=&lfs=` | — | Stream a `zip` (default) or `tar.gz` archive of the tree. |
| `GET` | `/repositories/{id}/archive?ref=&format=&lfs=&prefix=` | — | Stream a `zip` (default) or `tar.gz` archive of the tree. |
| `POST` | `/repositories/{id}/blobs` | _raw bytes_ | Upload content into the repo's object database; returns `{sha, size}`. |
| `POST` | `/repositories/{id}/commits` | JSON | Create a commit on a branch from uploaded blobs. |

Expand Down Expand Up @@ -196,7 +196,9 @@ Every request requires `Authorization: Bearer <ADMIN_TOKEN>`. Responses are enve

`GET /blob` streams the file bytes with `Content-Length`, a strong `ETag` (the blob sha — content-addressed, so `If-None-Match` caching works perfectly), and `X-HeadlessGit-Commit` carrying the resolved commit. With `lfs=true`, an LFS pointer file is replaced by the real object; a missing object is a `404` rather than silently serving the pointer.

`GET /archive` streams the whole tree as an artifact, named `<repo>-<shortsha>.zip` with a matching top-level folder. With `lfs=true`, pointer files are swapped for the real objects **in-flight** — the archive is re-encoded entry by entry, nothing is buffered or written to disk:
`GET /archive` streams the whole tree as an artifact, named `<repo>-<shortsha>.zip`. By default its entries are under the matching `<repo>-<shortsha>/` directory. Set `prefix=release/source` to choose another directory, or explicitly set `prefix=` to place entries at the archive root. Prefixes are relative directory paths and a trailing `/` is optional.

With `lfs=true`, pointer files are swapped for the real objects **in-flight** — the archive is re-encoded entry by entry, nothing is buffered or written to disk:

![archive](images/archive.png)

Expand Down
15 changes: 15 additions & 0 deletions internal/archive/transform_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,21 @@ func TestTransformWithoutSmudge(t *testing.T) {
}
}

func TestTransformWithoutPrefix(t *testing.T) {
var out bytes.Buffer
if err := Transform(bytes.NewReader(buildTestTar(t, "not a pointer")), "", nil, NewZipEncoder(&out)); err != nil {
t.Fatal(err)
}

got := readZip(t, out.Bytes())
if got["README.md"] != "hello\n" {
t.Errorf("README.md = %q", got["README.md"])
}
if _, ok := got["src/"]; !ok {
t.Errorf("missing root-level directory entry")
}
}

func TestTransformTarGz(t *testing.T) {
oid := strings.Repeat("ab", 32)
content := "REAL LFS CONTENT"
Expand Down
35 changes: 34 additions & 1 deletion internal/domain/archive.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
package domain

import "fmt"
import (
"fmt"
"strings"
"unicode"
"unicode/utf8"
)

const archivePrefixMaxLength = 255

type ArchiveFormat string

Expand Down Expand Up @@ -31,6 +38,32 @@ type ArchiveRequest struct {
CommitSHA string
Format ArchiveFormat
IncludeLFS bool
Prefix string
}

func NormalizeArchivePrefix(prefix string) (string, bool) {
if prefix == "" {
return "", true
}
if len(prefix) > archivePrefixMaxLength || !utf8.ValidString(prefix) {
return "", false
}
if strings.HasPrefix(prefix, "/") || strings.ContainsAny(prefix, `\:`) {
return "", false
}
for _, r := range prefix {
if unicode.IsControl(r) {
return "", false
}
}

prefix = strings.TrimSuffix(prefix, "/")
for _, segment := range strings.Split(prefix, "/") {
if segment == "" || segment == "." || segment == ".." {
return "", false
}
}
return prefix + "/", true
}

// Filename is the suggested artifact name: <repo>-<shortsha>.<ext>
Expand Down
40 changes: 40 additions & 0 deletions internal/domain/archive_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package domain

import (
"strings"
"testing"
)

func TestNormalizeArchivePrefix(t *testing.T) {
cases := []struct {
name string
prefix string
want string
ok bool
}{
{"empty", "", "", true},
{"directory", "release", "release/", true},
{"nested", "release/source", "release/source/", true},
{"trailing slash", "release/source/", "release/source/", true},
{"unicode", "releases/été", "releases/été/", true},
{"absolute", "/release", "", false},
{"dot segment", "release/./source", "", false},
{"parent segment", "release/../source", "", false},
{"empty segment", "release//source", "", false},
{"multiple trailing slashes", "release//", "", false},
{"backslash", `release\source`, "", false},
{"windows volume", "C:/release", "", false},
{"control character", "release\nsource", "", false},
{"too long", strings.Repeat("a", 256), "", false},
{"invalid utf8", string([]byte{0xff}), "", false},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, ok := NormalizeArchivePrefix(tc.prefix)
if ok != tc.ok || got != tc.want {
t.Fatalf("NormalizeArchivePrefix(%q) = %q, %v; want %q, %v", tc.prefix, got, ok, tc.want, tc.ok)
}
})
}
}
15 changes: 14 additions & 1 deletion internal/server/control/repositories/archive.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package repositories

import (
"crypto/sha256"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -29,7 +30,15 @@ func (h *handlers) getArchive(w http.ResponseWriter, r *http.Request) error {
}
}

req, err := h.service.PrepareArchive(r.Context(), id, q.Get("ref"), q.Get("format"), includeLFS)
var prefix *string
if values, ok := q["prefix"]; ok {
if len(values) != 1 {
return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "prefix must be specified once")
}
prefix = &values[0]
}

req, err := h.service.PrepareArchive(r.Context(), id, q.Get("ref"), q.Get("format"), includeLFS, prefix)
switch {
case errors.Is(err, reposervice.ErrRepositoryNotFound):
return response.NewError(http.StatusNotFound, response.CodeRepositoryNotFound, "repository not found")
Expand All @@ -39,6 +48,8 @@ func (h *handlers) getArchive(w http.ResponseWriter, r *http.Request) error {
return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid ref")
case errors.Is(err, reposervice.ErrUnsupportedFormat):
return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "unsupported archive format")
case errors.Is(err, reposervice.ErrInvalidArchivePrefix):
return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "invalid archive prefix")
case errors.Is(err, reposervice.ErrLFSNotEnabled):
return response.NewError(http.StatusBadRequest, response.CodeInvalidRequest, "lfs is not enabled")
case err != nil:
Expand Down Expand Up @@ -91,6 +102,8 @@ func archiveETag(req domain.ArchiveRequest) string {
if req.IncludeLFS {
variant += "-lfs"
}
prefixHash := sha256.Sum256([]byte(req.Prefix))
variant += fmt.Sprintf("-p%x", prefixHash)
return fmt.Sprintf(`W/"%s-%s"`, req.CommitSHA, variant)
}

Expand Down
2 changes: 1 addition & 1 deletion internal/server/control/repositories/handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ type RepositoryManager interface {
SetVisibility(ctx context.Context, repositoryID int64, visibility domain.RepoVisibility) (domain.Repository, error)
ListByOwner(ctx context.Context, ownerID int64) ([]domain.Repository, error)
Contents(ctx context.Context, repositoryID int64, ref, treePath string) (domain.RepositoryContents, error)
PrepareArchive(ctx context.Context, repositoryID int64, ref, format string, includeLFS bool) (domain.ArchiveRequest, error)
PrepareArchive(ctx context.Context, repositoryID int64, ref, format string, includeLFS bool, prefix *string) (domain.ArchiveRequest, error)
StreamArchive(ctx context.Context, req domain.ArchiveRequest, out io.Writer) error
PrepareBlob(ctx context.Context, repositoryID int64, ref, treePath string, includeLFS bool) (domain.BlobRequest, error)
StreamBlob(ctx context.Context, req domain.BlobRequest, out io.Writer) error
Expand Down
51 changes: 48 additions & 3 deletions internal/server/control/repositories/handlers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ type fakeManager struct {
RepositoryManager
prepareReq domain.ArchiveRequest
prepareErr error
prefix string
prefixSet bool
streamBody string
streamErr error

Expand Down Expand Up @@ -79,7 +81,11 @@ func (f *fakeManager) Commit(ctx context.Context, repositoryID int64, req domain
return f.commitResult, f.commitErr
}

func (f fakeManager) PrepareArchive(ctx context.Context, repositoryID int64, ref, format string, includeLFS bool) (domain.ArchiveRequest, error) {
func (f *fakeManager) PrepareArchive(ctx context.Context, repositoryID int64, ref, format string, includeLFS bool, prefix *string) (domain.ArchiveRequest, error) {
if prefix != nil {
f.prefix = *prefix
f.prefixSet = true
}
return f.prepareReq, f.prepareErr
}

Expand Down Expand Up @@ -136,6 +142,7 @@ func testArchiveRequest() domain.ArchiveRequest {
Repository: domain.Repository{ID: 7, RepositoryName: "myrepo"},
CommitSHA: testSHA,
Format: domain.ArchiveFormatZip,
Prefix: "myrepo-aaaabbbbcccc/",
}
}

Expand All @@ -154,7 +161,7 @@ func TestGetArchive(t *testing.T) {
if got := rec.Header().Get("Content-Disposition"); got != `attachment; filename="myrepo-aaaabbbbcccc.zip"` {
t.Errorf("Content-Disposition = %q", got)
}
if got := rec.Header().Get("ETag"); got != `W/"`+testSHA+`-zip"` {
if got := rec.Header().Get("ETag"); got != archiveETag(testArchiveRequest()) {
t.Errorf("ETag = %q", got)
}
if got := rec.Header().Get("X-HeadlessGit-Commit"); got != testSHA {
Expand All @@ -175,7 +182,7 @@ func TestGetArchiveNotModified(t *testing.T) {
router := newTestRouter(&fakeManager{prepareReq: testArchiveRequest(), streamBody: "hello"})

req := httptest.NewRequest(http.MethodGet, "/repositories/7/archive?ref=main", nil)
req.Header.Set("If-None-Match", `W/"`+testSHA+`-zip"`)
req.Header.Set("If-None-Match", archiveETag(testArchiveRequest()))
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)

Expand All @@ -187,6 +194,42 @@ func TestGetArchiveNotModified(t *testing.T) {
}
}

func TestGetArchivePrefixParameter(t *testing.T) {
cases := []struct {
name string
target string
want string
wantSet bool
}{
{"omitted", "/repositories/7/archive", "", false},
{"custom", "/repositories/7/archive?prefix=release%2Fsource", "release/source", true},
{"empty", "/repositories/7/archive?prefix=", "", true},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
svc := &fakeManager{prepareReq: testArchiveRequest(), streamBody: "hello"}
rec := httptest.NewRecorder()
newTestRouter(svc).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, tc.target, nil))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d: %s", rec.Code, rec.Body.String())
}
if svc.prefixSet != tc.wantSet || svc.prefix != tc.want {
t.Errorf("prefix = %q, set %v; want %q, set %v", svc.prefix, svc.prefixSet, tc.want, tc.wantSet)
}
})
}
}

func TestArchiveETagVariesByPrefix(t *testing.T) {
a := testArchiveRequest()
b := testArchiveRequest()
b.Prefix = "release/"
if archiveETag(a) == archiveETag(b) {
t.Fatal("archive ETags must vary by prefix")
}
}

func TestGetArchiveErrors(t *testing.T) {
cases := []struct {
name string
Expand All @@ -198,10 +241,12 @@ func TestGetArchiveErrors(t *testing.T) {
}{
{"bad id", "/repositories/abc/archive", nil, nil, http.StatusBadRequest, "invalid_request"},
{"bad lfs param", "/repositories/7/archive?lfs=maybe", nil, nil, http.StatusBadRequest, "invalid_request"},
{"duplicate prefix", "/repositories/7/archive?prefix=a&prefix=b", nil, nil, http.StatusBadRequest, "invalid_request"},
{"repo not found", "/repositories/7/archive", reposervice.ErrRepositoryNotFound, nil, http.StatusNotFound, "repository_not_found"},
{"ref not found", "/repositories/7/archive?ref=nope", reposervice.ErrRefNotFound, nil, http.StatusNotFound, "ref_not_found"},
{"invalid ref", "/repositories/7/archive?ref=--x", reposervice.ErrInvalidRef, nil, http.StatusBadRequest, "invalid_request"},
{"bad format", "/repositories/7/archive?format=rar", reposervice.ErrUnsupportedFormat, nil, http.StatusBadRequest, "invalid_request"},
{"bad prefix", "/repositories/7/archive?prefix=..", reposervice.ErrInvalidArchivePrefix, nil, http.StatusBadRequest, "invalid_request"},
{"lfs disabled", "/repositories/7/archive?lfs=true", reposervice.ErrLFSNotEnabled, nil, http.StatusBadRequest, "invalid_request"},
{"stream fails before first byte", "/repositories/7/archive", nil, io.ErrUnexpectedEOF, http.StatusInternalServerError, "internal_error"},
}
Expand Down
5 changes: 3 additions & 2 deletions internal/services/repositories/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@ var (
ErrInvalidRef = errors.New("invalid ref")
ErrInvalidPath = errors.New("invalid path")

ErrUnsupportedFormat = errors.New("unsupported archive format")
ErrLFSNotEnabled = errors.New("lfs is not enabled")
ErrUnsupportedFormat = errors.New("unsupported archive format")
ErrInvalidArchivePrefix = errors.New("invalid archive prefix")
ErrLFSNotEnabled = errors.New("lfs is not enabled")

ErrNotAFile = errors.New("path is not a file")
ErrLFSObjectNotFound = errors.New("lfs object not found")
Expand Down
19 changes: 16 additions & 3 deletions internal/services/repositories/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,11 +247,20 @@ func (s *Service) GetRepositoryByPath(ctx context.Context, namespace, name strin
return toDomain(repo), nil
}

func (s *Service) PrepareArchive(ctx context.Context, repositoryID int64, ref, format string, includeLFS bool) (domain.ArchiveRequest, error) {
func (s *Service) PrepareArchive(ctx context.Context, repositoryID int64, ref, format string, includeLFS bool, prefix *string) (domain.ArchiveRequest, error) {
f, ok := domain.ParseArchiveFormat(format)
if !ok {
return domain.ArchiveRequest{}, ErrUnsupportedFormat
}

normalizedPrefix := ""
if prefix != nil {
normalizedPrefix, ok = domain.NormalizeArchivePrefix(*prefix)
if !ok {
return domain.ArchiveRequest{}, ErrInvalidArchivePrefix
}
}

if includeLFS && s.lfs == nil {
return domain.ArchiveRequest{}, ErrLFSNotEnabled
}
Expand All @@ -271,11 +280,16 @@ func (s *Service) PrepareArchive(ctx context.Context, repositoryID int64, ref, f
return domain.ArchiveRequest{}, err
}

if prefix == nil {
normalizedPrefix = fmt.Sprintf("%s-%s/", repo.RepositoryName, domain.ShortSHA(sha))
}

return domain.ArchiveRequest{
Repository: repo,
CommitSHA: sha,
Format: f,
IncludeLFS: includeLFS,
Prefix: normalizedPrefix,
}, nil
}

Expand Down Expand Up @@ -315,8 +329,7 @@ func (s *Service) StreamArchive(ctx context.Context, req domain.ArchiveRequest,
}
}

prefix := fmt.Sprintf("%s-%s/", req.Repository.RepositoryName, domain.ShortSHA(req.CommitSHA))
return archive.Transform(pr, prefix, smudge, enc)
return archive.Transform(pr, req.Prefix, smudge, enc)
}

func (s *Service) PrepareBlob(ctx context.Context, repositoryID int64, ref, treePath string, includeLFS bool) (domain.BlobRequest, error) {
Expand Down
Loading
Loading