diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8059ad7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,13 @@ +name: CI +on: [push, pull_request] +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v6 + with: { go-version: '1.27.1' } + - run: go vet ./... + - run: go install honnef.co/go/tools/cmd/staticcheck@2026.2.1 + - run: staticcheck ./... + - run: go test -race ./... diff --git a/README.md b/README.md new file mode 100644 index 0000000..d62c078 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# firecrest-go + +`firecrest-go` is a small Go library and command-line client for [FirecREST v2](https://eth-cscs.github.io/firecrest-v2/). It targets the v2 path-based API: the HPC system is part of each URL, not an `X-Machine-Name` header. + +It covers OAuth2 client credentials, JSON job submission, job metadata/status, and direct small-file upload/download. Large asynchronous transfers deliberately remain out of scope for this first version. + +## Quick start + +```go +httpClient := firecrest.NewClientCredentialsHTTPClient(ctx, firecrest.ClientCredentials{ + TokenURL: "https://keycloak.example/auth/realms/myrealm/protocol/openid-connect/token", + ClientID: "my-client", ClientSecret: os.Getenv("CLIENT_SECRET"), +}) +client, err := firecrest.NewClient("https://firecrest.example", "my-system", httpClient) +id, err := client.Submit(ctx, firecrest.SubmitRequest{Job: firecrest.JobDescription{ + Script: "#!/bin/bash\necho hello", WorkingDirectory: "/home/me", +}}) +``` + +Every network method takes `context.Context`; cancel it to stop an in-flight request. API failures are `*firecrest.APIError`; use `errors.Is(err, firecrest.ErrUnauthorized)` or `errors.Is(err, firecrest.ErrNotFound)`. + +## CLI + +The CLI uses an existing bearer token, so it never puts a client secret on a command line: + +```bash +export FIRECREST_URL=https://firecrest.example FIRECREST_SYSTEM=my-system FIRECREST_TOKEN=… +firecrest-go submit job.sh /home/me +firecrest-go status 12345 +firecrest-go download /home/me/output.txt ./output.txt +``` + +## Verification + +Tests use `httptest`, require no running FirecREST stack, and are checked with `go vet`, `staticcheck`, and `go test -race`. No real Alps credentials or systems are exercised by this repository. + +## License + +Apache-2.0. See `LICENSE`. + +## Related work + +This is the Go companion to [firecrest-agentic-workbench](https://github.com/VadaLinux/firecrest-agentic-workbench). The workbench itself is intentionally not modified by this repository. diff --git a/auth.go b/auth.go new file mode 100644 index 0000000..6bd1aaf --- /dev/null +++ b/auth.go @@ -0,0 +1,29 @@ +package firecrest + +import ( + "context" + "net/http" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" +) + +// ClientCredentials configures Keycloak's OAuth2 client-credentials flow. +type ClientCredentials struct { + TokenURL string + ClientID string + ClientSecret string + Scopes []string +} + +// NewClientCredentialsHTTPClient returns an HTTP client that obtains and safely +// reuses OAuth2 tokens. oauth2's TokenSource serializes refreshes for callers. +func NewClientCredentialsHTTPClient(ctx context.Context, credentials ClientCredentials) *http.Client { + config := clientcredentials.Config{TokenURL: credentials.TokenURL, ClientID: credentials.ClientID, ClientSecret: credentials.ClientSecret, Scopes: credentials.Scopes} + return config.Client(ctx) +} + +// NewStaticTokenHTTPClient is useful for tests and externally managed tokens. +func NewStaticTokenHTTPClient(ctx context.Context, token string) *http.Client { + return oauth2.NewClient(ctx, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token, TokenType: "Bearer"})) +} diff --git a/cmd/firecrest-go/main.go b/cmd/firecrest-go/main.go new file mode 100644 index 0000000..77f1b8e --- /dev/null +++ b/cmd/firecrest-go/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "os" + "path/filepath" + + firecrest "github.com/VadaLinux/firecrest-go" +) + +func main() { + if len(os.Args) < 2 { + usage() + } + fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError) + baseURL := fs.String("url", os.Getenv("FIRECREST_URL"), "FirecREST base URL") + system := fs.String("system", os.Getenv("FIRECREST_SYSTEM"), "FirecREST system name") + token := fs.String("token", os.Getenv("FIRECREST_TOKEN"), "Bearer token") + fs.Parse(os.Args[2:]) + if *baseURL == "" || *system == "" || *token == "" { + fmt.Fprintln(os.Stderr, "-url, -system and -token (or matching environment variables) are required") + os.Exit(2) + } + client, err := firecrest.NewClient(*baseURL, *system, firecrest.NewStaticTokenHTTPClient(context.Background(), *token)) + if err != nil { + fatal(err) + } + switch os.Args[1] { + case "submit": + if fs.NArg() != 2 { + usage() + } + script, err := os.ReadFile(fs.Arg(0)) + if err != nil { + fatal(err) + } + id, err := client.Submit(context.Background(), firecrest.SubmitRequest{Job: firecrest.JobDescription{Script: string(script), WorkingDirectory: fs.Arg(1)}}) + if err != nil { + fatal(err) + } + fmt.Println(id) + case "status": + if fs.NArg() != 1 { + usage() + } + status, err := client.JobStatus(context.Background(), fs.Arg(0)) + if err != nil { + fatal(err) + } + fmt.Printf("job=%s stdout=%s stderr=%s\n", status.JobID, status.StandardOutput, status.StandardError) + case "download": + if fs.NArg() != 2 { + usage() + } + body, err := client.Download(context.Background(), fs.Arg(0)) + if err != nil { + fatal(err) + } + defer body.Close() + out, err := os.Create(filepath.Clean(fs.Arg(1))) + if err != nil { + fatal(err) + } + _, err = io.Copy(out, body) + closeErr := out.Close() + if err != nil { + fatal(err) + } + if closeErr != nil { + fatal(closeErr) + } + default: + usage() + } +} +func fatal(err error) { fmt.Fprintln(os.Stderr, "error:", err); os.Exit(1) } +func usage() { + fmt.Fprintln(os.Stderr, "usage: firecrest-go [flags] args") + os.Exit(2) +} diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..02905b3 --- /dev/null +++ b/errors.go @@ -0,0 +1,38 @@ +package firecrest + +import ( + "errors" + "fmt" + "io" + "net/http" +) + +var ( + ErrUnauthorized = errors.New("FirecREST unauthorized") + ErrNotFound = errors.New("FirecREST resource not found") +) + +// APIError exposes the HTTP status and FirecREST error envelope. +type APIError struct { + StatusCode int + Detail string + Err error +} + +func (e *APIError) Error() string { + return fmt.Sprintf("FirecREST API error: status=%d detail=%q", e.StatusCode, e.Detail) +} +func (e *APIError) Unwrap() error { return e.Err } + +func newAPIError(resp *http.Response) error { + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + apiErr := &APIError{StatusCode: resp.StatusCode, Detail: string(data)} + switch resp.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + apiErr.Err = ErrUnauthorized + case http.StatusNotFound: + apiErr.Err = ErrNotFound + } + return apiErr +} diff --git a/firecrest.go b/firecrest.go new file mode 100644 index 0000000..c7589c5 --- /dev/null +++ b/firecrest.go @@ -0,0 +1,186 @@ +// Package firecrest provides a small, context-aware client for FirecREST v2. +package firecrest + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/url" + "path" + "strings" +) + +// Client talks to one FirecREST installation and one configured HPC system. +type Client struct { + baseURL *url.URL + system string + http *http.Client +} + +// NewClient makes a client. httpClient must provide authentication (for example +// a client returned by NewClientCredentialsHTTPClient). +func NewClient(baseURL, system string, httpClient *http.Client) (*Client, error) { + u, err := url.Parse(baseURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return nil, fmt.Errorf("invalid base URL %q", baseURL) + } + if system == "" { + return nil, fmt.Errorf("system must not be empty") + } + if httpClient == nil { + httpClient = http.DefaultClient + } + return &Client{baseURL: u, system: system, http: httpClient}, nil +} + +// SubmitRequest is the JSON body accepted by POST /compute/{system}/jobs. +type SubmitRequest struct { + Job JobDescription `json:"job"` +} + +// JobDescription is deliberately limited to portable submission fields. +type JobDescription struct { + Script string `json:"script"` + WorkingDirectory string `json:"working_directory"` + Name string `json:"name,omitempty"` + Account string `json:"account,omitempty"` + Partition string `json:"partition,omitempty"` + StandardOutput string `json:"standard_output,omitempty"` + StandardError string `json:"standard_error,omitempty"` + Environment map[string]string `json:"env,omitempty"` +} + +// Submit submits a job and returns its scheduler identifier. +func (c *Client) Submit(ctx context.Context, request SubmitRequest) (string, error) { + var response struct { + JobID json.RawMessage `json:"jobId"` + } + if err := c.doJSON(ctx, http.MethodPost, "compute", "jobs", request, &response); err != nil { + return "", err + } + var id string + if err := json.Unmarshal(response.JobID, &id); err == nil { + return id, nil + } + var number json.Number + if err := json.Unmarshal(response.JobID, &number); err != nil { + return "", fmt.Errorf("decode jobId: %w", err) + } + return number.String(), nil +} + +// JobMetadata contains the script and standard streams returned in one v2 call. +type JobMetadata struct { + JobID string `json:"jobId"` + Script string `json:"script"` + StandardInput string `json:"standardInput"` + StandardOutput string `json:"standardOutput"` + StandardError string `json:"standardError"` +} + +// JobStatus returns metadata for a job, including standard output and error paths. +func (c *Client) JobStatus(ctx context.Context, jobID string) (JobMetadata, error) { + var response struct { + Jobs []JobMetadata `json:"jobs"` + } + if err := c.doJSON(ctx, http.MethodGet, "compute", path.Join("jobs", jobID, "metadata"), nil, &response); err != nil { + return JobMetadata{}, err + } + if len(response.Jobs) != 1 { + return JobMetadata{}, fmt.Errorf("decode job metadata: expected one job, got %d", len(response.Jobs)) + } + return response.Jobs[0], nil +} + +// Download fetches a small file through the synchronous filesystem ops endpoint. +// The caller owns closing the returned body. +func (c *Client) Download(ctx context.Context, remotePath string) (io.ReadCloser, error) { + req, err := c.request(ctx, http.MethodGet, "filesystem", "ops/download", nil) + if err != nil { + return nil, err + } + q := req.URL.Query() + q.Set("path", remotePath) + req.URL.RawQuery = q.Encode() + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("FirecREST request: %w", err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, newAPIError(resp) + } + return resp.Body, nil +} + +// Upload sends a small file through the synchronous multipart ops endpoint. +func (c *Client) Upload(ctx context.Context, directory, filename string, content io.Reader) error { + var body bytes.Buffer + w := multipart.NewWriter(&body) + part, err := w.CreateFormFile("file", filename) + if err == nil { + _, err = io.Copy(part, content) + } + if err == nil { + err = w.Close() + } + if err != nil { + return fmt.Errorf("build upload: %w", err) + } + req, err := c.request(ctx, http.MethodPost, "filesystem", "ops/upload", &body) + if err != nil { + return err + } + q := req.URL.Query() + q.Set("path", directory) + req.URL.RawQuery = q.Encode() + req.Header.Set("Content-Type", w.FormDataContentType()) + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("FirecREST request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return newAPIError(resp) + } + return nil +} + +func (c *Client) doJSON(ctx context.Context, method, area, endpoint string, input, output any) error { + var body io.Reader + if input != nil { + encoded, err := json.Marshal(input) + if err != nil { + return fmt.Errorf("encode request: %w", err) + } + body = bytes.NewReader(encoded) + } + req, err := c.request(ctx, method, area, endpoint, body) + if err != nil { + return err + } + if input != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("FirecREST request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return newAPIError(resp) + } + if err := json.NewDecoder(resp.Body).Decode(output); err != nil { + return fmt.Errorf("decode response: %w", err) + } + return nil +} + +func (c *Client) request(ctx context.Context, method, area, endpoint string, body io.Reader) (*http.Request, error) { + u := *c.baseURL + u.Path = strings.TrimRight(u.Path, "/") + "/" + path.Join(area, c.system, endpoint) + return http.NewRequestWithContext(ctx, method, u.String(), body) +} diff --git a/firecrest_test.go b/firecrest_test.go new file mode 100644 index 0000000..0306099 --- /dev/null +++ b/firecrest_test.go @@ -0,0 +1,128 @@ +package firecrest + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func testClient(t *testing.T, handler http.HandlerFunc) *Client { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + client, err := NewClient(server.URL, "cluster-a", NewStaticTokenHTTPClient(context.Background(), "test-token")) + if err != nil { + t.Fatal(err) + } + return client +} + +func TestSubmitUsesV2PathAndJSON(t *testing.T) { + client := testClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/compute/cluster-a/jobs" { + t.Errorf("path = %s", r.URL.Path) + } + if got := r.Header.Get("Authorization"); got != "Bearer test-token" { + t.Errorf("authorization = %q", got) + } + if got := r.Header.Get("X-Machine-Name"); got != "" { + t.Errorf("legacy header set: %q", got) + } + if got := r.Header.Get("Content-Type"); got != "application/json" { + t.Errorf("content type = %q", got) + } + body, _ := io.ReadAll(r.Body) + if !strings.Contains(string(body), `"working_directory":"/work"`) { + t.Errorf("body = %s", body) + } + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"jobId":42}`)) + }) + id, err := client.Submit(context.Background(), SubmitRequest{Job: JobDescription{Script: "echo hi", WorkingDirectory: "/work"}}) + if err != nil { + t.Fatal(err) + } + if id != "42" { + t.Fatalf("job ID = %q", id) + } +} + +func TestJobStatusUsesMetadataEndpoint(t *testing.T) { + client := testClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/compute/cluster-a/jobs/42/metadata" { + t.Errorf("path = %s", r.URL.Path) + } + _, _ = w.Write([]byte(`{"jobs":[{"jobId":"42","standardOutput":"/work/out","standardError":"/work/err"}]}`)) + }) + status, err := client.JobStatus(context.Background(), "42") + if err != nil { + t.Fatal(err) + } + if status.StandardOutput != "/work/out" || status.StandardError != "/work/err" { + t.Fatalf("status = %#v", status) + } +} + +func TestUploadAndDownload(t *testing.T) { + var uploaded bool + client := testClient(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/filesystem/cluster-a/ops/upload": + if r.URL.Query().Get("path") != "/work" { + t.Errorf("upload path = %q", r.URL.Query().Get("path")) + } + if err := r.ParseMultipartForm(1024); err != nil { + t.Fatal(err) + } + file, header, err := r.FormFile("file") + if err != nil { + t.Fatal(err) + } + defer file.Close() + body, _ := io.ReadAll(file) + if header.Filename != "input.txt" || string(body) != "hello" { + t.Errorf("file = %q %q", header.Filename, body) + } + uploaded = true + w.WriteHeader(http.StatusNoContent) + case "/filesystem/cluster-a/ops/download": + if r.URL.Query().Get("path") != "/work/out.txt" { + t.Errorf("download path = %q", r.URL.Query().Get("path")) + } + _, _ = w.Write([]byte("output")) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + }) + if err := client.Upload(context.Background(), "/work", "input.txt", strings.NewReader("hello")); err != nil { + t.Fatal(err) + } + if !uploaded { + t.Fatal("upload was not received") + } + body, err := client.Download(context.Background(), "/work/out.txt") + if err != nil { + t.Fatal(err) + } + defer body.Close() + got, _ := io.ReadAll(body) + if string(got) != "output" { + t.Fatalf("download = %q", got) + } +} + +func TestAPIErrorIsQueryable(t *testing.T) { + client := testClient(t, func(w http.ResponseWriter, r *http.Request) { http.Error(w, "missing", http.StatusNotFound) }) + _, err := client.JobStatus(context.Background(), "unknown") + if !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } + var apiErr *APIError + if !errors.As(err, &apiErr) || apiErr.StatusCode != http.StatusNotFound { + t.Fatalf("expected APIError, got %#v", err) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..83beedb --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/VadaLinux/firecrest-go + +go 1.27.1 + +require golang.org/x/oauth2 v0.30.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c05b468 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=