diff --git a/.commitlintrc.yaml b/.commitlintrc.yaml new file mode 100644 index 00000000..7ed6a30d --- /dev/null +++ b/.commitlintrc.yaml @@ -0,0 +1,3 @@ +--- +extends: + - '@commitlint/config-conventional' diff --git a/.github/workflows/lint-commit.yml b/.github/workflows/lint-commit.yml index 15fdea85..167d1315 100644 --- a/.github/workflows/lint-commit.yml +++ b/.github/workflows/lint-commit.yml @@ -1,9 +1,5 @@ name: Lint Commit Messages on: - push: - branches: - - main - - develop pull_request: permissions: diff --git a/.github/workflows/quality-gates.yml b/.github/workflows/quality-gates.yml index e4ce74cc..2bcbcdbf 100644 --- a/.github/workflows/quality-gates.yml +++ b/.github/workflows/quality-gates.yml @@ -24,7 +24,7 @@ jobs: extends: relaxed rules: line-length: - max: 80 + max: 100 level: warning indentation: disable diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..714dee38 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,51 @@ +name: Release + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + packages: write +env: + CONTAINER_REGISTRY: ghcr.io + +jobs: + release: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + # history is needed for the goreleaser to generate changelogs and couple of other things + fetch-depth: 0 + + - uses: actions/setup-go@v5 + with: + go-version: '1.24.4' + + - name: Extract spec version + run: | + MARGO_SPEC_VERSION=$(grep -A 0 ' version:' ./standard/snapshot.spec.yaml | tail -1 | awk '{print $2}') + echo "MARGO_SPEC_VERSION=${MARGO_SPEC_VERSION}" >> $GITHUB_ENV + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.CONTAINER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ github.TOKEN }} + + # Builds binaries and generates the release changelog. + # For details on the release artifacts, see: docs/release.md + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v7 + with: + version: v2 + args: release --clean + env: + GITHUB_TOKEN: ${{ github.token }} + MARGO_SPEC_VERSION: ${{ env.MARGO_SPEC_VERSION }} diff --git a/.golangci.yml b/.golangci.yml index 3810bff7..6e5d7727 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -50,14 +50,12 @@ linters: generated: lax warn-unused: false presets: [] - rules: # Tests can have unused variables and short error handling - path: "_test\\.go$" linters: - unused - errcheck - # Mock files excluded from unused checks - path: "mock_.*\\.go$" linters: diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 00000000..7877320f --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,75 @@ +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +version: 2 + +project_name: margo-sandbox + +before: + hooks: + - go mod tidy + - go mod download + +builds: + - id: "linux-amd64" + main: ./poc/device/agent + # Binary name. + # Can be a path (e.g. `bin/app`). + binary: device-agent + flags: + - -trimpath + ldflags: + - -X main.build={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} + goos: + - linux + goarch: + - amd64 + - arm64 + +dockers_v2: + - images: + - "ghcr.io/margo/workload-fleet-management-client" + dockerfile: ./poc/device/agent/Dockerfile + platforms: + - linux/amd64 + - linux/arm64 + build_args: + VERSION: "{{.Version}}" + DATE: "{{.CommitDate}}" + COMMIT: "{{.Commit}}" + labels: + org.opencontainers.image.title: "{{.ProjectName}}" + org.opencontainers.image.description: "Margo sandbox workload fleet management client. A reference implementation." + org.opencontainers.image.source: "{{.GitURL}}" + org.opencontainers.image.version: "{{.Version}}" + org.opencontainers.image.revision: "{{.FullCommit}}" + org.opencontainers.image.created: "{{.CommitDate}}" + +report_sizes: true + +archives: + - formats: + - tar.gz + name_template: '{{.ProjectName}}_{{.Version}}_{{.Os}}-{{.Arch}}' + files: + - README.md + - LICENSE + +checksum: + name_template: "{{.ProjectName}}_checksums.txt" + +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + - 'README' + - Merge pull request + - Merge branch + +release: + footer: | + ## This release targets the Margo specification version: {{env.MARGO_SPEC_VERSION}}, and can be tracked here, https://github.com/margo/specification . + github: + owner: margo + name: sandbox + disable: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..f253c02d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,258 @@ +# Contributing to Margo Sandbox + +Thank you for contributing! This guide covers development workflow, commit standards, and code quality expectations. + +## Getting Started + +1. **Fork & Clone**: Fork the repository and clone locally +2. **Branch**: Create a feature branch from `development` (see Branch Naming below) +3. **Setup**: Install dependencies and run tests locally +4. **Code**: Make your changes following the style guide +5. **Test**: Ensure all tests pass and coverage is maintained +6. **Commit**: Use conventional commits (see below) +7. **Push**: Push to your fork +8. **PR**: Create a pull request against `development` + +--- + +## Commit Style + +All commits must follow [Conventional Commits](https://conventionalcommits.org/) format: + +``` +type(scope): description + +[optional body] + +[optional footer: BREAKING CHANGE: ...] +``` + +### Commit Types Sample + +| Type | Purpose | Affects Version | +|------|---------|-----------------| +| `feat` | New feature or capability | **MINOR** or **MAJOR** bump depending upon whether there is a breaking change | +| `fix` | Bug fix or issue resolution | **PATCH** bump | +| `docs` | Documentation only | No version bump | +| `test` | Test additions/improvements | No version bump | +| `refactor` | Code restructuring (no feature/fix) | No version bump | +| `chore` | Maintenance, deps, tooling | No version bump* | + +*`chore(deps)` (dependency updates) = **PATCH** bump + +### Rules + +- **Scope** (optional): e.g., `cli`, `device-agent`, `helm`, `shared-lib` +- **Description**: lowercase + - ✅ Good: `feat(cli): add config validation` + - ❌ Bad: `Added config validation to CLI` +- **BREAKING CHANGE**: Include in footer for API/config or other incompatibilities; triggers **MAJOR** version bump + ``` + feat(api): change user endpoint structure + + BREAKING CHANGE: The endpoint now returns a different JSON structure. + See migration guide in docs/API-MIGRATION.md + ``` + +### Example Commits + +``` +feat(cli): add support for config validation +fix(device-agent): resolve connection timeout +chore(deps): update Go to 1.21 +feat(wfm): add metrics collection (BREAKING CHANGE: API response structure changed) +docs: update installation guide +``` + +### Enforcement + +- **CommitLint** validates all commits in CI +- Non-conformant commits block PR merge +- Use `git commit --amend` to fix messages before pushing + +For details on versioning implications, see [docs/release.md](docs/release.md#versioning). + +--- + +## Branch Naming + +Create branches from the appropriate base: + +| Branch Type | Pattern | Base | Purpose | +|-------------|---------|------|---------| +| Feature | `feature/short-description` | `development` | New feature (e.g., `feature/device-telemetry`) | +| Bug fix | `fix/short-description` | `development` | Regular bug fixes (e.g., `fix/connection-leak`) | +| Hotfix | `hotfix/short-description` | `main` | Critical production bugs only | +| Maintenance | `chore/short-description` | `development` | Deps, tooling, CI (e.g., `chore/update-go`) | +| Refactoring | `refactor/short-description` | `development` | Code restructuring (e.g., `refactor/api-client`) | +| Documentation | `docs/short-description` | `development` | Doc-only updates | + +### Workflow + +1. **Create branch**: + ```bash + git checkout develop + git pull origin develop + git checkout -b feature/my-feature + ``` + +2. **Work locally**: + - Commit regularly with conventional format + - Push to your fork: `git push origin feature/my-feature` + - Keep branch up-to-date: `git rebase origin/develop` + +3. **Open PR**: + - Describe what you changed and why + - Link related issues (e.g., "Fixes #123") + - Request review from CODEOWNERS + +4. **Merge**: + - Get approval from at least 1 maintainer + - Squash & merge to base branch (CI enforces this) + - Delete your branch after merge + +--- + +## Testing & Coverage + +### Before You Submit a PR + +1. **Run tests locally**: + ```bash + go test ./... + go test -coverprofile=coverage.out ./... + go tool cover -html=coverage.out + ``` + +2. **Ensure coverage**: + - Minimum **80%** repo-wide coverage required + - New code should maintain/improve coverage + - Check [Codecov.io](https://codecov.io) after pushing + +3. **Run security scan**: + ```bash + go install github.com/securego/gosec/v2/cmd/gosec@latest + gosec ./... + ``` + +4. **Check for lint issues**: + - Follow Go conventions (gofmt, golint) + - Tests must pass in CI before merge + +### Test Guidelines + +- Write tests for new features and bug fixes +- Include both happy path and error cases +- Aim for clear, descriptive test names +- Use table-driven tests for multiple scenarios + +--- + +## Code Style + +### Go Standards + +- Follow [Effective Go](https://golang.org/doc/effective_go) +- Use `gofmt` for automatic formatting +- Package names in lowercase, no underscores +- Exported names should be descriptive + +### Documentation + +- Document public functions and types +- Include examples in package-level comments +- Update README.md if user-facing features change + +--- + +## Dependencies + +### Adding New Packages + +1. Add to `go.mod` via `go get` +2. **Check license**: MIT, Apache 2.0, BSD, GPL (with approval), or OSI-approved only + ```bash + go mod graph | grep import-path + ``` +3. Document why in your commit + +### Updating Dependencies + +- Dependabot creates PRs automatically +- Critical vulnerabilities are prioritized +- Review and test before merging updates + +--- + +## Pull Request Process + +### PR Checklist + +- [ ] Branch created from correct base (`development` or `main` for hotfix) +- [ ] Commits follow conventional format +- [ ] All tests pass locally +- [ ] Coverage maintained or improved +- [ ] No high-severity security issues (GoSec passes) +- [ ] PR description is clear and links related issues +- [ ] Requested reviews from CODEOWNERS + +### PR Description Template + +```markdown +## Description +[Brief summary of changes] + +## Motivation +[Why this change? What problem does it solve?] + +## Testing +[How did you test this? What scenarios?] + +## Checklist +- [ ] Tests added/updated +- [ ] Documentation updated +- [ ] No breaking changes (or documented in footer) +- [ ] Conventional commits used +``` + +### Responding to Reviews + +- Address feedback promptly +- Push additional commits (CI will squash on merge) +- Request re-review when ready +- Ask questions if feedback is unclear + +--- + +## Reporting Issues + +1. **Search existing issues** before creating new ones +2. **Include details**: + - Steps to reproduce + - Expected vs actual behavior + - OS, Go version, environment + - Error logs/stack traces +3. **Use labels**: `bug`, `enhancement`, `question`, etc. +4. **Link related PRs** if applicable + +--- + +## Release Process + +For details on versioning, scheduling, and release procedures, see [docs/release.md](docs/release.md). + +Key points for contributors: +- You **don't need to release**; maintainers handle it +- Your commits' `type` (feat, fix, docs) automatically determine version bumps +- Use `BREAKING CHANGE:` footer if you make incompatible changes +- Review release notes after your PR merges to ensure they reflect your contribution + +--- + +## Questions? + +- **Bug reports**: [GitHub Issues](https://github.com/margo/sandbox/issues) +- **Feature requests**: Via [SUPs](https://github.com/margo) +- **Help**: [Discourse Channel](https://discourse.margo.org/) + +Thank you for contributing! diff --git a/README.md b/README.md index 283a38b5..4e1f0c8d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ - [Design and Mapping to Margo Architecture](#design-and-mapping-to-margo-architecture) - [Symphony WFM](#symphony-wfm) - [Repositories and Registry](#repositories-and-registry) - - [Telemetry and Monitoring](#telemetry-and-monitoring) + - [Telemetry and Monitoring](#telemetry-and-monitoring) - [HTTP/1.1 and API Security](#http11-and-api-security) - [Release Notes](#release-notes) - [Comments and Feedback](#comments-and-feedback) @@ -54,7 +54,7 @@ Here is [Setup Guide](./docs/setup-guide.md) to get you started quickly. The repository is divided into three main parts. You can find more details here on [Repository Structure](./docs/repo-structure.md): - `shared-lib`: Reusable libraries and utilities (Open Source Components) -- `standard`: Implementation of the components as per Margo specification +- `standard`: Implementation of the components as per Margo specification, a snapshot of the the implemented spec is copied from the official sources in this directory for traceability, in case the original source gets changed later on. - `non-standard`: Enabling components, which are not defined by Margo, but required for an overall implementation --- diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 00000000..7f979b10 --- /dev/null +++ b/docs/release.md @@ -0,0 +1,247 @@ +# Release Process + +This document is for **release managers and maintainers**, outlining versioning strategy, release workflow, and deliverables for the Sandbox. + +The 'Sandbox' release corresponds to a specific version of the MARGO Specification, including the OpenAPI swagger definition of the specification, as mentioned in the release notes. + +The release may contain SUPs (features) or bug-fixes / improvement suggestions raised by the community, The release notes will have a list of submissions which are part of the release. + +**For developers**: See [CONTRIBUTING.md](../CONTRIBUTING.md) for commit standards, branch naming, and PR workflow. + +## Scope and Release Strategy + +**Key principle**: All sandbox components receive **a single version number** per release. Any codebase change in (device agent, WFM, shared-lib) results in a version bump for the entire repository. This ensures clear dependency management and simplified compatibility tracking. + +**Note**: This process applies to the sandbox repo only. The Margo spec project has its own versioning cycle and can be tracked in its respective repository. + +--- + +## Versioning + +We follow [Semantic Versioning (SemVer)](https://semver.org/): `MAJOR.MINOR.PATCH` + +**Developer commits** follow [Conventional Commits](https://conventionalcommits.org/) format (see [CONTRIBUTING.md](../CONTRIBUTING.md#commit-style)): + +| Change Type | Version Bump | When | +|-------------|--------------|------| +| Breaking changes | MAJOR | Any commit with `BREAKING CHANGE:` footer | +| New features | MINOR | `feat` commits | +| Bug fixes | PATCH | `fix` commits or dependency updates | +| Docs/chore | No bump | Documentation or non-dependency maintenance | + +**When multiple commit types exist**, highest priority wins: `BREAKING CHANGE` > `feat` > `fix/chore(deps)` > `docs` + +Example: `1.0.0` + feat + fix + docs → `1.1.0` (MINOR takes precedence) + +### Release Types + +**Stable Release** (`v1.2.3`): +- Production-ready, tested, immutable once released +- Tagged from `main` → CI builds and publishes artifacts + +**Release Candidate** (`v1.2.3-rc.1`): +- Pre-release for community testing during code freeze +- Published to GHCR with full tag (e.g., `ghcr.io/margo/device-agent:v1.2.3-rc.1`) +- Community decides when to promote RC to stable (typically 2+ weeks of testing) + +**Nightly Build** (`nightly`): +- Automated daily snapshot from `development` branch +- For development/testing only, **NOT for production** +- Tagged as `ghcr.io/margo/device-agent:nightly` (overwritten daily) +- Only latest nightly retained + +## Tools and Automation + +| Tool | Purpose | Config | Notes | +|------|---------|--------|-------| +| **GoReleaser** | Build binaries, images, checksums, release notes | [.goreleaser.yaml](.goreleaser.yaml) | Triggered on `v*` tags | +| **GHCR** | Docker image registry | Built-in | Multi-platform manifest support | +| **CommitLint** | Validate conventional commits in CI | `.commitlint.config.js` | Blocks non-compliant PRs | +| **GoSec** | Security scan on source code | Built-in | Runs in CI pipeline | +| **Go Test** | Golang based unit/integration tests | Built-in | N/A | + +**CI Pipelines**: +- **Release Pipeline**: Triggered on `git tag v*` (automatic GoReleaser build + publish) +- **Nightly Pipeline**: Daily schedule against `development` branch (tagged `nightly`) + +## Testing and Verification + +- Deploy to staging; verify its working +- Collect community testing feedback via Discourse and GitHub Issues +- If critical issues found: Create hotfix release or rollback + +## Release Workflow + +The typical workflow for preparing a release is as follows: + +### Phase 1: Code Freeze & Stabilization (1-2 weeks) + +1. **Declare Code Freeze**: Announce code freeze on the `development` branch via discussion channels (e.g., Discourse, Teams, GitHub Discussions). Specify the freeze window (typically 1-2 weeks). + +2. **Stabilization Criteria**: During the freeze window, the release is considered stable when: + - All test cases pass (unit, integration, end-to-end, and conformance tests) + - Test coverage meets the defined threshold + - All critical and high-priority bugs are resolved + - Community feedback on `-rc.N` versions is addressed (if applicable) + +3. **Hotfixes During Freeze**: See the [Immutable Releases / Hotfix Procedure](#immutable-releases) section below for the full hotfix workflow. + +4. **Who Decides Stabilization**: Release lead (determined by designated maintainer or CODEOWNERS) confirms when freeze criteria are met and stabilization is complete. + +### Phase 2: Release & Tagging + +5. **Merge to Main**: Merge all approved changes from `development` into `main`. Ensure all tests pass in CI. + +6. **Create Tag**: Create a Git tag from `main` following SemVer (e.g., `git tag v1.2.3`). + - Tag format: `v{MAJOR}.{MINOR}.{PATCH}` for stable, `v{MAJOR}.{MINOR}.{PATCH}-{prerelease}` for RCs. + +7. **Push Tag & Trigger Release**: Push the tag to the repository (see Roles and Responsibilities for who performs this). + - CI pipeline automatically triggers on tag push (no manual trigger needed) + - GoReleaser builds binaries, Docker images, checksums, changelog and generates release notes + +8. **Publish Artifacts**: + - Binary artifacts and checksums are published to GitHub Releases. + - Docker images are pushed to GHCR with version tags and platform-specific tags. + +### Phase 3: Post-Release + +9. **Create Discussion Thread**: Open a release discussion thread in Discourse for the Margo community to: + - Provide feedback and suggestions. + - Ask questions about the release. + +10. **Continue Development**: Meanwhile, development continues on the `development` branch. + +11. **Announce Release**: Notify the community via Teams, Discourse, mailing lists, or other communication channels with highlights and/or release notes. + +### Automation Implementation Details + +- Releases are automated using [GoReleaser](https://goreleaser.com/) triggered by tags matching `v*` in [.github/workflows/release.yml](.github/workflows/release.yml). +- GoReleaser handles building binaries, docker images, checksums, and changelogs as defined in [.goreleaser.yaml](.goreleaser.yaml). + +## Immutable Releases + +**Principle**: Once a version is released, artifacts cannot be modified. This ensures reproducibility and security. + +**Hotfix Procedure** (for critical bugs discovered during code freeze or post-release): +1. Create `hotfix/` branch from `main` (the commit tagged as the previous release) +2. Fix the issue, test thoroughly +3. Merge hotfixes to both `main` and `development` +4. During code freeze: Each hotfix will result in an RC increment (e.g., `v1.2.0-rc.1` → `v1.2.0-rc.2`) +5. Post-release: Tag a new patch version (e.g., `v1.2.1`) and release +6. Never re-upload or modify artifacts from the original release + +**Emergency Pre-releases** (rare): For critical security issues, release as RC (e.g., `v1.2.1-rc.1`) with accelerated testing and community consensus. + +## Release Checklist + +### Pre-Release (Code Freeze) +- [ ] Code freeze announced with dates and criteria +- [ ] All features for this release merged to `development` +- [ ] All tests pass (unit, integration, e2e) +- [ ] Code coverage as defined in this document +- [ ] GoSec security scan passes +- [ ] No critical bugs open +- [ ] Helm deployment tested on staging +- [ ] Release lead approves readiness + +### Release Execution +- [ ] Merge `development` → `main` with PR review +- [ ] CI checks pass on `main` +- [ ] Tag created (`v1.2.3`) from `main` +- [ ] Tag pushed (triggers CI automatically) +- [ ] GoReleaser completes successfully +- [ ] Binary artifacts + checksums verified +- [ ] Docker images pushed to GHCR +- [ ] Helm chart version updated +- [ ] Release notes generated +- [ ] SBOM generated and attached +- [ ] GitHub Release page published + +### Post-Release +- [ ] Release announced via Discourse/Teams/channels +- [ ] Community feedback collected +- [ ] Monitor for critical issues + +## Deliverables + +Each release produces the following artifacts, published to GitHub Releases and GHCR: + +### 1. Executable Binaries +- **Device Agent** binary for multiple platforms: + - Linux amd64 (x86-64) + - Linux arm64 +- **Format**: ELF executables +- **Checksums**: SHA256 checksums provided for each binary +- **Location**: GitHub Releases page + +### 2. Container Images +- **Multi-platform Docker images** built for platforms: + - amd64 (x86-64 architecture) + - arm64 (ARM 64-bit architecture) +- **Registry**: GitHub Container Registry (GHCR) +- **Tag examples**: + - Release: `ghcr.io/margo/device-agent:v1.2.3` + - Release candidate: `ghcr.io/margo/device-agent:v1.2.3-rc.1` + - Nightly: `ghcr.io/margo/device-agent:nightly` (continuously updated) +- **Manifest**: Multi-platform manifests are created automatically by Docker buildx to allow `docker pull` to fetch the correct platform. + +### 3. Release Notes +- **Auto-generated** summary of changes from commits since the previous release +- **Content**: List of features, fixes, breaking changes, and dependency updates. Documentation-only commits are excluded. +- **Location**: GitHub Releases page + +### 4. Changelog +- **Auto-generated** by GoReleaser from conventional commits + +### 5. Software Bill of Materials (SBOM) +- **Format**: SPDX +- **Location**: GitHub Releases attachment + +## Backwards Compatibility & Deprecation + +**Within Major Versions**: Code written for `v2.0.0` should work for all the minor/patch versions like `v2.3.5` without changes. + +**Breaking Changes** (requires MAJOR bump): +- API schema changes +- Config file format changes +- Binary input/output format changes +- Include clear migration guide in release notes (with code examples and rollback instructions), if Margo agrees on providing the backwards compatibility. + +## Communication + +### Release Announcements + +- **Timing**: Announce after release is published (artifacts available in GitHub Releases and GHCR) +- **Channels**: + - [Discourse](https://discourse.margo.org/) + - Teams + - Other Margo Community channels +- **Content**: + - Link to GitHub Release page + +### Support & Feedback + +- **Issue reporting**: Community members report issues in GitHub Issues +- **Discussion response**: Maintainers respond promptly to Discourse and Github Issues + +## Roles and Responsibilities + +| Role | Who | Responsibilities | +|------|-----|------------------| +| **Maintainers** | Dev Team (See CODEOWNERS) | Approve PRs, declare freeze, assert readiness, push tags, monitor release | +| **QA/Community** | Dev Team (See CODEOWNERS) + SUP Owners | Execute tests, report bugs, verify RCs, smoke tests | +| **DevOps/Release Manager** | Dev Team (See CODEOWNERS) | Maintain workflows, push tags, verify artifacts, manage nightly builds | + +**Release Lead** (per CODEOWNERS, @ajcraig, @phil-abb): Confirms code freeze criteria met and signs off on stabilization readiness + +See [CONTRIBUTING.md](../CONTRIBUTING.md) for developer responsibilities (commit standards, testing, PR process). + +--- + +### Steps currently not covered + +- Minimum test coverage check in the pipeline +- SBOM generation +- HELM chart and docker-compose as part of release artifacts +- Section for `security and compliances strategy` for the release pipeline +- Signing of release artifacts diff --git a/standard/generate.sh b/standard/generate.sh index 933e55e5..2fa45e10 100644 --- a/standard/generate.sh +++ b/standard/generate.sh @@ -5,9 +5,10 @@ export PATH="$PATH:$HOME/go/bin" # Configuration #WFM_SBI_SPEC=("spec/wfm-sbi.yaml") -TMP_SPEC=$(mktemp /tmp/wfm-sbi-tmp-XXXXXX.yaml) +TMP_SPEC="snapshot.spec.yaml" +SPEC_URL="https://raw.githubusercontent.com/margo/specification/pre-draft/system-design/specification/margo-management-interface/workload-management-api-1.0.0.yaml" curl -sSL -o "$TMP_SPEC" \ - "https://raw.githubusercontent.com/margo/specification/pre-draft/system-design/specification/margo-management-interface/workload-management-api-1.0.0.yaml" + "$SPEC_URL" WFM_SBI_SPEC="$TMP_SPEC" OUTPUT_DIR="./generatedCode" WFM_SBI_PACKAGE_NAME="github.com/margo/sandbox/standard/generatedCode/wfm" @@ -89,4 +90,4 @@ main() { main "$@" #Delete temporary spec file -rm -f "$TMP_SPEC" +# rm -f "$TMP_SPEC" diff --git a/standard/snapshot.spec.yaml b/standard/snapshot.spec.yaml new file mode 100644 index 00000000..28b3debc --- /dev/null +++ b/standard/snapshot.spec.yaml @@ -0,0 +1,698 @@ +openapi: 3.0.3 +info: + title: Margo Workload Management API + version: 1.0.0 + description: + API for managing workloads on Margo-compliant edge devices. + Includes the APIs for exchanging desired state and current state. + Communication is secured using server-side TLS (TLS 1.3 preferred), + and payloads are signed using X.509 certificates. + +servers: + - url: https://wfm.margo.org/ + description: Workload Fleet Manager API + +security: + - PayloadSignature: [] + +paths: + /api/v1/onboarding/certificate: + get: + summary: Download Root CA certificate + security: [] + responses: + '200': + description: Root CA certificate + content: + application/json: + schema: + type: object + properties: + certificate: + type: string + description: Base64-encoded certificate text + /api/v1/onboarding: + post: + requestBody: + content: + application/json: + schema: + type: object + required: [apiVersion, kind, certificate] + properties: + apiVersion: + type: string + description: API version identifier + kind: + type: string + enum: [OnboardingRequest] + description: Resource kind + certificate: + description: Base64-encoded client certificate + type: string + required: true + responses: + '201': + content: + application/json: + schema: + properties: + clientId: + type: string + type: object + description: New client onboarded successfully. + '400': + content: + application/json: + schema: + properties: + error: + example: Invalid certificate + type: string + type: object + description: Invalid certificate format or structure. + '403': + content: + application/json: + schema: + properties: + error: + example: Client rejected + type: string + type: object + description: Client certificate not trusted or client rejected. + security: + - PayloadSignature: [] + summary: Complete onboarding with client certificate + + /api/v1/clients/{clientId}/capabilities: + post: + summary: Report device capabilities + security: + - PayloadSignature: [] + parameters: + - name: clientId + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DeviceCapabilitiesManifest' + responses: + '201': + description: Capabilities reported successfully + '400': + description: Missing or invalid content-digest header. Ensure the SHA256 hash of the payload is included. + '401': + description: Signature verification failed. Ensure you are signing with the correct X.509 private key. + '403': + description: Client certificate is not trusted or has been revoked. + '422': + description: Request body includes a semantic error. + put: + summary: Update device capabilities (Update) + security: + - PayloadSignature: [] + parameters: + - name: clientId + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DeviceCapabilitiesManifest' + responses: + '201': + description: Capabilities reported successfully + '400': + description: Missing or invalid content-digest header. Ensure the SHA256 hash of the payload is included. + '401': + description: Signature verification failed. Ensure you are signing with the correct X.509 private key. + '403': + description: Client certificate is not trusted or has been revoked. + '422': + description: Request body includes a semantic error. + + /api/v1/clients/{clientId}/bundles/{digest}: + get: + summary: Retrieve bundle information for a specific device and digest + security: + - PayloadSignature: [] + parameters: + - name: clientId + in: path + required: true + schema: + type: string + description: Unique identifier of the device-client + - name: digest + in: path + required: true + schema: + type: string + description: Content-addressable digest of the bundle archive. MUST conform to the 'digest' attribute in the Digest Specification and MUST equal the digest computed over the exact sequence of bytes (per Exact Bytes Rule) in the HTTP 200 OK response body. If the server cannot produce content whose digest matches this value it MUST return 404 Not Found. + - in: header + name: If-None-Match + required: false + schema: + type: string + description: Quoted ETag (same as digest) previously returned for this bundle. + responses: + '200': + description: Bundle archive (immutable) + headers: + ETag: + schema: + type: string + description: New ETag for the returned manifest + Cache-Control: + schema: + type: string + description: public, max-age=31536000, immutable + content: + application/vnd.margo.bundle.v1+tar+gzip: + schema: + type: string + format: binary + description: Gzip-compressed tar containing one YAML file per deployment. + '304': + description: Representation not modified + '404': + description: Bundle not found for the given digest + '400': + description: Invalid request. + # TBD + # '500': + # $ref: '#/components/responses/ErrorResponse' + + /api/v1/clients/{clientId}/deployments: + get: + summary: Retrieve the complete desired state for all workloads assigned to a device + security: + - PayloadSignature: [] + parameters: + - name: clientId + in: path + required: true + schema: + type: string + description: The unique identifier of the Edge Compute Device making the request + - name: If-None-Match + in: header + required: false + schema: + type: string + description: > + ETag value of the last successfully synced manifest. The ETag is returned to the client from the /deployments endpoint, it is the digest of the state manifest. + - name: Accept + in: header + required: false + schema: + type: string + description: > + Indicates which manifest formats the client supports. + Supported values: application/vnd.margo.manifest.v1+json. + responses: + '200': + description: Manifest returned in the negotiated format + headers: + Content-Type: + schema: + type: string + description: Format of the returned manifest + ETag: + schema: + type: string + description: New ETag for the returned manifest + content: + application/vnd.margo.manifest.v1+json: + schema: + $ref: '#/components/schemas/UnsignedAppStateManifest' + '304': + description: Not Modified - Manifest has not changed + '406': + description: Not Acceptable - Server cannot generate a response matching the Accept header + + + /api/v1/clients/{clientId}/deployments/{deploymentId}/{digest}: + get: + summary: Retrieve an individual ApplicationDeployment YAML file + security: + - PayloadSignature: [] + description: > + This endpoint is used by the client to fetch the YAML for a single ApplicationDeployment after it has processed a new State Manifest and identified a small number of new or updated deployments. This allows for highly efficient, incremental updates without needing to download the full bundle. + To make individual workload retrievals race-free and cache-friendly, this endpoint is content-addressable: the digest of the expected YAML is part of the URL. This guarantees immutability of the fetched resource and prevents a time-of-check / time-of-use race where a deployment changes between manifest retrieval and content fetch. + parameters: + - name: clientId + in: path + required: true + schema: + type: string + description: Unique identifier of the Edge Compute Device + - name: deploymentId + in: path + required: true + schema: + type: string + description: UUID of the ApplicationDeployment (metadata.annotations.id) + - name: digest + in: path + required: true + schema: + type: string + description: > + Content-addressable digest of the ApplicationDeployment YAML file. MUST conform to the Digest Specification and MUST equal the digest computed over the exact sequence of bytes (per Exact Bytes Rule) in the HTTP 200 OK response body. If the server cannot produce content whose digest matches this value it MUST return 404 Not Found. + - name: If-None-Match + in: header + required: false + schema: + type: string + description: > + Optional ETag for caching. The ETag is returned to the client from the /deployments endpoint, it is the digest of the state manifest. + - name: Accept-Encoding + in: header + required: false + schema: + type: string + description: Indicates supported compression formats (e.g., gzip, br) + responses: + '200': + description: > + The response body is the raw ApplicationDeployment YAML file (Content-Type: application/yaml). The content MUST match the {digest} path segment; the server MUST return 404 if it does not have the exact digest referenced. + headers: + Content-Type: + schema: + type: string + description: application/yaml + ETag: + schema: + type: string + description: > + The ETag is returned to the client from the /deployments endpoint, it is the digest of the state manifest. + Cache-Control: + schema: + type: string + description: public, max-age=31536000, immutable + Vary: + schema: + type: string + description: Accept-Encoding + content: + application/yaml: + schema: + type: string + description: Raw YAML content of the ApplicationDeployment + '404': + description: Deployment not found for the given digest + + /api/v1/clients/{clientId}/deployments/{deploymentId}/status: + post: + summary: Report deployment status + security: + - PayloadSignature: [] + parameters: + - name: clientId + in: path + required: true + schema: + type: string + - name: deploymentId + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DeploymentStatusManifest' + responses: + '200': + description: The deployment status was added, or updated, successfully. + '400': + description: Missing or invalid content-digest header. Ensure the SHA256 hash of the base64-encoded payload is included. + '401': + description: Signature verification failed. Ensure you are signing with the correct X.509 private key. + '403': + description: Client certificate is not trusted or has been revoked. + '422': + description: Request body includes a semantic error. + +components: + securitySchemes: + # TODO: fix this as we are following RFC 9421, instead of a custom signature header field + PayloadSignature: + type: apiKey + in: header + name: X-Payload-Signature + description: > + Base64-encoded payload signature using SHA-256 and device certificate. + Format: public_key;digital_signature + schemas: + ManifestVersion: + type: number + description: > + Monotonically increasing unsigned 64-bit integer in the inclusive range [1, 2^64-1]. + Prevents rollback attacks. The first manifest MUST use 1. + DeploymentBundleRef: + type: object + nullable: true + description: > + Describes a single archive containing all ApplicationDeployment documents. If there are zero deployments (deployments array is empty) the property MUST be present with the value null (it MUST NOT be omitted). + properties: + mediaType: + type: string + description: > + MUST be application/vnd.margo.bundle.v1+tar+gzip; a gzip-compressed tar whose root contains one or more ApplicationDeployment YAML files. If there are zero deployments then bundle MUST be null (an empty archive MUST NOT be served). The archive MUST contain exactly the set of YAML files referenced by deployments. + digest: + type: string + description: > + The digest of the bundle archive. MUST equal the digest computed over the exact sequence of bytes (per Exact Bytes Rule) in the bundle endpoint's HTTP 200 OK response body. + sizeBytes: + type: number + description: > + Unsigned 64-bit advisory estimate of the decoded payload length in bytes for the bundle archive. Provided for bandwidth estimation and update planning. MUST NOT be used for integrity; digest verification remains mandatory. + url: + type: string + description: > + Content-addressable retrieval endpoint of the form /api/v1/clients/{clientId}/bundles/{digest} where {digest} equals bundle.digest. + DeploymentManifestRef: + type: object + description: > + Reference to a deployment manifest with content addressing and integrity verification. + required: + - deploymentId + - digest + - url + properties: + deploymentId: + type: string + description: > + The unique UUID from the ApplicationDeployment's metadata.annotations.id. + digest: + type: string + description: > + The digest of the individual ApplicationDeployment YAML file. MUST equal the digest computed over the exact sequence of bytes (per Exact Bytes Rule) in that deployment endpoint's HTTP 200 OK response body. + sizeBytes: + type: number + description: > + Unsigned 64-bit advisory estimate of the decoded payload length in bytes for the deployment YAML. Provided for planning or progress display. MUST NOT be used for integrity; digest verification remains mandatory. + url: + type: string + description: > + Content-addressable endpoint of the form /api/v1/clients/{clientId}/deployments/{deploymentId}/{digest}. The {digest} MUST equal deployments[].digest; the referenced resource is immutable + UnsignedAppStateManifest: + type: object + required: + - manifestVersion + - bundle + - bundle.mediaType + - bundle.digest + - bundle.url + - deployments + properties: + manifestVersion: + $ref: '#/components/schemas/ManifestVersion' + bundle: + $ref: '#/components/schemas/DeploymentBundleRef' + deployments: + type: array + description: A list of deployment object references for the device. The reference contains some meta info and reference to the url where the deployment is available. + items: + $ref: '#/components/schemas/DeploymentManifestRef' + DeviceCapabilitiesManifest: + type: object + required: [apiVersion, kind, properties] + properties: + apiVersion: + type: string + kind: + type: string + enum: [DeviceCapabilitiesManifest] + properties: + type: object + required: [id, vendor, modelNumber, serialNumber, roles, resources] + properties: + id: + type: string + vendor: + type: string + modelNumber: + type: string + serialNumber: + type: string + roles: + type: array + items: + type: string + enum: [Standalone Cluster, Cluster Leader, Standalone Device] + resources: + type: object + required: [cpu, memory, storage, peripherals, interfaces] + properties: + cpu: + type: object + required: [cores] + properties: + cores: + type: number + architecture: + type: string + enum: [amd64, x86_64, arm64, arm] + memory: + type: string + storage: + type: string + peripherals: + type: array + items: + $ref: '#/components/schemas/DevicePeripheral' + interfaces: + type: array + items: + $ref: '#/components/schemas/DeviceCommunicationInterface' + + ComponentStatus: + type: object + required: [name, state] + properties: + name: + type: string + state: + type: string + enum: [pending, installing, installed, failed, removing, removed] + error: + type: object + properties: + code: + type: string + message: + type: string + + DeploymentStatusManifest: + type: object + required: [apiVersion, kind, deploymentId, status, components] + properties: + apiVersion: + type: string + kind: + type: string + enum: [DeploymentStatusManifest] + deploymentId: + type: string + status: + type: object + required: [state] + properties: + state: + type: string + enum: [pending, installing, installed, failed, removing, removed] + error: + type: object + properties: + code: + type: string + message: + type: string + components: + type: array + items: + $ref: '#/components/schemas/ComponentStatus' + + DevicePeripheral: + type: object + required: [type] + properties: + type: + type: string + enum: [gpu, display, camera, microphone, speaker] + manufacturer: + type: string + model: + type: string + + DeviceCommunicationInterface: + type: object + required: [type] + properties: + type: + type: string + enum: [ethernet, wifi, cellular, bluetooth, usb, canbus, rs232] + + # app deployment struct added here for ease of programming, the code generators will generate the structs + # for the actual app deployment yaml and parsing would be easy to do + appDeploymentManifest: + type: object + description: Application Deployment manifest + required: [apiVersion, kind, metadata, spec] + properties: + apiVersion: + type: string + default: margo.org + description: API version + kind: + type: string + default: ApplicationDeployment + description: Resource kind + metadata: + $ref: '#/components/schemas/appDeploymentMetadata' + spec: + $ref: '#/components/schemas/appDeploymentSpec' + appDeploymentMetadata: + type: object + required: [name] + properties: + id: + type: string + description: Unique identifier for the application deployment + name: + type: string + description: Name of the resource + namespace: + type: string + description: Namespace of the resource + labels: + type: object + additionalProperties: { type: string } + description: Labels for the resource + annotations: + type: object + additionalProperties: { type: string } + description: Annotations for the resource + helmApplicationDeploymentProfileComponent: + type: object + description: Helm Application Deployment Profile Component + required: [name, properties] + properties: + name: + type: string + description: Name of the component + properties: + type: object + required: [repository] + properties: + repository: + type: string + description: Repository of the component + revision: + type: string + description: Revision of the component + timeout: + type: string + description: Timeout for the component + wait: + type: boolean + description: Wait for the component to be ready + composeApplicationDeploymentProfileComponent: + type: object + description: Compose Application Deployment Profile Component + required: [name, properties] + properties: + name: + type: string + description: Name of the component + properties: + type: object + required: [packageLocation] + properties: + packageLocation: + type: string + description: Package location of the component + keyLocation: + type: string + description: Key location of the component + timeout: + type: string + description: Timeout for the component + wait: + type: boolean + description: Wait for the component to be ready + appDeploymentProfile: + type: object + description: Application Deployment Profile + required: [type, components] + properties: + type: + type: string + enum: ["helm.v3", "compose"] + description: Type of deployment profile + components: + type: array + items: + oneOf: + - $ref: '#/components/schemas/helmApplicationDeploymentProfileComponent' + - $ref: '#/components/schemas/composeApplicationDeploymentProfileComponent' + description: Components of the deployment profile + appParameterTarget: + type: object + description: Application Parameter Target + required: [pointer, components] + properties: + pointer: + type: string + description: Pointer to the parameter + components: + type: array + items: + type: string + description: Components of the parameter + appParameterValue: + type: object + description: Application Parameter Value + required: [value, targets] + properties: + value: + # type: object + description: Value of the parameter + additionalProperties: true + x-go-type: interface{} + targets: + type: array + items: + $ref: '#/components/schemas/appParameterTarget' + description: Targets of the parameter + appDeploymentParams: + type: object + description: Application Parameters + additionalProperties: + $ref: '#/components/schemas/appParameterValue' + appDeploymentSpec: + type: object + description: Application Deployment specification + required: [appPackageRef, deploymentProfile] + properties: + deploymentProfile: + $ref: '#/components/schemas/appDeploymentProfile' + description: Deployment profile + parameters: + $ref: '#/components/schemas/appDeploymentParams' + description: Parameters for the deployment \ No newline at end of file