Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,22 @@ jobs:
go-version-file: go.mod
cache: true

- name: Set up Node
uses: actions/setup-node@v4
with:
node-version: "22.12.0"
cache: npm
cache-dependency-path: web/package-lock.json

- name: Run unit tests
run: go test -tags=unit ./...

- name: Build UI, Go packages, and release binaries
run: make all

- name: Verify Node-free fallback build
run: make build-go

- name: Run Ginkgo BDD specs
run: make test-bdd

Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,6 @@ coverage.out
.idea/
.vscode/
.DS_Store
web/node_modules/
web/dist/
web/.vite/
33 changes: 30 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,22 +1,49 @@
GO ?= go
BUF ?= buf
NPM ?= npm
UI_DIR ?= web
BIN_DIR ?= bin
CONTROLLER_GEN ?= controller-gen
COVERAGE_OUT ?= coverage.out

.PHONY: all build test test-bdd lint proto generate tools integration e2e fault perf upgrade coverage coverage-check csi-sanity phase0
.PHONY: all binaries build build-go ui clean-assets test test-bdd lint proto generate tools integration e2e fault perf upgrade coverage coverage-check csi-sanity phase0

all: build
# Release layout: build the embedded UI and all command binaries in bin/.
all: binaries

build:
binaries: build
mkdir -p $(BIN_DIR)
CGO_ENABLED=0 $(GO) build -trimpath -o $(BIN_DIR)/kouja-agent ./cmd/kouja-agent
CGO_ENABLED=0 $(GO) build -trimpath -o $(BIN_DIR)/kouja-csi-controller ./cmd/kouja-csi-controller
CGO_ENABLED=0 $(GO) build -trimpath -o $(BIN_DIR)/kouja-csi-node ./cmd/kouja-csi-node
install -m 0644 deploy/config.example.yaml $(BIN_DIR)/config.example.yaml
install -m 0755 deploy/kouja-init.sh $(BIN_DIR)/kouja-init.sh

build: ui
$(GO) build ./...

# Node-free build. Clean generated assets first so a previous full build does
# not accidentally embed the browser bundle.
build-go: clean-assets
$(GO) build ./...

ui: clean-assets
cd $(UI_DIR) && $(NPM) ci
cd $(UI_DIR) && $(NPM) run build
test -f internal/adminui/assets/index.html

clean-assets:
find internal/adminui/assets -mindepth 1 -maxdepth 1 \
! -name .gitkeep ! -name .gitignore -exec rm -rf {} +

test:
$(GO) test -tags=unit ./...

test-bdd:
$(GO) test -v ./test/bdd

lint:
bash -n deploy/kouja-init.sh deploy/install/install.sh
$(GO) vet ./...
golangci-lint run

Expand Down
114 changes: 107 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ serialized per volume and per volume group.
| Recovery | Startup and periodic reconciliation, drift reporting, and admin actions. |
| Kubernetes | CSI controller and node plugin for provisioning, attach/detach, mount, expansion, and cleanup. |
| Observability | `/healthz`, `/readyz`, Prometheus metrics, structured logs, and audit rows. |
| Local admin UI | Optional loopback-only configuration and status page; no authentication, disabled by default. |

Not supported by v1: thin provisioning, snapshots, clones, replication,
multi-initiator/RWX volumes, automatic high availability, or encryption of data
Expand Down Expand Up @@ -120,7 +121,9 @@ Go 1.26 or newer is required.
git clone https://github.com/voidstarit/kouja.git
cd kouja

make build # compile all packages
make all # build the embedded UI, binaries, bin/kouja-init.sh, and bin/config.example.yaml (Node 22.12)
make build # build the Preact UI, then all Go packages (Node 22.12)
make build-go # Go-only build with the fallback admin page, no Node
make test # unit tests
make test-bdd # Ginkgo/Gomega BDD suite
make coverage-check # enforce >= 80% coverage under ./internal
Expand Down Expand Up @@ -165,6 +168,60 @@ See [Client SDK](docs/clients.md) for complete examples, request-ID
propagation, and regeneration instructions. Error reasons and retryability are
documented in [Client Errors and Retries](docs/client-errors.md).

## Local admin UI

The optional admin UI is embedded in `kouja-agent` and is disabled by default.
Enable it only on a single-administrator host:

```yaml
admin_ui:
enabled: true
address: 127.0.0.1:9444
client_cert_dir: /etc/kouja/tls/clients
```

Restart the agent once, then open `http://127.0.0.1:9444`. The listener is
loopback-only and has **no authentication**. Any local process that can reach
it can rewrite the root-owned configuration and change live VG/class mappings;
this is a root-equivalent trust boundary. Do not proxy or port-forward it, and
do not use it on a host with untrusted local users or host-network containers.
The accepted deployment precondition is a single-administrator host with no
SSRF-capable local service.

The UI shows per-volume state, serving status, sessions, observations, VGs,
iSCSI enablement/IQN/portals, the TLS server certificate’s DNS/IP subject
alternative names, client certificate files, and problem operations. It also
shows live iSCSI traffic from LIO configfs: cumulative read/write bytes,
commands, session RX/TX, sampled read/write/command rates, LUN status, and
login/session/abort/reset error counters. It also shows established client
IP/port connections, matched to the target and initiator where the session
endpoint is unambiguous. These counters are best-effort and never block the
status view. Each live iSCSI target has a **Disable** action: it force-removes
the live LIO target and drops active sessions while retaining the volume and
export definition in SQLite. The volume row then shows **Enable target** to
recreate the target from the stored definition and reinstall the same CHAP
credential.
It can edit the full YAML file with validation and atomic writes, and it can
register/deregister VGs and add/remove class mappings live through the same VG
logic as the mTLS admin API. It never restarts the service. A binding-socket
change sets `restart_required`; restart manually with
`systemctl restart kouja-agent`. Other startup-bound changes produce warnings.

Client certificates are listed from `admin_ui.client_cert_dir` (default:
`<tls cert directory>/clients`). Only regular `.crt`, `.key`, and `.pem` files
directly inside that directory are downloadable; symlinks, the CA key, and the
server private key are never exposed. The public client CA
(`listen.tls.client_ca_file`) is available from
**Configuration → Client certificates → Download client CA** or directly at
`GET /api/ca`. The list includes client private keys, so keep that directory
root-owned at `0700` and treat downloads as sensitive.
`.bak` is the immediate pre-save copy; `.last-good` is the startup
configuration that last reached readiness. Live class changes may need to be
reapplied after a failed-start recovery.

`make build` requires Node 22.12 and builds the embedded UI. `make build-go`
builds without Node and serves the fallback page from `internal/adminui`.

## Install the agent

### 1. Prepare the host
Expand All @@ -191,24 +248,66 @@ Make sure `/etc/tlshd.conf` includes the `.nvme` keyring:
keyrings=.nvme
```

### 2. Install TLS material
### 2. Initialize TLS and configuration

The fastest path on a fresh host is the interactive initializer shipped with
the release binaries:

```sh
sudo ./bin/kouja-init.sh
# or, from a checkout:
sudo ./deploy/kouja-init.sh
```

It requires `openssl` and walks through the configuration step by step, then:

The agent serves TLS 1.3 mTLS. Install the files before running the installer:
- generates a local test CA, server certificate, and `client-ca.crt`;
- generates an admin client certificate and a caller client certificate by
default, with optional controller and node client certificates;
- writes the certificate files under `/etc/kouja/tls` with private keys at
`0600` and the TLS directory at `0700`;
- validates the generated YAML with `kouja-agent --check-config`;
- backs up any existing config and installs `/etc/kouja/config.yaml` at `0600`.

The generated CA is a convenience for a self-contained host. For production,
prefer your operator PKI and choose the “use existing certificate files” path,
or install the files manually:

```text
/etc/kouja/tls/server.crt # server certificate
/etc/kouja/tls/server.key # server private key
/etc/kouja/tls/client-ca.crt # CA used to verify API client certificates
```

The directory should be `0700` root-owned and the files `0600`. Client
certificate URI SANs determine roles:
The server certificate must include the DNS names and IP addresses clients use
to reach the agent. Client certificate URI SANs determine roles:

- `spiffe://kouja/admin/<name>` — administrator
- `spiffe://kouja/caller/<name>` — volume caller
- `spiffe://kouja/controller/<name>` — CSI controller
- `spiffe://kouja/node/<name>` — CSI node

You can validate any configuration, including one written by hand, without
starting the agent:

```sh
bin/kouja-agent --config /etc/kouja/config.yaml --check-config
```

The generated CA certificate is public and can be copied to clients manually:

```sh
sudo cat /etc/kouja/tls/client-ca.crt
# or
scp root@storage-host:/etc/kouja/tls/client-ca.crt .
```

When the admin UI is enabled, use **Configuration → Client certificates →
Download client CA**. The CA private key (`ca.key`) is never served by the UI.

`make all` places the initializer, the release binaries, and
`bin/config.example.yaml` in `bin/`.

### 3. Run the installer

From the repository root:
Expand All @@ -226,8 +325,9 @@ The installer:
- disables the legacy `target.service`;
- enables and starts `kouja-agent.service`.

The installer does **not** create certificates or register volume groups. Edit
the configuration and restart:
The installer itself does **not** create certificates or register volume
groups; `kouja-init.sh` is the separate interactive helper for that. If you
install first, run the initializer afterward and restart:

```sh
sudoedit /etc/kouja/config.yaml
Expand Down
63 changes: 62 additions & 1 deletion cmd/kouja-agent/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@ import (
"os"
"os/signal"
"path/filepath"
"sync"
"sync/atomic"
"syscall"
"time"

"github.com/voidstarit/kouja/internal/adminui"
"github.com/voidstarit/kouja/internal/agent"
"github.com/voidstarit/kouja/internal/backup"
"github.com/voidstarit/kouja/internal/config"
Expand Down Expand Up @@ -40,12 +42,17 @@ func main() {

func run() error {
configPath := flag.String("config", "/etc/kouja/config.yaml", "path to the agent configuration")
checkConfig := flag.Bool("check-config", false, "validate the configuration and exit")
flag.Parse()

cfg, err := config.Load(*configPath)
cfg, configData, err := config.LoadWithData(*configPath)
if err != nil {
return err
}
if *checkConfig {
_, _ = fmt.Fprintln(os.Stdout, "configuration valid")
return nil
}
logger := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
slog.SetDefault(logger)

Expand Down Expand Up @@ -92,6 +99,9 @@ func run() error {
if err := agent.InitializeVGs(context.Background(), store, lvmBackend, cfg.EffectiveVGs()); err != nil {
return err
}
vgClassGate := &sync.Mutex{}
classMap := agent.NewClassMap(cfg.Storage.Classes, *configPath)
vgManager := agent.NewVGManager(store, lvmBackend, classMap)

stepTimeouts := make(map[operations.Action]time.Duration)
for action, seconds := range cfg.Operations.StepTimeoutSecs {
Expand Down Expand Up @@ -124,18 +134,35 @@ func run() error {
Ready: ready.Load,
Admission: admission,
StorageClasses: cfg.Storage.Classes,
Classes: classMap,
})
adminService := agent.NewAdminService(store, lvmBackend,
agent.WithTargetExporters(map[string]target.Exporter{"iscsi": iscsi, "nvme-tcp": nvme}),
agent.WithCredentialManager(credentials),
agent.WithDriftRetrier(manager),
agent.WithStorageClasses(cfg.Storage.Classes),
agent.WithClassRegistry(classMap),
agent.WithVGService(vgManager),
agent.WithVGClassGate(vgClassGate),
)
server, err := agent.NewServer(cfg, volumeService, adminService,
agent.WithReadiness(ready.Load), agent.WithMetrics(metrics))
if err != nil {
return err
}
metrics.Set("kouja_admin_ui_up", 0)
adminServer := startAdminServer(cfg, adminui.Options{
ConfigPath: *configPath, Config: cfg, ConfigData: configData,
Store: store, Capacity: lvmBackend, VGs: vgManager, Classes: classMap,
Ready: ready.Load, HostUUID: hostUUID, AgentVersion: agentVersion,
APIVersion: apiVersion, StartedAt: time.Now(), Logger: logger, Metrics: metrics,
TargetMetrics: iscsi, DisableTarget: manager, EnableTarget: manager,
VGClassGate: vgClassGate,
})
if adminServer != nil {
logger.Warn("local admin UI enabled without authentication; any local process is root-equivalent",
"address", cfg.AdminUI.Address)
}

ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
Expand All @@ -150,18 +177,40 @@ func run() error {
reconciler.Run(ctx)
}()
ready.Store(true)
if adminServer != nil {
if err := adminServer.MarkRunning(); err != nil {
logger.Error("write last-good configuration", "error", err)
}
go func() {
if err := adminServer.Serve(); err != nil {
logger.Error("admin UI server stopped", "error", err)
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
_ = adminServer.Shutdown(shutdownCtx)
cancel()
}
}()
}
logger.Info("agent listening", "address", cfg.Listen.Address, "host_uuid", hostUUID)
serveErr := make(chan error, 1)
go func() { serveErr <- server.Serve() }()
shutdownAdmin := func() {
if adminServer != nil {
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
_ = adminServer.Shutdown(shutdownCtx)
cancel()
}
}

select {
case <-ctx.Done():
shutdownAdmin()
server.GracefulStop()
<-reconcileDone
<-backupDone
return nil
case err := <-serveErr:
ready.Store(false)
shutdownAdmin()
server.Stop()
<-reconcileDone
<-backupDone
Expand All @@ -172,6 +221,18 @@ func run() error {
}
}

func startAdminServer(cfg *config.Config, options adminui.Options) *adminui.Server {
if cfg == nil || !cfg.AdminUI.Enabled {
return nil
}
server, err := adminui.New(options)
if err != nil {
slog.Error("admin UI listener failed; continuing without it", "error", err)
return nil
}
return server
}

func acquireLock(path string) (*os.File, error) {
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) //nolint:gosec // path is the configured data-directory lock
if err != nil {
Expand Down
Loading
Loading