Add multiple LVM storage classes per host

README.md

@@ -95,7 +95,7 @@ supported for the TLS-required NVMe path.
 - `ktls-utils` with `tlshd` active for NVMe/TCP
 - `nvme-cli` for PSK generation and validation
 - configfs mounted at `/sys/kernel/config`
-- an LVM volume group dedicated to kouja; do not point kouja at the root VG
+- one or more LVM volume groups dedicated to kouja; do not point kouja at the root VG
 - a server certificate, server private key, and client CA for mTLS
 
 ### Initiator nodes
@@ -288,6 +288,37 @@ targets:
 The VGs listed under `storage.vgs` are registered on startup. An administrator
 can also register one through the admin API.
 
+### Multiple storage classes on one host
+
+A single agent can serve several LVM backends. Map a stable class name to each
+volume group, for example the NVMe, SSD, and HDD pools on one physical machine:
+
+```yaml
+storage:
+  headroom_percent: 5
+  headroom_min_bytes: 1073741824
+  wipe_timeout_seconds: 1800
+  classes:
+    nvme: kouja-nvme
+    ssd: kouja-ssd
+    hdd: kouja-hdd
+```
+
+The VGs referenced by `classes` are added to the first-start registration set.
+
+```sh
+sudo vgcreate kouja-nvme /dev/disk/by-id/<nvme-device>
+sudo vgcreate kouja-ssd  /dev/disk/by-id/<ssd-device>
+sudo vgcreate kouja-hdd  /dev/disk/by-id/<hdd-device>
+```
+
+Callers select a class in the API with `storage_class`, or in Kubernetes with
+the StorageClass parameter `class`. The agent resolves the class to its VG and
+reports the class on capacity and volume responses. A raw `vg` parameter still
+works for existing callers; when no class map is configured, the VG name is
+used as its own class. The VG remains the durable backend identity; the class
+is the selection and presentation label.
+
 ## API quick start
 
 All API calls require a client certificate trusted by the agent. A caller with
@@ -301,6 +332,25 @@ CURL=(curl --http2 --cacert ca.crt --cert admin.crt --key admin.key)
 "${CURL[@]}" https://storage.example:9443/healthz
 "${CURL[@]}" https://storage.example:9443/v1/info
 "${CURL[@]}" https://storage.example:9443/v1/capacity
+"${CURL[@]}" https://storage.example:9443/v1/storage-classes
+```
+
+`GET /v1/storage-classes` is available to any authenticated client, not only
+administrators. It returns the configured class name, backing VG, availability,
+and capacity:
+
+```json
+{
+  "classes": [
+    {
+      "name": "hdd",
+      "vgName": "kouja-hdd",
+      "vgState": "active",
+      "allocatableBytes": "264241152",
+      "maxVolumeBytes": "264241152"
+    }
+  ]
+}
 ```
 
 Create and export a 1 GiB iSCSI volume:
@@ -345,6 +395,8 @@ POST   /v1/volumes/{id}:rotate-credential
 DELETE /v1/volumes/{id}
 GET    /v1/volumes
 GET    /v1/volumes/{id}/credential
+GET    /v1/capacity
+GET    /v1/storage-classes
 POST   /v1/admin/vgs
 GET    /v1/admin/vgs
 POST   /v1/admin/drift/{id}:resolve

cmd/kouja-agent/main.go

@@ -89,7 +89,7 @@ func run() error {
 	)
 	iscsi := target.NewISCSI(target.NewConfigFS(""))
 	nvme := target.NewNVMe(target.NewConfigFS(""))
-	if err := agent.InitializeVGs(context.Background(), store, lvmBackend, cfg.Storage.VGs); err != nil {
+	if err := agent.InitializeVGs(context.Background(), store, lvmBackend, cfg.EffectiveVGs()); err != nil {
 		return err
 	}
 
@@ -112,22 +112,24 @@ func run() error {
 		cfg.Admission.CapacityRequestsPerSec,
 	)
 	volumeService := agent.NewVolumeService(store, lvmBackend, manager, credentials, agent.ServiceOptions{
-		HostUUID:     hostUUID,
-		AgentVersion: agentVersion,
-		APIVersion:   apiVersion,
-		ISCSIEnabled: cfg.Targets.ISCSI.Enabled,
-		ISCSIPrefix:  cfg.Targets.ISCSI.IQNPrefix,
-		ISCSIPortals: cfg.Targets.ISCSI.Portals,
-		NVMeEnabled:  cfg.Targets.NVMeTCP.Enabled,
-		NVMePrefix:   cfg.Targets.NVMeTCP.NQNPrefix,
-		NVMePortals:  cfg.Targets.NVMeTCP.Portals,
-		Ready:        ready.Load,
-		Admission:    admission,
+		HostUUID:       hostUUID,
+		AgentVersion:   agentVersion,
+		APIVersion:     apiVersion,
+		ISCSIEnabled:   cfg.Targets.ISCSI.Enabled,
+		ISCSIPrefix:    cfg.Targets.ISCSI.IQNPrefix,
+		ISCSIPortals:   cfg.Targets.ISCSI.Portals,
+		NVMeEnabled:    cfg.Targets.NVMeTCP.Enabled,
+		NVMePrefix:     cfg.Targets.NVMeTCP.NQNPrefix,
+		NVMePortals:    cfg.Targets.NVMeTCP.Portals,
+		Ready:          ready.Load,
+		Admission:      admission,
+		StorageClasses: cfg.Storage.Classes,
 	})
 	adminService := agent.NewAdminService(store, lvmBackend,
 		agent.WithTargetExporters(map[string]target.Exporter{"iscsi": iscsi, "nvme-tcp": nvme}),
 		agent.WithCredentialManager(credentials),
 		agent.WithDriftRetrier(manager),
+		agent.WithStorageClasses(cfg.Storage.Classes),
 	)
 	server, err := agent.NewServer(cfg, volumeService, adminService,
 		agent.WithReadiness(ready.Load), agent.WithMetrics(metrics))

deploy/install/install.sh

@@ -37,6 +37,7 @@ storage:
   headroom_min_bytes: 1073741824
   wipe_timeout_seconds: 1800
   vgs: []
+  classes: {}
 targets:
   iscsi:
     enabled: true

deploy/kubernetes/csi/kouja-csi.yaml

@@ -105,6 +105,8 @@ spec:
   volumeLifecycleModes:
     - Persistent
 ---
+# The parameters can select a raw LVM VG (`vg`) or a host-configured storage
+# class (`class`, for example nvme/ssd/hdd). Use one or the other.
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:

docs/csi.md

@@ -129,7 +129,8 @@ Parameters:
 
 | Parameter | Required | Meaning |
 |---|---|---|
-| `vg` | yes | Registered LVM volume group on the selected kouja host. |
+| `class` | one of `class`/`vg` | Storage class configured on the agent, for example `nvme`, `ssd`, or `hdd`. The agent maps it to a registered LVM volume group. |
+| `vg` | one of `class`/`vg` | Registered LVM volume group on the selected kouja host. Use this for existing deployments or when no class mapping exists. |
 | `protocol` | no | `iscsi` (default) or `nvme-tcp`. The NVMe path requires TLS 1.3 PSK support on the target and `tlshd` on the node. |
 | `initiator` | yes for attach | IQN for iSCSI or NQN for NVMe-oF/TCP. It must match the node plugin's initiator. |
 | `csi.storage.k8s.io/fstype` | no | `ext4` (default in the example). `xfs` is accepted by the node formatter only when `mkfs.xfs` is present in the node image. |
@@ -140,6 +141,38 @@ The initiator value is passed through the PV's volume context. A production
 deployment should derive it from the CSINode annotation described in the
 design rather than storing a single cluster-wide IQN.
 
+To expose multiple backends from the same host, map classes on the agent and
+reference them from separate Kubernetes StorageClasses:
+
+```yaml
+# agent /etc/kouja/config.yaml
+storage:
+  classes:
+    nvme: kouja-nvme
+    ssd: kouja-ssd
+    hdd: kouja-hdd
+---
+# Kubernetes
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: kouja-hdd
+provisioner: kouja.csi.voidstarit.com
+parameters:
+  class: hdd
+  protocol: iscsi
+  initiator: iqn.2026-11.io.kouja:kind-node
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+volumeBindingMode: Immediate
+```
+
+The controller filters hosts by the capacity response's storage class, so a
+PVC is only placed on an agent that advertises that class. A class that is not
+mapped on the agent falls back to the raw VG name for backward compatibility.
+Clients can inspect the available classes with `GET /v1/storage-classes`; the
+response includes the backing VG, availability, and current capacity.
+
 ## Controller flags
 
 ```text
@@ -155,7 +188,7 @@ design rather than storing a single cluster-wide IQN.
 
 `--hosts` may contain one or more agent endpoints. The controller calls
 `GetInfo` and `GetCapacity` to select a ready host that supports the requested
-protocol and has enough allocatable space in the requested VG.
+protocol and has enough allocatable space in the requested class or VG.
 
 ## Deployment notes
 

docs/designs/kouja.md

@@ -208,6 +208,7 @@ Key request fields:
 * `ResizeVolumeRequest`: `size_bytes`, `allow_shrink` (default false).
 * `DeleteVolumeRequest`: `force` (default false).
 * `GetCapacityResponse`: one entry per registered VG with `vg_name`, `vg_uuid`, `vg_state`, `total_bytes`, `free_bytes`, `allocatable_bytes`, `extent_size_bytes`, `max_volume_bytes`, and `observed_at`.
+* `ListStorageClassesResponse`: one entry per configured class with `name`, backing VG, state, and the same capacity fields.
 * `ImportVolumeRequest`: `volume_id`, `owner`, `vg`, `lv_uuid` (LVM UUID) or `lv_name`, `size_bytes`, `export_spec`, `target_name`, `lun` or `nsid`, `credential` (oneof CHAP or TLS-PSK), `allow_active_sessions`.
 * `GrantCredentialRequest`: `volume_id`, `node_id`, `node_caller_id`, `expires_at`.
 
@@ -260,7 +261,7 @@ HTTP paths:
 * `POST /v1/volumes/{volume_id}:export`, `POST /v1/volumes/{volume_id}:unexport`
 * `POST /v1/volumes/{volume_id}:resize`, `DELETE /v1/volumes/{volume_id}`
 * `GET /v1/volumes/{volume_id}/credential`, `POST /v1/volumes/{volume_id}:rotate-credential`
-* `GET /v1/capacity`, `GET /v1/info`
+* `GET /v1/capacity`, `GET /v1/storage-classes`, `GET /v1/info`
 * `POST /v1/admin/vgs`, `DELETE /v1/admin/vgs/{vg_name}`, `GET /v1/admin/vgs`
 * `POST /v1/admin/import`, `POST /v1/admin/drift/{volume_id}:resolve`, `GET /v1/admin/orphans`
 * `POST /v1/admin/ownership:transfer`, `DELETE /v1/admin/volumes/{volume_id}`
@@ -644,13 +645,13 @@ Credentials are encrypted at rest with AES-256-GCM. Each row stores a unique non
 * `vg_state` and `observed_at`.
 * The agent queries LVM on each request and does not persist a capacity cache.
 
-`CreateVolume` requires `vg`. The agent serializes the capacity check and the LVM allocation with a per-VG lock. The per-VG lock prevents racing kouja allocations. `ResourceExhausted` is returned when the actual LVM allocation fails or the request exceeds the current allocatable size. The CSI driver tries the next eligible host up to three times.
+`CreateVolume` requires either a configured `storage_class` or a raw `vg`. The agent resolves a class to its VG and serializes the capacity check and the LVM allocation with a per-VG lock. The per-VG lock prevents racing kouja allocations. `ResourceExhausted` is returned when the actual LVM allocation fails or the request exceeds the current allocatable size. The CSI driver tries the next eligible host up to three times.
 
 The CSI driver selects a host by:
 
 1. Resolving the StorageClass SRV record.
 2. Calling `GetInfo` and `GetCapacity` on every endpoint in parallel, with a 2-second timeout.
-3. Filtering to hosts that are ready, support the requested protocol, have the requested VG, have `vg_state=active`, and have `allocatable_bytes >= requested size` and `max_volume_bytes >= requested size`.
+3. Filtering to hosts that are ready, support the requested protocol, have the requested class or VG, have `vg_state=active`, and have `allocatable_bytes >= requested size` and `max_volume_bytes >= requested size`.
 4. Choosing the host with the largest `allocatable_bytes`; ties break by host UUID.
 5. Calling `CreateVolume`; on `ResourceExhausted`, removing that host and trying the next.
 6. Caching the inventory for at most 10 seconds; the cache is invalidated on an unreachable host or a `ResourceExhausted` response.
@@ -670,12 +671,12 @@ The driver name is `kouja.csi.voidstarit.com`. It advertises these capabilities:
 
 `CreateVolume`:
 
-1. Parse the StorageClass parameters: `srvRecord`, `vg` (required), `protocol`, optional `wipe`.
+1. Parse the StorageClass parameters: `srvRecord`, `class` or `vg` (one required), `protocol`, optional `wipe`.
 2. Derive a stable placement key from the CSI request name and StorageClass.
 3. Look up a `KoujaVolumePlacement` custom resource with `resourceVersion` compare-and-swap; the external-provisioner supplies leader election. If it exists and is `Bound`, use that host. If it is `Pending`, query the recorded host; if the volume exists, bind it; if not, resume the create on that host. If it is `Failed`, follow the recorded retry path. If the host is unreachable, return `Unavailable` and do not create on another host.
-4. If no record exists, resolve the SRV record, select a host by capacity, and write a `Pending` placement record with the selected host and an idempotency key before calling `CreateVolume` with `export=false`, the size, the VG, and `wipe`.
+4. If no record exists, resolve the SRV record, select a host by capacity, and write a `Pending` placement record with the selected host and an idempotency key before calling `CreateVolume` with `export=false`, the size, the class or VG, and `wipe`.
 5. On success, update the record to `Bound` with the returned `host_uuid` and `resource_uuid`. On failure, delete the pending record or mark it `Failed` with the error; a retry resolves the record before creating.
-6. Return `v1:<host-uuid>:<resource-uuid>` as the volume handle and the actual capacity. The volume context includes the host UUID, resource UUID, protocol, VG, control endpoint, and portal.
+6. Return `v1:<host-uuid>:<resource-uuid>` as the volume handle and the actual capacity. The volume context includes the host UUID, resource UUID, protocol, class, VG, control endpoint, and portal.
 
 If the pending record cannot be written, the driver does not call `CreateVolume`. If the response is lost, the pending record identifies the host and idempotency key, and a retry queries that host before creating anywhere else. The driver never returns a handle that is not durably recorded.
 
@@ -705,7 +706,7 @@ If the pending record cannot be written, the driver does not call `CreateVolume`
 
 `DeleteVolume` calls `DeleteVolume` with `force=false`. `SESSION_ACTIVE` is returned as a retryable error. An administrator can force-delete through the API if the consumer is gone.
 
-`GetCapacity` sums `allocatable_bytes` across ready hosts that match the StorageClass SRV record, protocol, and required VG. It returns the sum as `available_capacity` and the maximum of `max_volume_bytes` as `maximum_volume_size`. It does not reserve capacity.
+`GetCapacity` sums `allocatable_bytes` across ready hosts that match the StorageClass SRV record, protocol, and required class or VG. It returns the sum as `available_capacity` and the maximum of `max_volume_bytes` as `maximum_volume_size`. It does not reserve capacity.
 
 Node topology is empty. The first version assumes every Kubernetes node can reach every portal returned by the SRV record. An operator who needs network zones scopes the SRV record per zone and creates one StorageClass per zone.
 
@@ -809,6 +810,7 @@ storage:
   headroom_min_bytes: 1073741824
   wipe_timeout_seconds: 1800  # base; deadline is max(base, 1 second per GiB)
   vgs: []  # names to seed on first start; the admin API is authoritative afterward
+  classes: {}  # optional class -> VG mappings, e.g. nvme: vg-nvme
 targets:
   iscsi:
     enabled: true
@@ -1478,6 +1480,9 @@ message RotateCredentialResponse { string credential = 1; string operation_id =
 message Capacity { string vg_name = 1; string vg_uuid = 2; string vg_state = 3; int64 total_bytes = 4; int64 free_bytes = 5; int64 allocatable_bytes = 6; int64 extent_size_bytes = 7; int64 max_volume_bytes = 8; google.protobuf.Timestamp observed_at = 9; }
 message GetCapacityRequest {}
 message GetCapacityResponse { repeated Capacity vgs = 1; }
+message ListStorageClassesRequest {}
+message StorageClass { string name = 1; string vg_name = 2; string vg_uuid = 3; string vg_state = 4; int64 total_bytes = 5; int64 free_bytes = 6; int64 allocatable_bytes = 7; int64 extent_size_bytes = 8; int64 max_volume_bytes = 9; google.protobuf.Timestamp observed_at = 10; }
+message ListStorageClassesResponse { repeated StorageClass classes = 1; }
 message GetInfoRequest {}
 message GetInfoResponse { string host_uuid = 1; string agent_version = 2; string api_version = 3; repeated Protocol protocols = 4; repeated string auth_modes = 5; bool ready = 6; }
 
@@ -1515,6 +1520,7 @@ service VolumeService {
   rpc GetCredential(GetCredentialRequest) returns (GetCredentialResponse) { option (google.api.http) = { get: "/v1/volumes/{volume_id}/credential" }; }
   rpc RotateCredential(RotateCredentialRequest) returns (RotateCredentialResponse) { option (google.api.http) = { post: "/v1/volumes/{volume_id}:rotate-credential" body: "*" }; }
   rpc GetCapacity(GetCapacityRequest) returns (GetCapacityResponse) { option (google.api.http) = { get: "/v1/capacity" }; }
+  rpc ListStorageClasses(ListStorageClassesRequest) returns (ListStorageClassesResponse) { option (google.api.http) = { get: "/v1/storage-classes" }; }
   rpc GetInfo(GetInfoRequest) returns (GetInfoResponse) { option (google.api.http) = { get: "/v1/info" }; }
 }
 

docs/e2e.md

@@ -27,6 +27,32 @@ The agent used SQLite state, an mTLS gRPC/HTTP listener, a loop-backed LVM VG, t
 
 The final two-VM test used the agent binary built after the operation-journal tombstone-retirement fix, so a stale delete operation from a previous volume cannot replay against a new volume.
 
+## Multiple storage classes on one host
+
+A separate two-VM QEMU/KVM run validated the class-aware backend selection:
+
+* The storage host had two loop-backed LVM VGs, `kouja-nvme` and `kouja-hdd`,
+  configured only through `storage.classes`:
+  `nvme: kouja-nvme` and `hdd: kouja-hdd`.
+* `GET /v1/capacity` returned both VGs with `storageClass: nvme` and
+  `storageClass: hdd`.
+* `GET /v1/storage-classes`, called with a non-admin caller certificate,
+  returned both classes with their backing VGs and active capacity.
+* Creating a volume with `storageClass: hdd` and no `vg` produced an LV in
+  `kouja-hdd`, and the API response reported `storageClass: hdd`.
+* The consumer logged into the iSCSI target with CHAP, wrote 8 MiB, read it
+  back, and matched the MD5 (`class-hdd-iscsi-write-read-ok`).
+* Deleting while the session was active returned `SESSION_ACTIVE`; after logout
+  the delete succeeded and left no LV, backstore, or target.
+* Creating a volume with `storageClass: nvme` landed in `kouja-nvme`.
+  A request with `storageClass: nvme` and `vg: kouja-hdd` was rejected with
+  `InvalidArgument`.
+* A CSI controller pointed at the agent selected the `hdd` class for
+  `GetCapacity`, created and published a volume, returned
+  `storage_class: hdd` in the volume context, and completed an 8 MiB write/read
+  through the publish context before unpublish and delete
+  (`csi-class-publish-write-read-ok`).
+
 ## NVMe/TCP TLS
 
 `TestNVMeTLSAdapterLifecycle` passed on the storage VM: