From 3b60ad73a486757d644c08465d8e70d75d57a4b2 Mon Sep 17 00:00:00 2001 From: suin Date: Thu, 18 Jun 2026 19:12:25 +0900 Subject: [PATCH 1/6] Add k3d registry support --- README.md | 116 ++++++- devbox.json | 4 +- devbox.lock | 96 ++++++ docs/design/k8s-distribution-and-registry.md | 338 +++++++++++++++++++ docs/design/minimum-cli-runbook.md | 2 +- docs/design/minimum-cli.md | 4 +- docs/design/project-up.md | 4 +- docs/design/roadmap.md | 2 +- docs/design/snapshot.md | 2 +- examples/k3d/spind.yaml | 11 + examples/tilt/Dockerfile | 3 + examples/tilt/Tiltfile | 5 + examples/tilt/app/index.html | 15 + examples/tilt/k8s.yaml | 35 ++ examples/tilt/spind.yaml | 7 + internal/spind/cli/cli_test.go | 6 +- internal/spind/cli/output/snapshot.go | 85 ++--- internal/spind/cli/output/vm_detail.go | 33 ++ internal/spind/cli/output/vm_json.go | 46 +++ internal/spind/cli/output/vm_start.go | 17 + internal/spind/kind/ready_snapshot.go | 68 +++- internal/spind/kind/ready_snapshot_test.go | 29 ++ internal/spind/kind/registry.go | 71 ++++ internal/spind/kind/registry_test.go | 35 ++ internal/spind/kind/types.go | 27 +- internal/spind/snapshot/create/command.go | 6 +- internal/spind/snapshot/create/options.go | 2 +- internal/spind/snapshot/create/run.go | 2 +- internal/spind/snapshot/store.go | 1 + internal/spind/snapshot/types.go | 40 +-- internal/spind/up/run.go | 18 +- internal/spind/up/run_test.go | 4 +- internal/spind/vm/start/kind_ready.go | 152 ++++++++- internal/spind/vm/start/manager.go | 13 +- internal/spind/vm/start/run.go | 1 + internal/spind/vm/start/snapshot.go | 5 + internal/spind/vm/status/run.go | 12 +- internal/spind/vmstore/types.go | 19 ++ spind.yaml | 2 +- tests/e2e/helpers.ts | 9 +- tests/e2e/k3d-registry.test.ts | 74 ++++ tests/e2e/kind-ready.test.ts | 2 +- tests/e2e/up.test.ts | 2 +- 43 files changed, 1302 insertions(+), 123 deletions(-) create mode 100644 docs/design/k8s-distribution-and-registry.md create mode 100644 examples/k3d/spind.yaml create mode 100644 examples/tilt/Dockerfile create mode 100644 examples/tilt/Tiltfile create mode 100644 examples/tilt/app/index.html create mode 100644 examples/tilt/k8s.yaml create mode 100644 examples/tilt/spind.yaml create mode 100644 internal/spind/kind/registry.go create mode 100644 internal/spind/kind/registry_test.go create mode 100644 tests/e2e/k3d-registry.test.ts diff --git a/README.md b/README.md index 2bf4601..3b03561 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ spind is a fast local development environment for people who build web applications that use the Kubernetes API, and for people who build Kubernetes Operators. -It supports kind-based Kubernetes environments and makes clean environments fast to create, reset, and start. +It supports kind-based and k3d-based Kubernetes environments and makes clean environments fast to create, reset, and start. ## Why spind? @@ -29,13 +29,13 @@ On macOS, spind starts a small Linux VM with Virtualization.framework. On Linux, The Linux VM is not a special Kubernetes system. It is a small Docker machine. You can use your normal Docker client, kind, kubectl, and helm. -spind sets up the runtime VM once, then saves that ready state as a snapshot. After that, spind restores from the snapshot. This lets you start an environment where the kind cluster and Helm charts are already prepared. +spind sets up the runtime VM once, then saves that ready state as a snapshot. After that, spind restores from the snapshot. This lets you start an environment where the Kubernetes cluster and Helm charts are already prepared. ## Use cases - Web applications that use the Kubernetes API - Kubernetes Operator development -- Local Kubernetes development with kind +- Local Kubernetes development with kind or k3d - Clean environments for end-to-end tests and integration tests - Faster Kubernetes environment setup in CI/CD @@ -86,7 +86,7 @@ Put `spind.yaml` in your project root. ```yaml name: sample image: docker -kind: true +k8s: kind setup: - "kind create cluster" - "kubectl --context kind-kind wait node --all --for=condition=Ready --timeout=180s" @@ -100,6 +100,114 @@ spind up On the first run, spind creates a VM, runs the commands in `setup`, and saves the ready state as a snapshot. Later runs restore from that snapshot. +## Guide for kind users + +Use `k8s: kind` when your project uses kind. + +```yaml +name: sample +image: docker +k8s: kind +setup: + - "kind create cluster" + - "kubectl --context kind-kind wait node --all --for=condition=Ready --timeout=180s" +``` + +Run `spind up`. + +```sh +spind up +``` + +After the environment starts, spind prints shell exports for the restored VM. + +```sh +export DOCKER_HOST='unix:///.../.spind/vms/sample/docker.sock' +export KUBECONFIG='/.../.spind/vms/sample/kubeconfig' +``` + +Use those values in the current shell, then use your usual tools. + +```sh +kubectl get nodes +docker ps +``` + +spind does not update your global kubeconfig by default. The kubeconfig path printed by `spind up` is the VM-specific kubeconfig. + +## Guide for k3d users + +Use `k8s: k3d` when your project uses k3d. + +```yaml +name: k3d-sample +image: docker +k8s: k3d +setup: + - "k3d cluster create k3d-sample --registry-create k3d-sample-registry --kubeconfig-update-default=false" + - 'k3d kubeconfig get k3d-sample > "$SPIND_KUBECONFIG"' + - "kubectl --context k3d-k3d-sample wait node --all --for=condition=Ready --timeout=180s" +``` + +`SPIND_KUBECONFIG` is a path that spind sets for setup commands. Write the k3d kubeconfig there so spind can save it in the snapshot and rewrite it after restore. + +When the k3d cluster has a local registry, spind detects it and prints `REGISTRY`. + +```sh +export DOCKER_HOST='unix:///.../.spind/vms/k3d-sample/docker.sock' +export KUBECONFIG='/.../.spind/vms/k3d-sample/kubeconfig' +export REGISTRY='localhost:61702' +``` + +Use `REGISTRY` with the `DOCKER_HOST` value printed by spind. + +```sh +docker build -t "$REGISTRY/my-app:dev" . +docker push "$REGISTRY/my-app:dev" +``` + +For k3d, `k3d registry list` may show a port that is different from the `REGISTRY` value printed by spind. In a spind environment, use the `REGISTRY` value printed by `spind up` or `spind vm start`. + +See `examples/k3d/spind.yaml` for a k3d sample with cert-manager. + +## Guide for Tilt users + +Tilt works with the same `DOCKER_HOST` and `KUBECONFIG` values that spind prints. + +Start the spind environment first. + +```sh +spind up +``` + +Then export the values printed by spind in your shell. + +```sh +export DOCKER_HOST='unix:///.../.spind/vms/tilt-sample/docker.sock' +export KUBECONFIG='/.../.spind/vms/tilt-sample/kubeconfig' +export REGISTRY='localhost:61702' +``` + +Tilt can auto-detect a k3d local registry through the `kube-public/local-registry-hosting` ConfigMap. For that flow, do not set `default_registry()` in the Tiltfile. + +A minimal Tiltfile looks like this. + +```python +docker_build("tilt-sample", ".") + +k8s_yaml("k8s.yaml") + +k8s_resource("tilt-sample", port_forwards=8080) +``` + +Run Tilt after the spind environment is ready. + +```sh +tilt up +``` + +See `examples/tilt/` for a small Tilt sample. + ## Development Use these commands when working on this repository. diff --git a/devbox.json b/devbox.json index 44aeb50..8a89e98 100644 --- a/devbox.json +++ b/devbox.json @@ -16,7 +16,9 @@ "terraform": "latest", "kind": "latest", "kubectl": "latest", - "kubernetes-helm": "latest" + "kubernetes-helm": "latest", + "k3d": "latest", + "tilt": "latest" }, "shell": { "init_hook": ["export PATH=\"$PWD/bin:$PATH\"", "echo 'spind devbox ready'"] diff --git a/devbox.lock b/devbox.lock index b7a0265..c0ef74b 100644 --- a/devbox.lock +++ b/devbox.lock @@ -441,6 +441,54 @@ } } }, + "k3d@latest": { + "last_modified": "2026-06-11T01:27:03Z", + "resolved": "github:NixOS/nixpkgs/b503dde361500433ca25a32e8f4d218bf58fb659#k3d", + "source": "devbox-search", + "version": "5.9.0", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/g91b1mz18bq5q567naa54kvka1ci6466-k3d-5.9.0", + "default": true + } + ], + "store_path": "/nix/store/g91b1mz18bq5q567naa54kvka1ci6466-k3d-5.9.0" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/4ly383iik2mca7561g5y4sinvmdmkbxl-k3d-5.9.0", + "default": true + } + ], + "store_path": "/nix/store/4ly383iik2mca7561g5y4sinvmdmkbxl-k3d-5.9.0" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/lyx64rqz0hc1b6rcyxmsw1cz9qh1lbcf-k3d-5.9.0", + "default": true + } + ], + "store_path": "/nix/store/lyx64rqz0hc1b6rcyxmsw1cz9qh1lbcf-k3d-5.9.0" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/v69k5siw9w41lnbz34dk9nwkn5f3dbi2-k3d-5.9.0", + "default": true + } + ], + "store_path": "/nix/store/v69k5siw9w41lnbz34dk9nwkn5f3dbi2-k3d-5.9.0" + } + } + }, "kind@latest": { "last_modified": "2026-05-21T08:15:18Z", "resolved": "github:NixOS/nixpkgs/4a29d733e8a7d5b824c3d8c958a946a9867b3eb2#kind", @@ -752,6 +800,54 @@ "store_path": "/nix/store/f8yvzw63qd1mc44d8bx6hq7smr7zvxnr-terraform-1.15.4" } } + }, + "tilt@latest": { + "last_modified": "2026-05-21T08:15:18Z", + "resolved": "github:NixOS/nixpkgs/4a29d733e8a7d5b824c3d8c958a946a9867b3eb2#tilt", + "source": "devbox-search", + "version": "0.37.3", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/srcj3zqmc8k83pmhcvzimmwyanx9bv05-tilt-0.37.3", + "default": true + } + ], + "store_path": "/nix/store/srcj3zqmc8k83pmhcvzimmwyanx9bv05-tilt-0.37.3" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/l5lcv1pma6a6sv9a11m1af875hx1l3vd-tilt-0.37.3", + "default": true + } + ], + "store_path": "/nix/store/l5lcv1pma6a6sv9a11m1af875hx1l3vd-tilt-0.37.3" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/57n2dz3pq26g0410f4zpc6c62d0gy2j9-tilt-0.37.3", + "default": true + } + ], + "store_path": "/nix/store/57n2dz3pq26g0410f4zpc6c62d0gy2j9-tilt-0.37.3" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/a0mypx8rw6gpd0fc9jjbws7vqkj5113r-tilt-0.37.3", + "default": true + } + ], + "store_path": "/nix/store/a0mypx8rw6gpd0fc9jjbws7vqkj5113r-tilt-0.37.3" + } + } } } } diff --git a/docs/design/k8s-distribution-and-registry.md b/docs/design/k8s-distribution-and-registry.md new file mode 100644 index 0000000..642eb27 --- /dev/null +++ b/docs/design/k8s-distribution-and-registry.md @@ -0,0 +1,338 @@ +# Kubernetes Distribution and k3d Registry + +## 目的 + +spindは、kindだけでなくk3dも選べるKubernetes-ready snapshotを提供する。 + +この文書は、既存のkind-ready snapshotを `k8s: kind|k3d` に広げ、spindが提示する `DOCKER_HOST` 環境からk3d local registryを使えるようにする実装方針を定める。 + +## 決定 + +- `spind.yaml` は `k8s: kind|k3d` を使う。 +- `spind snapshot create` は `--k8s=kind|k3d` を使う。 +- 既存の `kind: true` と `--kind` との互換性は持たない。 +- k3d registry対応は今回のk3d対応スコープに含める。 +- Docker/k3d containerの `HostPort` は書き換えない。 +- spindはregistry target portと必要なrelay portを管理する。 +- spindは `kube-public/local-registry-hosting` の `localRegistryHosting.v1.host` を、`DOCKER_HOST` が指すVM Docker daemonから使えるregistry URLへ更新する。 +- `localRegistryHosting.v1.hostFromContainerRuntime` と `hostFromClusterNetwork` はk3dの値を維持する。 +- `k3d registry list` が表示するportと、hostから使えるregistry portが一致しない場合がある。これは制限事項として扱う。 +- `spind up` と `spind vm start` は、registryが利用できる場合に `REGISTRY=localhost:` をshell export候補として表示する。このURLは、同時にexportされる `DOCKER_HOST` のDocker daemonから見えるregistry URLである。 +- `REGISTRY_FROM_CLUSTER` は表示しない。 + +## 設定 + +`spind.yaml` のKubernetes distribution指定: + +```yaml +name: sample +image: docker +k8s: k3d +setup: + - "k3d cluster create sample --registry-create sample-registry --kubeconfig-update-default=false" + - "k3d kubeconfig get sample > \"$KUBECONFIG\"" + - "kubectl wait node --all --for=condition=Ready --timeout=180s" +``` + +kindの場合: + +```yaml +name: sample +image: docker +k8s: kind +setup: + - "kind create cluster --kubeconfig \"$KUBECONFIG\"" + - "kubectl wait node --all --for=condition=Ready --timeout=180s" +``` + +`k8s` を省略した場合はKubernetes-ready snapshotを作らない。 + +`k8s` に `kind` または `k3d` 以外の値が指定された場合は設定エラーにする。 + +## CLI + +snapshot作成: + +```sh +spind snapshot create dev-ready --vm dev-base --k8s=kind +spind snapshot create dev-ready --vm dev-base --k8s=k3d +``` + +`--kubeconfig` と `--context` は既存と同じ意味を持つ。 + +```sh +spind snapshot create dev-ready \ + --vm dev-base \ + --k8s=k3d \ + --kubeconfig /path/to/kubeconfig \ + --context k3d-dev +``` + +`spind snapshot create` の `--k8s` 省略時は、通常snapshotを作る。 + +## 内部モデル + +`CreateOptions.Kind bool` はdistributionを表す値に置き換える。 + +```go +type Distribution string + +const ( + K8sNone Distribution = "" + K8sKind Distribution = "kind" + K8sK3d Distribution = "k3d" +) + +type CreateOptions struct { + K8s Distribution + KubeconfigPath string + Context string +} +``` + +既存の `kind` packageとmetadata名は、実装時に `k8s` へ改名する。 + +snapshot artifactはdistributionを保存する。 + +```json +{ + "k8sReady": true, + "distribution": "k3d", + "sourceKubeconfigPath": "...", + "sourceContext": "k3d-sample", + "sourceCluster": "k3d-sample", + "sourceUser": "admin@k3d-sample", + "sourceServer": "https://0.0.0.0:59569", + "apiServerTargetPort": 59569, + "nodes": [] +} +``` + +`kindReady` は新しいartifactでは使わない。 + +## Kubernetes API検出 + +spindは、distributionごとにKubernetes API serverのVM内target portを検出する。 + +kind: + +- `io.x-k8s.kind.role=control-plane` labelを持つcontainerを探す。 +- またはcontainer名が `-control-plane` で終わるcontainerを探す。 +- そのcontainerの `6443/tcp` published portをAPI server target portとする。 + +k3d: + +- `k3d.role=loadbalancer` labelを持つcontainerを探す。 +- そのcontainerの `6443/tcp` published portをAPI server target portとする。 +- k3dのserver containerではなくload balancer containerを見る。 + +kubeconfig server hostの扱い: + +- kindはloopback hostを要求する。 +- k3dは `0.0.0.0`、`127.0.0.1`、`localhost`、IPv6 loopbackを受け入れる。 +- k3dのkubeconfig templateは保存してよいが、restore後に生成するVM別kubeconfigではspindのAPI server relay URLへ必ず差し替える。 + +## Restore + +Kubernetes-ready snapshotから作成されたVMの `spind vm start` は次を行う。 + +1. Docker API endpointをreadyにする。 +2. Kubernetes API server用のhost空きportを割り当てる。 +3. Kubernetes API serverへのhost loopback relayを起動する。 +4. snapshot内のkubeconfig templateからVM別kubeconfigを生成する。 +5. kubeconfig内のserverを `https://127.0.0.1:` に書き換える。 +6. cluster、user、context名を `spind-` に書き換える。 +7. `kubectl --kubeconfig get nodes` 相当のready checkを行う。 +8. k3d registryがある場合はregistry relayを起動する。 +9. `local-registry-hosting` ConfigMapの `host` をVM Docker daemonから見えるregistry URLへ更新する。 + +registry relayとConfigMap更新は、Kubernetes API ready check成功後に行う。 + +## k3d Registry検出 + +k3d registry containerは、VM内Docker APIから検出する。 + +対象container: + +- `k3d.role=registry` labelを持つ。 +- 対象k3d clusterに接続されているregistryを優先する。 +- cluster専用registryと共有registryの両方を扱えるようにする。 + +registry target port: + +- containerの `5000/tcp` published portをVM内target portとする。 +- 実際にlistenしているportを知るため、Docker APIの `NetworkSettings.Ports` を優先する。 +- `NetworkSettings.Ports` が取れない場合は `HostConfig.PortBindings` を見る。 +- label `k3s.registry.port.external` は補助情報として扱う。 + +`k3d registry list` は通常 `HostConfig.PortBindings` 由来のportを表示する。spindは、`DOCKER_HOST` が指すDocker daemonから使えるregistry URLのsource of truthとして、spindの出力と `localRegistryHosting.v1.host` を使う。 + +## Registry Relay + +spindはVM内registry target portを検出する。必要に応じてhost側relayも張れるが、`REGISTRY` と `localRegistryHosting.v1.host` は `DOCKER_HOST` が指すVM Docker daemonから見えるURLを使う。 + +```text +host localhost: + -> spind relay + -> VM Linux 127.0.0.1: + -> registry container:5000 +``` + +relay port割り当て: + +- VM内target portと同じ番号をhost側で確保できる場合は、その番号を優先する。 +- 使えない場合は別の空きportを割り当てる。 +- 割り当てたrelay portはVM状態ファイルに保存する。 +- 再起動時に保存済みportが使える場合は再利用する。 +- 複数VMが同時に起動する場合、host側relay portは衝突しないようにする。 + +Docker/k3d containerの `HostPort` は変更しない。Dockerのpublished portはcontainer作成時の設定であり、後から安全に追加や変更をしない。 + +## localRegistryHosting + +k3dは `kube-public/local-registry-hosting` ConfigMapでlocal registryを広告する。 + +spindはrestore後に、このConfigMapの `data.localRegistryHosting.v1` を読み、`host` だけを更新する。 + +更新後の例: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-registry-hosting + namespace: kube-public +data: + localRegistryHosting.v1: | + host: localhost:52793 + hostFromContainerRuntime: k3d-sample-registry:5000 + hostFromClusterNetwork: k3d-sample-registry:5000 + help: https://k3d.io/stable/usage/registries/#using-a-local-registry +``` + +更新ルール: + +- `host` は `localhost:` にする。これは `DOCKER_HOST` が指すVM Docker daemonから見えるregistry URLである。 +- `hostFromContainerRuntime` は既存値を維持する。 +- `hostFromClusterNetwork` は既存値を維持する。 +- `help` は既存値を維持する。 +- ConfigMapが存在しない場合は、registry情報から作成する。 +- 複数registryがある場合は、対象clusterに接続されているregistryを優先する。判断できない場合はエラーにする。 + +Tiltなどのlocal registry auto-detectは、このConfigMapの `host` をpush先として使う。spindの推奨shell exportでは `DOCKER_HOST` もVMへ向くため、`host` はVM Docker daemonから見えるURLである必要がある。 + +## 起動時出力 + +registryが利用できる場合、`spind vm start` はregistry情報を表示する。 + +```text +started VM "sample" +docker: unix:///Users/suin/.spind/vms/sample/docker.sock +kubernetes: ready +kubeconfig: /Users/suin/.spind/vms/sample/kubeconfig +context: spind-sample +api server: https://127.0.0.1:49231 +registry: localhost:52793 +``` + +`spind up` はshell export候補に `REGISTRY` を含める。 + +POSIX shell: + +```sh +export DOCKER_HOST='unix:///Users/suin/.spind/vms/sample/docker.sock' +export KUBECONFIG='/Users/suin/.spind/vms/sample/kubeconfig' +export REGISTRY='localhost:52793' +``` + +fish: + +```fish +set -gx DOCKER_HOST 'unix:///Users/suin/.spind/vms/sample/docker.sock' +set -gx KUBECONFIG '/Users/suin/.spind/vms/sample/kubeconfig' +set -gx REGISTRY 'localhost:52793' +``` + +`REGISTRY_FROM_CLUSTER` は表示しない。 + +## JSON出力 + +VM statusやstart結果のJSONにはregistry情報を含める。 + +```json +{ + "kubernetes": { + "ready": true, + "distribution": "k3d", + "kubeconfig": "/Users/suin/.spind/vms/sample/kubeconfig", + "context": "spind-sample", + "apiServer": "https://127.0.0.1:49231" + }, + "registry": { + "ready": true, + "url": "localhost:52793", + "relayPort": 61234, + "targetPort": 52793, + "localRegistryHostingUpdated": true + } +} +``` + +`registry.url` は、`DOCKER_HOST` が指すDocker daemonからpush/pullするためのURLである。 + +## 制限事項 + +- `k3d registry list` はVM内Docker hostから見たportを表示する。host側relay portと一致しない場合がある。 +- `DOCKER_HOST` が指すDocker daemonからpush/pullするregistry URLは、spindの出力または `localRegistryHosting.v1.host` を正とする。 +- Docker API responseを書き換えて、`docker inspect` や `k3d registry list` をhost側relay portに見せることはしない。 +- spindはk3d registry containerを再作成しない。 +- 複数registryがあり対象clusterとの関係を一意に判断できない場合は、自動選択しない。 + +## E2E + +k3d対応のE2Eは、kind-ready E2Eとは別に追加する。 + +基本E2E: + +1. Docker imageからbase VMを作る。 +2. base VMを起動する。 +3. `k3d cluster create` でclusterを作る。 +4. `spind snapshot create --k8s=k3d` でsnapshotを作る。 +5. snapshotからwork VMを作る。 +6. work VMを起動する。 +7. 生成されたkubeconfigで `kubectl get nodes` が成功する。 + +registry E2E: + +1. `spind.yaml` のsetupで `k3d cluster create --registry-create ...` を実行し、registry付きclusterを作る。 +2. setup内で小さなtest imageをbuildし、VM内Dockerからk3d registryへpushする。 +3. snapshotを作る。 +4. snapshotからwork VMを起動する。 +5. `REGISTRY` が出力される。 +6. spindが出力した `DOCKER_HOST` を使い、Docker clientから `docker pull "$REGISTRY/:"` が成功する。 +7. cluster側でそのimageを使うPodが起動できる。 + +registry E2Eでは、`local-registry-hosting` ConfigMapの値確認を主検証にしない。`REGISTRY` 経由の `docker pull` により、snapshot内registry dataと、`DOCKER_HOST` が指すDocker daemonからのregistry URLが実際に使えることを確認する。 + +Tilt連携は別のacceptance E2Eとして扱う。 + +Tilt acceptance E2E: + +1. k3d registry付きsnapshotからwork VMを起動する。 +2. Tiltfileに `default_registry()` を書かず、Tiltのlocal registry auto-detectを使う。 +3. Tiltが `local-registry-hosting` を読んでimage build、push、deployを完了できることを確認する。 +4. PodがReadyになることを確認する。 + +## 実装順序 + +1. 設定とCLIを `k8s: kind|k3d` / `--k8s=kind|k3d` に変更する。 +2. internal modelとmetadataを `kind-ready` からKubernetes-readyへ改名する。 +3. kind adapterを既存実装から移す。 +4. k3d API server adapterを追加する。 +5. k3d kubeconfig server hostの受け入れとrestore時差し替えを実装する。 +6. k3d registry検出を追加する。 +7. registry relayを追加する。 +8. `local-registry-hosting` ConfigMap更新を追加する。 +9. `REGISTRY` の起動時表示とshell exportを追加する。 +10. JSON出力にKubernetes distributionとregistry情報を追加する。 +11. kindとk3dのE2Eを通す。 diff --git a/docs/design/minimum-cli-runbook.md b/docs/design/minimum-cli-runbook.md index 2bfaf1a..d67469c 100644 --- a/docs/design/minimum-cli-runbook.md +++ b/docs/design/minimum-cli-runbook.md @@ -182,7 +182,7 @@ devbox run task e2e ### kind-ready snapshot - host側kindでDocker Host上にkind clusterを作成できる。 -- `spind snapshot create kind-ready --vm kind-base --kind --kubeconfig ~/.kube/config --context kind-dev` が成功する。 +- `spind snapshot create kind-ready --vm kind-base --k8s=kind --kubeconfig ~/.kube/config --context kind-dev` が成功する。 - 指定contextが存在しない場合は失敗する。 - snapshot作成前ready checkで `kubectl get nodes` 相当が成功する。 - nodeが `Ready` でない場合は失敗する。 diff --git a/docs/design/minimum-cli.md b/docs/design/minimum-cli.md index 1c2f7bb..6394780 100644 --- a/docs/design/minimum-cli.md +++ b/docs/design/minimum-cli.md @@ -148,7 +148,7 @@ spind は、ローカルに用意されたベースイメージからVMを作成 - 一時provisioning VM名は `-provisioning` とする。 - provisioned snapshotがない場合、provisioning VMを作成し、`setup` command列をhost側で実行し、snapshotを作成する。 - setup commandには `DOCKER_HOST`、`KUBECONFIG`、`SPIND_DOCKER_HOST`、`SPIND_KUBECONFIG`、`SPIND_PROJECT_NAME`、`SPIND_PROJECT_ROOT`、`SPIND_VM_NAME` を渡す。 -- `kind: true` の場合、snapshot作成時にkind-ready snapshotとして扱う。 +- `k8s: kind` または `k8s: k3d` の場合、snapshot作成時にKubernetes-ready snapshotとして扱う。 - provisioning VMはsnapshot作成成功後に削除する。 - 起動後、Docker endpointまたはVM別kubeconfigが利用できる場合は、`SHELL` に応じてfishまたはPOSIX shell向けの環境変数設定を表示する。 - `--reprovision` は既存provisioned snapshotを使わず、setupからやり直す。 @@ -198,7 +198,7 @@ spind は、ローカルに用意されたベースイメージからVMを作成 - 起動中かつexec readyな `` からsaved state snapshotを作成する。 - 停止中VMからのsnapshot作成は失敗する。 -- `--kind --kubeconfig --context ` が指定された場合、kind-ready snapshotとして作成する。 +- `--k8s=kind|k3d --kubeconfig --context ` が指定された場合、Kubernetes-ready snapshotとして作成する。 - 詳細は `docs/design/snapshot.md` に従う。 - kind-ready snapshotの詳細は `docs/design/kind-ready-snapshot.md` に従う。 diff --git a/docs/design/project-up.md b/docs/design/project-up.md index f3efedb..8e3b6fd 100644 --- a/docs/design/project-up.md +++ b/docs/design/project-up.md @@ -22,7 +22,7 @@ setup: - `name`: project名。省略時は `spind.yaml` があるdirectoryのbasename。 - `image`: base image名。省略時は `docker`。 -- `kind`: kind-ready snapshotとして作るかどうか。省略時は `false`。 +- `k8s`: Kubernetes-ready snapshotとして作るdistribution。`kind` または `k3d`。省略時は通常snapshot。 - `setup`: 初回provision時にhost側で実行するcommand列。省略時は空。 ## 生成名 @@ -58,7 +58,7 @@ setup commandには次の環境変数を渡す。 - `SPIND_PROJECT_ROOT`: project root。 - `SPIND_VM_NAME`: provisioning VM名。 -`kind: true` の場合、snapshot作成時にkind-ready snapshotとして扱い、setup commandへ渡した `KUBECONFIG` をsnapshot作成時にも使う。 +`k8s: kind` または `k8s: k3d` の場合、snapshot作成時にKubernetes-ready snapshotとして扱い、setup commandへ渡した `KUBECONFIG` をsnapshot作成時にも使う。 起動後、Docker endpointまたはVM別kubeconfigが利用できる場合、`spind up` は現在のshellで使える環境変数設定を表示する。 diff --git a/docs/design/roadmap.md b/docs/design/roadmap.md index a4170ef..3353a36 100644 --- a/docs/design/roadmap.md +++ b/docs/design/roadmap.md @@ -122,7 +122,7 @@ Phase 11では、kind cluster起動済みDocker Host VMをsnapshot化し、resto - spindは初期スコープでは `kind create cluster` を包まない。 - 利用者はhost側kindとDocker Host endpointを使ってkind clusterを作る。 -- `spind snapshot create --kind --kubeconfig --context ` でkind-ready snapshotを作る。 +- `spind snapshot create --k8s=kind|k3d --kubeconfig --context ` でKubernetes-ready snapshotを作る。 - `spind vm start` はkind-ready snapshot由来VMにhost空きportを割り当て、VM別kubeconfigを生成する。 - 標準kubeconfig pathは `~/.spind/vms//kubeconfig` とする。 - `~/.kube/config` へのmergeは初期スコープ外にする。 diff --git a/docs/design/snapshot.md b/docs/design/snapshot.md index 958dbc4..610a649 100644 --- a/docs/design/snapshot.md +++ b/docs/design/snapshot.md @@ -104,7 +104,7 @@ Cloud Hypervisor backendの必須ファイル: - `` が既に存在する場合、上書きせずに失敗する。 - Virtualization.framework backendのsnapshot作成は、Swift実行部にsaved state保存を要求し、`vf/state.vzvmsave`、`vf/disk.img`、`vf/kernel`、`vf/initramfs`、必要なmetadataを保存して行う。 - Cloud Hypervisor backendのsnapshot作成は、Cloud Hypervisor REST APIでVMをpauseし、snapshot directoryへ `config.json`、`memory-ranges`、`state.json` を保存し、対応する `kernel`、`initramfs`、`disk.img` と必要なmetadataを保存して行う。 -- `--kind --kubeconfig --context ` が指定された場合、kind-ready snapshotとして扱い、詳細は `docs/design/kind-ready-snapshot.md` に従う。 +- `--k8s=kind|k3d --kubeconfig --context ` が指定された場合、Kubernetes-ready snapshotとして扱い、詳細は `docs/design/k8s-distribution-and-registry.md` に従う。 - snapshot作成後、元VMは停止済みとして扱う。 ### `spind snapshot list` diff --git a/examples/k3d/spind.yaml b/examples/k3d/spind.yaml new file mode 100644 index 0000000..ed25b3c --- /dev/null +++ b/examples/k3d/spind.yaml @@ -0,0 +1,11 @@ +name: k3d-sample +image: docker +k8s: k3d +setup: + - "k3d cluster create k3d-sample --registry-create k3d-sample-registry --kubeconfig-update-default=false" + - 'k3d kubeconfig get k3d-sample > "$SPIND_KUBECONFIG"' + - "kubectl --context k3d-k3d-sample wait node --all --for=condition=Ready --timeout=180s" + - "helm install cert-manager oci://quay.io/jetstack/charts/cert-manager --version v1.20.2 --namespace cert-manager --create-namespace --set crds.enabled=true" + - "kubectl --context k3d-k3d-sample --namespace cert-manager rollout status deployment/cert-manager --timeout=300s" + - "kubectl --context k3d-k3d-sample --namespace cert-manager rollout status deployment/cert-manager-cainjector --timeout=300s" + - "kubectl --context k3d-k3d-sample --namespace cert-manager rollout status deployment/cert-manager-webhook --timeout=300s" diff --git a/examples/tilt/Dockerfile b/examples/tilt/Dockerfile new file mode 100644 index 0000000..ab401db --- /dev/null +++ b/examples/tilt/Dockerfile @@ -0,0 +1,3 @@ +FROM nginx:1.27-alpine + +COPY app/index.html /usr/share/nginx/html/index.html diff --git a/examples/tilt/Tiltfile b/examples/tilt/Tiltfile new file mode 100644 index 0000000..1c0421a --- /dev/null +++ b/examples/tilt/Tiltfile @@ -0,0 +1,5 @@ +docker_build("tilt-sample", ".") + +k8s_yaml("k8s.yaml") + +k8s_resource("tilt-sample", port_forwards=8080) diff --git a/examples/tilt/app/index.html b/examples/tilt/app/index.html new file mode 100644 index 0000000..2236ffd --- /dev/null +++ b/examples/tilt/app/index.html @@ -0,0 +1,15 @@ + + + + + + tilt-sample + + +

tilt-sample

+

Served from a spind k3d cluster.

+ + diff --git a/examples/tilt/k8s.yaml b/examples/tilt/k8s.yaml new file mode 100644 index 0000000..0dffb06 --- /dev/null +++ b/examples/tilt/k8s.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tilt-sample + labels: + app.kubernetes.io/name: tilt-sample +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: tilt-sample + template: + metadata: + labels: + app.kubernetes.io/name: tilt-sample + spec: + containers: + - name: web + image: tilt-sample + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: tilt-sample + labels: + app.kubernetes.io/name: tilt-sample +spec: + selector: + app.kubernetes.io/name: tilt-sample + ports: + - name: http + port: 80 + targetPort: 80 diff --git a/examples/tilt/spind.yaml b/examples/tilt/spind.yaml new file mode 100644 index 0000000..67fe18a --- /dev/null +++ b/examples/tilt/spind.yaml @@ -0,0 +1,7 @@ +name: tilt-sample +image: docker +k8s: k3d +setup: + - "k3d cluster create tilt-sample --registry-create tilt-sample-registry --kubeconfig-update-default=false" + - 'k3d kubeconfig get tilt-sample > "$SPIND_KUBECONFIG"' + - "kubectl --context k3d-tilt-sample wait node --all --for=condition=Ready --timeout=180s" diff --git a/internal/spind/cli/cli_test.go b/internal/spind/cli/cli_test.go index 13a024c..3dbe5b9 100644 --- a/internal/spind/cli/cli_test.go +++ b/internal/spind/cli/cli_test.go @@ -183,8 +183,8 @@ func TestParseCommandLineAcceptsSnapshotCreate(t *testing.T) { } } -func TestParseCommandLineAcceptsKindSnapshotCreate(t *testing.T) { - cli, ctx, exitCode, handled := Parse([]string{"snapshot", "create", "kind-ready", "--vm", "kind-base", "--kind", "--kubeconfig", "/tmp/config", "--context", "kind-dev"}, io.Discard, io.Discard) +func TestParseCommandLineAcceptsK8sSnapshotCreate(t *testing.T) { + cli, ctx, exitCode, handled := Parse([]string{"snapshot", "create", "kind-ready", "--vm", "kind-base", "--k8s=kind", "--kubeconfig", "/tmp/config", "--context", "kind-dev"}, io.Discard, io.Discard) if handled { t.Fatalf("parseCommandLine handled with exitCode=%d", exitCode) } @@ -192,7 +192,7 @@ func TestParseCommandLineAcceptsKindSnapshotCreate(t *testing.T) { t.Fatalf("ctx.Command() = %q, want snapshot create ", ctx.Command()) } create := cli.Snapshot.Create - if create.Name != "kind-ready" || create.VM != "kind-base" || !create.Kind || create.Kubeconfig != "/tmp/config" || create.Context != "kind-dev" { + if create.Name != "kind-ready" || create.VM != "kind-base" || create.K8s != "kind" || create.Kubeconfig != "/tmp/config" || create.Context != "kind-dev" { t.Fatalf("Snapshot.Create = %#v", create) } } diff --git a/internal/spind/cli/output/snapshot.go b/internal/spind/cli/output/snapshot.go index cec9fc5..2cf7454 100644 --- a/internal/spind/cli/output/snapshot.go +++ b/internal/spind/cli/output/snapshot.go @@ -9,24 +9,25 @@ import ( ) type SnapshotInfo struct { - Name string `json:"name"` - SourceVM string `json:"sourceVM,omitempty"` - Backend string `json:"backend,omitempty"` - CreatedAt string `json:"createdAt,omitempty"` - CPUCount int `json:"cpuCount,omitempty"` - MemoryMiB int `json:"memoryMiB,omitempty"` - ExecPort uint32 `json:"execPort,omitempty"` - KernelCommand string `json:"kernelCommand,omitempty"` - DiskSizeBytes int64 `json:"diskSizeBytes"` - StateSizeBytes int64 `json:"stateSizeBytes"` - TotalSizeBytes int64 `json:"totalSizeBytes"` - Health string `json:"health"` - HealthMessage string `json:"healthMessage,omitempty"` - SnapshotDir string `json:"snapshotDir"` - KindReady bool `json:"kindReady,omitempty"` - KindMetadata spindkind.Metadata `json:"kindMetadata,omitempty"` - KindTemplate bool `json:"kindKubeconfigTemplate,omitempty"` - Artifacts []SnapshotArtifact `json:"artifacts,omitempty"` + Name string `json:"name"` + SourceVM string `json:"sourceVM,omitempty"` + Backend string `json:"backend,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + CPUCount int `json:"cpuCount,omitempty"` + MemoryMiB int `json:"memoryMiB,omitempty"` + ExecPort uint32 `json:"execPort,omitempty"` + KernelCommand string `json:"kernelCommand,omitempty"` + DiskSizeBytes int64 `json:"diskSizeBytes"` + StateSizeBytes int64 `json:"stateSizeBytes"` + TotalSizeBytes int64 `json:"totalSizeBytes"` + Health string `json:"health"` + HealthMessage string `json:"healthMessage,omitempty"` + SnapshotDir string `json:"snapshotDir"` + K8sReady bool `json:"k8sReady,omitempty"` + K8sDistribution string `json:"k8sDistribution,omitempty"` + K8sMetadata spindkind.Metadata `json:"k8sMetadata,omitempty"` + K8sTemplate bool `json:"k8sKubeconfigTemplate,omitempty"` + Artifacts []SnapshotArtifact `json:"artifacts,omitempty"` } type SnapshotArtifact struct { @@ -47,24 +48,25 @@ func NewSnapshotInfo(info spindsnapshot.Info) SnapshotInfo { }) } return SnapshotInfo{ - Name: info.Name, - SourceVM: info.SourceVM, - Backend: info.Backend, - CreatedAt: FormatTime(info.CreatedAt), - CPUCount: info.CPUCount, - MemoryMiB: info.MemoryMiB, - ExecPort: info.ExecPort, - KernelCommand: info.KernelCommand, - DiskSizeBytes: info.DiskSizeBytes, - StateSizeBytes: info.StateSizeBytes, - TotalSizeBytes: info.TotalSizeBytes, - Health: info.Health, - HealthMessage: info.HealthMessage, - SnapshotDir: info.SnapshotDir, - KindReady: info.KindReady, - KindMetadata: info.KindMetadata, - KindTemplate: info.KindTemplate, - Artifacts: artifacts, + Name: info.Name, + SourceVM: info.SourceVM, + Backend: info.Backend, + CreatedAt: FormatTime(info.CreatedAt), + CPUCount: info.CPUCount, + MemoryMiB: info.MemoryMiB, + ExecPort: info.ExecPort, + KernelCommand: info.KernelCommand, + DiskSizeBytes: info.DiskSizeBytes, + StateSizeBytes: info.StateSizeBytes, + TotalSizeBytes: info.TotalSizeBytes, + Health: info.Health, + HealthMessage: info.HealthMessage, + SnapshotDir: info.SnapshotDir, + K8sReady: info.KindReady, + K8sDistribution: info.K8sDistribution, + K8sMetadata: info.KindMetadata, + K8sTemplate: info.KindTemplate, + Artifacts: artifacts, } } @@ -122,12 +124,13 @@ func PrintSnapshotInfo(stdout io.Writer, info spindsnapshot.Info) { } fmt.Fprintf(stdout, "snapshotDir: %s\n", info.SnapshotDir) if info.KindReady { - fmt.Fprintln(stdout, "kindReady: true") - fmt.Fprintf(stdout, "kindContext: %s\n", DisplayValue(info.KindMetadata.SourceContext)) - fmt.Fprintf(stdout, "kindCluster: %s\n", DisplayValue(info.KindMetadata.SourceCluster)) - fmt.Fprintf(stdout, "kindKubeconfigTemplate: %t\n", info.KindTemplate) + fmt.Fprintln(stdout, "k8sReady: true") + fmt.Fprintf(stdout, "k8sDistribution: %s\n", DisplayValue(info.K8sDistribution)) + fmt.Fprintf(stdout, "k8sContext: %s\n", DisplayValue(info.KindMetadata.SourceContext)) + fmt.Fprintf(stdout, "k8sCluster: %s\n", DisplayValue(info.KindMetadata.SourceCluster)) + fmt.Fprintf(stdout, "k8sKubeconfigTemplate: %t\n", info.KindTemplate) if len(info.KindMetadata.Nodes) > 0 { - fmt.Fprintln(stdout, "kindNodes:") + fmt.Fprintln(stdout, "k8sNodes:") for _, node := range info.KindMetadata.Nodes { fmt.Fprintf(stdout, " %s\tready=%t\n", node.Name, node.Ready) } diff --git a/internal/spind/cli/output/vm_detail.go b/internal/spind/cli/output/vm_detail.go index ed55e7a..84cc280 100644 --- a/internal/spind/cli/output/vm_detail.go +++ b/internal/spind/cli/output/vm_detail.go @@ -39,6 +39,7 @@ func PrintVMInfo(stdout io.Writer, info spindvm.Info) { PrintDockerInfo(stdout, info) PrintHostShareInfo(stdout, info) PrintKubernetesInfo(stdout, info) + PrintRegistryInfo(stdout, info) if !info.StartedAt.IsZero() { fmt.Fprintf(stdout, "startedAt: %s\n", info.StartedAt.Format(time.RFC3339)) } @@ -241,6 +242,38 @@ func PrintKubernetesInfo(stdout io.Writer, info spindvm.Info) { } } +func PrintRegistryInfo(stdout io.Writer, info spindvm.Info) { + if info.RegistryURL == "" && info.RegistryLastError == "" && info.RegistryTargetPort == 0 { + return + } + if info.RegistryReady { + fmt.Fprintln(stdout, "registry: ready") + } else { + fmt.Fprintln(stdout, "registry: unavailable") + } + if info.RegistryURL != "" { + fmt.Fprintf(stdout, "registryUrl: %s\n", info.RegistryURL) + } + if info.RegistryPort != 0 { + fmt.Fprintf(stdout, "registryPort: %d\n", info.RegistryPort) + } + if info.RegistryTargetPort != 0 { + fmt.Fprintf(stdout, "registryTargetPort: %d\n", info.RegistryTargetPort) + } + if info.RegistryRelayPID != 0 { + fmt.Fprintf(stdout, "registryRelayPid: %d\n", info.RegistryRelayPID) + } + if info.RegistryHostFromCluster != "" { + fmt.Fprintf(stdout, "registryHostFromCluster: %s\n", info.RegistryHostFromCluster) + } + if info.RegistryLastError != "" { + fmt.Fprintf(stdout, "registryLastError: %s\n", info.RegistryLastError) + } + if info.RegistryRelayLogPath != "" { + fmt.Fprintf(stdout, "registryLogPath: %s\n", info.RegistryRelayLogPath) + } +} + func PrintVMLogHint(stderr io.Writer, info spindvm.Info) { if len(info.LogPaths) == 0 { return diff --git a/internal/spind/cli/output/vm_json.go b/internal/spind/cli/output/vm_json.go index ea713b2..5b46f11 100644 --- a/internal/spind/cli/output/vm_json.go +++ b/internal/spind/cli/output/vm_json.go @@ -19,6 +19,7 @@ type VMInfo struct { CloudHypervisorVsockPath string `json:"cloudHypervisorVsockPath,omitempty"` Docker DockerStatus `json:"docker"` Kubernetes KubernetesStatus `json:"kubernetes"` + Registry RegistryStatus `json:"registry,omitempty"` DockerSocketPath string `json:"dockerSocketPath,omitempty"` DockerEndpointURI string `json:"dockerEndpointUri,omitempty"` DockerRelayPID int `json:"dockerRelayPid,omitempty"` @@ -74,6 +75,15 @@ type VMInfo struct { KubernetesAPIServerTargetPort int `json:"kubernetesApiServerTargetPort,omitempty"` KubernetesRelayPID int `json:"kubernetesRelayPid,omitempty"` KubernetesRelayLogPath string `json:"kubernetesRelayLogPath,omitempty"` + RegistryReady bool `json:"registryReady"` + RegistryLastError string `json:"registryLastError,omitempty"` + RegistryURL string `json:"registryUrl,omitempty"` + RegistryPort int `json:"registryPort,omitempty"` + RegistryTargetPort int `json:"registryTargetPort,omitempty"` + RegistryRelayPID int `json:"registryRelayPid,omitempty"` + RegistryRelayLogPath string `json:"registryRelayLogPath,omitempty"` + RegistryHostFromCluster string `json:"registryHostFromCluster,omitempty"` + RegistryLocalHostingUpdated bool `json:"registryLocalHostingUpdated"` SerialLogPath string `json:"serialLogPath,omitempty"` BackendLogPath string `json:"backendLogPath,omitempty"` EventLogPath string `json:"eventLogPath,omitempty"` @@ -110,6 +120,18 @@ type KubernetesStatus struct { RelayPID int `json:"relayPid,omitempty"` } +type RegistryStatus struct { + Ready bool `json:"ready"` + URL string `json:"url,omitempty"` + RelayPort int `json:"relayPort,omitempty"` + TargetPort int `json:"targetPort,omitempty"` + RelayPID int `json:"relayPid,omitempty"` + Log string `json:"log,omitempty"` + Reason string `json:"reason,omitempty"` + HostFromCluster string `json:"hostFromCluster,omitempty"` + LocalRegistryHostingUpdated bool `json:"localRegistryHostingUpdated"` +} + func NewVMInfo(info spindvm.Info) VMInfo { return VMInfo{ Name: info.Name, @@ -128,6 +150,7 @@ func NewVMInfo(info spindvm.Info) VMInfo { CloudHypervisorVsockPath: info.CloudHypervisorVsockPath, Docker: newDockerStatus(info), Kubernetes: newKubernetesStatus(info), + Registry: newRegistryStatus(info), DockerSocketPath: info.DockerSocketPath, DockerEndpointURI: info.DockerEndpointURI, DockerRelayPID: info.DockerRelayPID, @@ -183,6 +206,15 @@ func NewVMInfo(info spindvm.Info) VMInfo { KubernetesAPIServerTargetPort: info.KubernetesAPIServerTargetPort, KubernetesRelayPID: info.KubernetesRelayPID, KubernetesRelayLogPath: info.KubernetesRelayLogPath, + RegistryReady: info.RegistryReady, + RegistryLastError: info.RegistryLastError, + RegistryURL: info.RegistryURL, + RegistryPort: info.RegistryPort, + RegistryTargetPort: info.RegistryTargetPort, + RegistryRelayPID: info.RegistryRelayPID, + RegistryRelayLogPath: info.RegistryRelayLogPath, + RegistryHostFromCluster: info.RegistryHostFromCluster, + RegistryLocalHostingUpdated: info.RegistryLocalHostingUpdated, SerialLogPath: info.SerialLogPath, BackendLogPath: info.BackendLogPath, EventLogPath: info.EventLogPath, @@ -193,6 +225,20 @@ func NewVMInfo(info spindvm.Info) VMInfo { } } +func newRegistryStatus(info spindvm.Info) RegistryStatus { + return RegistryStatus{ + Ready: info.RegistryReady, + URL: info.RegistryURL, + RelayPort: info.RegistryPort, + TargetPort: info.RegistryTargetPort, + RelayPID: info.RegistryRelayPID, + Log: info.RegistryRelayLogPath, + Reason: info.RegistryLastError, + HostFromCluster: info.RegistryHostFromCluster, + LocalRegistryHostingUpdated: info.RegistryLocalHostingUpdated, + } +} + func newKubernetesStatus(info spindvm.Info) KubernetesStatus { return KubernetesStatus{ Support: info.KubernetesSupport, diff --git a/internal/spind/cli/output/vm_start.go b/internal/spind/cli/output/vm_start.go index 42e1c92..50c964c 100644 --- a/internal/spind/cli/output/vm_start.go +++ b/internal/spind/cli/output/vm_start.go @@ -76,3 +76,20 @@ func PrintKubernetesStartLine(stdout io.Writer, stderr io.Writer, info spindvm.I fmt.Fprintf(stderr, "kubernetesLogPath: %s\n", info.KubernetesRelayLogPath) } } + +func PrintRegistryStartLine(stdout io.Writer, stderr io.Writer, info spindvm.Info) { + if info.RegistryURL == "" && info.RegistryLastError == "" { + return + } + if info.RegistryReady { + fmt.Fprintf(stdout, "registry: %s\n", info.RegistryURL) + return + } + fmt.Fprintln(stdout, "registry: unavailable") + if info.RegistryLastError != "" { + fmt.Fprintf(stderr, "warning: registry unavailable: %s\n", info.RegistryLastError) + } + if info.RegistryRelayLogPath != "" { + fmt.Fprintf(stderr, "registryLogPath: %s\n", info.RegistryRelayLogPath) + } +} diff --git a/internal/spind/kind/ready_snapshot.go b/internal/spind/kind/ready_snapshot.go index aa1ee12..530e2dc 100644 --- a/internal/spind/kind/ready_snapshot.go +++ b/internal/spind/kind/ready_snapshot.go @@ -39,7 +39,10 @@ type kubectlNode struct { func PrepareSnapshot(ctx context.Context, tmpDir string, dockerEndpointPath string, dockerReady bool, options SnapshotOptions) (*Metadata, error) { if !dockerReady { - return nil, errors.New("kind-ready snapshot requires Docker API ready") + return nil, errors.New("Kubernetes-ready snapshot requires Docker API ready") + } + if options.Distribution != DistributionKind && options.Distribution != DistributionK3d { + return nil, fmt.Errorf("unsupported Kubernetes distribution %q", options.Distribution) } kubeconfigPath, err := ResolveKubeconfigPath(options.KubeconfigPath) if err != nil { @@ -61,7 +64,7 @@ func PrepareSnapshot(ctx context.Context, tmpDir string, dockerEndpointPath stri if err != nil { return nil, err } - if err := ValidateKubeconfigMatchesDocker(ctx, dockerEndpointPath, sourceServer, targetPort); err != nil { + if err := ValidateKubeconfigMatchesDocker(ctx, dockerEndpointPath, options.Distribution, sourceServer, targetPort); err != nil { return nil, err } nodes, err := KubectlReadyNodes(ctx, kubeconfigPath, contextName) @@ -78,6 +81,7 @@ func PrepareSnapshot(ctx context.Context, tmpDir string, dockerEndpointPath stri } metadata := Metadata{ KindReady: true, + Distribution: options.Distribution, SourceKubeconfigPath: kubeconfigPath, SourceContext: contextName, SourceCluster: sourceCluster, @@ -88,6 +92,15 @@ func PrepareSnapshot(ctx context.Context, tmpDir string, dockerEndpointPath stri ReadyCheck: "ok", ReadyCheckCompletedAt: time.Now().UTC(), } + if options.Distribution == DistributionK3d { + registry, err := K3dRegistry(ctx, dockerEndpointPath) + if err != nil { + return nil, err + } + metadata.RegistryTargetPort = registry.TargetPort + metadata.RegistryHost = registry.Host + metadata.RegistryHostFromCluster = registry.HostFromCluster + } if err := store.WriteJSON(filepath.Join(kindDir, MetadataName), metadata, 0o644); err != nil { return nil, fmt.Errorf("write kind metadata: %w", err) } @@ -174,29 +187,33 @@ func KubectlReadyNodes(ctx context.Context, kubeconfigPath string, contextName s return nodes, nil } -func ValidateKubeconfigMatchesDocker(ctx context.Context, dockerEndpointPath string, server string, serverPort int) error { +func ValidateKubeconfigMatchesDocker(ctx context.Context, dockerEndpointPath string, distribution string, server string, serverPort int) error { if dockerEndpointPath == "" { - return errors.New("kind-ready snapshot requires Docker endpoint path") + return errors.New("Kubernetes-ready snapshot requires Docker endpoint path") } - publishedPorts, err := ControlPlanePublishedPorts(ctx, dockerEndpointPath) + publishedPorts, err := APIServerPublishedPorts(ctx, dockerEndpointPath, distribution) if err != nil { - return fmt.Errorf("list kind control-plane Docker ports: %w", err) + return fmt.Errorf("list Kubernetes API Docker ports: %w", err) } - return ValidateServerPublishedPort(server, serverPort, publishedPorts) + return ValidateDistributionServerPublishedPort(distribution, server, serverPort, publishedPorts) } -func ControlPlanePublishedPorts(ctx context.Context, dockerEndpointPath string) ([]uint16, error) { +func APIServerPublishedPorts(ctx context.Context, dockerEndpointPath string, distribution string) ([]uint16, error) { containers, err := spinddocker.ListContainers(ctx, dockerEndpointPath) if err != nil { return nil, err } - return ControlPlanePublishedPortsFromContainers(containers), nil + return APIServerPublishedPortsFromContainers(containers, distribution), nil } func ControlPlanePublishedPortsFromContainers(containers []spinddocker.ContainerSummary) []uint16 { + return APIServerPublishedPortsFromContainers(containers, DistributionKind) +} + +func APIServerPublishedPortsFromContainers(containers []spinddocker.ContainerSummary, distribution string) []uint16 { seen := map[uint16]struct{}{} for _, container := range containers { - if !isControlPlaneContainer(container) { + if !isAPIServerContainer(container, distribution) { continue } for _, port := range container.Ports { @@ -216,6 +233,17 @@ func ControlPlanePublishedPortsFromContainers(containers []spinddocker.Container return ports } +func isAPIServerContainer(container spinddocker.ContainerSummary, distribution string) bool { + switch distribution { + case DistributionKind: + return isControlPlaneContainer(container) + case DistributionK3d: + return container.Labels["k3d.role"] == "loadbalancer" + default: + return false + } +} + func isControlPlaneContainer(container spinddocker.ContainerSummary) bool { if container.Labels["io.x-k8s.kind.role"] == "control-plane" { return true @@ -230,6 +258,10 @@ func isControlPlaneContainer(container spinddocker.ContainerSummary) bool { } func ValidateServerPublishedPort(server string, serverPort int, publishedPorts []uint16) error { + return ValidateDistributionServerPublishedPort(DistributionKind, server, serverPort, publishedPorts) +} + +func ValidateDistributionServerPublishedPort(distribution string, server string, serverPort int, publishedPorts []uint16) error { parsed, err := url.Parse(server) if err != nil { return fmt.Errorf("parse kubeconfig server: %w", err) @@ -238,18 +270,28 @@ func ValidateServerPublishedPort(server string, serverPort int, publishedPorts [ if host == "" { return fmt.Errorf("kubeconfig server %q has no host", server) } - if !isLoopbackHost(host) { + if !isAcceptedServerHost(distribution, host) { return fmt.Errorf("kubeconfig server %q is not a loopback host", server) } if len(publishedPorts) == 0 { - return errors.New("target VM has no published kind control-plane API port") + if distribution == DistributionKind { + return errors.New("target VM has no published kind control-plane API port") + } + return errors.New("target VM has no published Kubernetes API port") } for _, publishedPort := range publishedPorts { if int(publishedPort) == serverPort { return nil } } - return fmt.Errorf("kubeconfig server %q uses port %d, but target VM kind control-plane publishes %s", server, serverPort, formatPortList(publishedPorts)) + return fmt.Errorf("kubeconfig server %q uses port %d, but target VM Kubernetes API publishes %s", server, serverPort, formatPortList(publishedPorts)) +} + +func isAcceptedServerHost(distribution string, host string) bool { + if distribution == DistributionK3d && host == "0.0.0.0" { + return true + } + return isLoopbackHost(host) } func isLoopbackHost(host string) bool { diff --git a/internal/spind/kind/ready_snapshot_test.go b/internal/spind/kind/ready_snapshot_test.go index 7f92734..9fec8ba 100644 --- a/internal/spind/kind/ready_snapshot_test.go +++ b/internal/spind/kind/ready_snapshot_test.go @@ -131,3 +131,32 @@ func TestControlPlanePublishedPortsFromContainers(t *testing.T) { t.Fatalf("ControlPlanePublishedPortsFromContainers = %#v, want %#v", ports, want) } } + +func TestK3dAPIServerPublishedPortsFromContainersUsesLoadBalancer(t *testing.T) { + ports := APIServerPublishedPortsFromContainers([]spinddocker.ContainerSummary{ + { + Names: []string{"/k3d-dev-server-0"}, + Labels: map[string]string{"k3d.role": "server"}, + Ports: []spinddocker.PortMapping{ + {PrivatePort: 6443, PublicPort: 35123, Type: "tcp"}, + }, + }, + { + Names: []string{"/k3d-dev-serverlb"}, + Labels: map[string]string{"k3d.role": "loadbalancer"}, + Ports: []spinddocker.PortMapping{ + {PrivatePort: 6443, PublicPort: 35124, Type: "tcp"}, + }, + }, + }, DistributionK3d) + want := []uint16{35124} + if !reflect.DeepEqual(ports, want) { + t.Fatalf("APIServerPublishedPortsFromContainers = %#v, want %#v", ports, want) + } +} + +func TestValidateK3dServerPublishedPortAcceptsWildcardHost(t *testing.T) { + if err := ValidateDistributionServerPublishedPort(DistributionK3d, "https://0.0.0.0:35123", 35123, []uint16{35123}); err != nil { + t.Fatalf("ValidateDistributionServerPublishedPort() error = %v", err) + } +} diff --git a/internal/spind/kind/registry.go b/internal/spind/kind/registry.go new file mode 100644 index 0000000..c7722a3 --- /dev/null +++ b/internal/spind/kind/registry.go @@ -0,0 +1,71 @@ +package kind + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + spinddocker "github.com/suin/spind/internal/spind/docker" +) + +type RegistryInfo struct { + TargetPort int + Host string + HostFromCluster string +} + +func K3dRegistry(ctx context.Context, dockerEndpointPath string) (RegistryInfo, error) { + containers, err := spinddocker.ListContainers(ctx, dockerEndpointPath) + if err != nil { + return RegistryInfo{}, fmt.Errorf("list k3d registry containers: %w", err) + } + registries := k3dRegistriesFromContainers(containers) + if len(registries) == 0 { + return RegistryInfo{}, nil + } + if len(registries) > 1 { + return RegistryInfo{}, errors.New("multiple k3d registries found; automatic registry selection is ambiguous") + } + return registries[0], nil +} + +func k3dRegistriesFromContainers(containers []spinddocker.ContainerSummary) []RegistryInfo { + registries := []RegistryInfo{} + for _, container := range containers { + if container.Labels["k3d.role"] != "registry" { + continue + } + targetPort := 0 + for _, port := range container.Ports { + if port.PrivatePort == 5000 && port.PublicPort != 0 && strings.ToLower(port.Type) == "tcp" { + targetPort = int(port.PublicPort) + break + } + } + if targetPort == 0 { + continue + } + host := firstContainerName(container) + registries = append(registries, RegistryInfo{ + TargetPort: targetPort, + Host: host, + HostFromCluster: host + ":5000", + }) + } + sort.Slice(registries, func(i int, j int) bool { + return registries[i].Host < registries[j].Host + }) + return registries +} + +func firstContainerName(container spinddocker.ContainerSummary) string { + for _, name := range container.Names { + name = strings.TrimPrefix(name, "/") + if name != "" { + return name + } + } + return "" +} diff --git a/internal/spind/kind/registry_test.go b/internal/spind/kind/registry_test.go new file mode 100644 index 0000000..7bf3305 --- /dev/null +++ b/internal/spind/kind/registry_test.go @@ -0,0 +1,35 @@ +package kind + +import ( + "reflect" + "testing" + + spinddocker "github.com/suin/spind/internal/spind/docker" +) + +func TestK3dRegistriesFromContainers(t *testing.T) { + registries := k3dRegistriesFromContainers([]spinddocker.ContainerSummary{ + { + Names: []string{"/k3d-dev-registry"}, + Labels: map[string]string{"k3d.role": "registry"}, + Ports: []spinddocker.PortMapping{ + {PrivatePort: 5000, PublicPort: 52793, Type: "tcp"}, + }, + }, + { + Names: []string{"/k3d-dev-serverlb"}, + Labels: map[string]string{"k3d.role": "loadbalancer"}, + Ports: []spinddocker.PortMapping{ + {PrivatePort: 6443, PublicPort: 35124, Type: "tcp"}, + }, + }, + }) + want := []RegistryInfo{{ + TargetPort: 52793, + Host: "k3d-dev-registry", + HostFromCluster: "k3d-dev-registry:5000", + }} + if !reflect.DeepEqual(registries, want) { + t.Fatalf("k3dRegistriesFromContainers = %#v, want %#v", registries, want) + } +} diff --git a/internal/spind/kind/types.go b/internal/spind/kind/types.go index 559c86f..975dc10 100644 --- a/internal/spind/kind/types.go +++ b/internal/spind/kind/types.go @@ -3,22 +3,28 @@ package kind import "time" const ( + DistributionKind = "kind" + DistributionK3d = "k3d" DirName = "kind" MetadataName = "metadata.json" KubeconfigTemplateName = "kubeconfig.template" ) type Metadata struct { - KindReady bool `json:"kindReady"` - SourceKubeconfigPath string `json:"sourceKubeconfigPath,omitempty"` - SourceContext string `json:"sourceContext,omitempty"` - SourceCluster string `json:"sourceCluster,omitempty"` - SourceUser string `json:"sourceUser,omitempty"` - SourceServer string `json:"sourceServer,omitempty"` - APIServerTargetPort int `json:"apiServerTargetPort,omitempty"` - Nodes []NodeSummary `json:"nodes,omitempty"` - ReadyCheck string `json:"readyCheck,omitempty"` - ReadyCheckCompletedAt time.Time `json:"readyCheckCompletedAt,omitempty"` + KindReady bool `json:"kindReady"` + Distribution string `json:"distribution,omitempty"` + SourceKubeconfigPath string `json:"sourceKubeconfigPath,omitempty"` + SourceContext string `json:"sourceContext,omitempty"` + SourceCluster string `json:"sourceCluster,omitempty"` + SourceUser string `json:"sourceUser,omitempty"` + SourceServer string `json:"sourceServer,omitempty"` + APIServerTargetPort int `json:"apiServerTargetPort,omitempty"` + RegistryTargetPort int `json:"registryTargetPort,omitempty"` + RegistryHost string `json:"registryHost,omitempty"` + RegistryHostFromCluster string `json:"registryHostFromCluster,omitempty"` + Nodes []NodeSummary `json:"nodes,omitempty"` + ReadyCheck string `json:"readyCheck,omitempty"` + ReadyCheckCompletedAt time.Time `json:"readyCheckCompletedAt,omitempty"` } type NodeSummary struct { @@ -27,6 +33,7 @@ type NodeSummary struct { } type SnapshotOptions struct { + Distribution string KubeconfigPath string Context string } diff --git a/internal/spind/snapshot/create/command.go b/internal/spind/snapshot/create/command.go index 0139555..1232360 100644 --- a/internal/spind/snapshot/create/command.go +++ b/internal/spind/snapshot/create/command.go @@ -28,8 +28,8 @@ func New(options *Options, runtime *cliruntime.Runtime) *cobra.Command { }, } command.Flags().StringVar(&options.VM, "vm", "", "Source VM name.") - command.Flags().BoolVar(&options.Kind, "kind", false, "Create a kind-ready snapshot.") - command.Flags().StringVar(&options.Kubeconfig, "kubeconfig", "", "Host kubeconfig path for kind-ready snapshot. Defaults to KUBECONFIG or ~/.kube/config.") - command.Flags().StringVar(&options.Context, "context", "", "Kubeconfig context for kind-ready snapshot. Defaults to current-context.") + command.Flags().StringVar(&options.K8s, "k8s", "", "Create a Kubernetes-ready snapshot for distribution: kind or k3d.") + command.Flags().StringVar(&options.Kubeconfig, "kubeconfig", "", "Host kubeconfig path for Kubernetes-ready snapshot. Defaults to KUBECONFIG or ~/.kube/config.") + command.Flags().StringVar(&options.Context, "context", "", "Kubeconfig context for Kubernetes-ready snapshot. Defaults to current-context.") return command } diff --git a/internal/spind/snapshot/create/options.go b/internal/spind/snapshot/create/options.go index b372f69..4a3e887 100644 --- a/internal/spind/snapshot/create/options.go +++ b/internal/spind/snapshot/create/options.go @@ -3,7 +3,7 @@ package create type Options struct { Name string VM string - Kind bool + K8s string Kubeconfig string Context string } diff --git a/internal/spind/snapshot/create/run.go b/internal/spind/snapshot/create/run.go index db895e4..d038880 100644 --- a/internal/spind/snapshot/create/run.go +++ b/internal/spind/snapshot/create/run.go @@ -16,7 +16,7 @@ import ( func Run(ctx context.Context, cfg config.Config, options Options, stdout io.Writer, stderr io.Writer) int { manager := vmstart.NewManagerFromConfig(cfg) createOptions := spindsnapshot.CreateOptions{ - Kind: options.Kind, + K8s: options.K8s, KubeconfigPath: options.Kubeconfig, Context: options.Context, } diff --git a/internal/spind/snapshot/store.go b/internal/spind/snapshot/store.go index 3368359..04473cc 100644 --- a/internal/spind/snapshot/store.go +++ b/internal/spind/snapshot/store.go @@ -177,6 +177,7 @@ func Inspect(snapshotDir string, name string) Info { info.ExecPort = metadata.ExecPort info.KernelCommand = metadata.KernelCommandLine info.KindReady = metadata.KindReady + info.K8sDistribution = metadata.K8sDistribution if metadata.KindReady { if kindMetadata, err := spindkind.ReadMetadata(snapshotDir); err == nil { info.KindMetadata = kindMetadata diff --git a/internal/spind/snapshot/types.go b/internal/spind/snapshot/types.go index b94e71d..f112b7d 100644 --- a/internal/spind/snapshot/types.go +++ b/internal/spind/snapshot/types.go @@ -26,28 +26,30 @@ type Metadata struct { MachineIdentifier string `json:"machineIdentifier,omitempty"` NetworkMAC string `json:"networkMac,omitempty"` KindReady bool `json:"kindReady,omitempty"` + K8sDistribution string `json:"k8sDistribution,omitempty"` Disks []spindimage.DiskMetadata `json:"disks,omitempty"` } type Info struct { - Name string - SourceVM string - Backend string - CreatedAt time.Time - CPUCount int - MemoryMiB int - ExecPort uint32 - KernelCommand string - DiskSizeBytes int64 - StateSizeBytes int64 - TotalSizeBytes int64 - Health string - HealthMessage string - SnapshotDir string - KindReady bool - KindMetadata spindkind.Metadata - KindTemplate bool - Artifacts []ArtifactInfo + Name string + SourceVM string + Backend string + CreatedAt time.Time + CPUCount int + MemoryMiB int + ExecPort uint32 + KernelCommand string + DiskSizeBytes int64 + StateSizeBytes int64 + TotalSizeBytes int64 + Health string + HealthMessage string + SnapshotDir string + KindReady bool + K8sDistribution string + KindMetadata spindkind.Metadata + KindTemplate bool + Artifacts []ArtifactInfo } type ArtifactInfo struct { @@ -58,7 +60,7 @@ type ArtifactInfo struct { } type CreateOptions struct { - Kind bool + K8s string KubeconfigPath string Context string } diff --git a/internal/spind/up/run.go b/internal/spind/up/run.go index a128e9e..cfe7649 100644 --- a/internal/spind/up/run.go +++ b/internal/spind/up/run.go @@ -35,7 +35,7 @@ const ( type projectConfig struct { Name string `yaml:"name"` Image string `yaml:"image"` - Kind bool `yaml:"kind"` + K8s string `yaml:"k8s"` Setup []string `yaml:"setup"` } @@ -44,7 +44,7 @@ type project struct { ConfigPath string Name string Image string - Kind bool + K8s string Setup []string VMName string ProvisioningName string @@ -132,6 +132,7 @@ func run(ctx context.Context, cfg config.Config, options Options, stdout io.Writ if info, err := manager.VMStatus(project.VMName); err == nil { output.PrintDockerStartLine(stdout, stderr, info) output.PrintKubernetesStartLine(stdout, stderr, info) + output.PrintRegistryStartLine(stdout, stderr, info) PrintShellExports(stdout, info, os.Getenv("SHELL")) } return nil @@ -168,6 +169,9 @@ func shellExportValues(info spindvm.Info) []shellExportValue { if info.KubernetesReady && info.KubernetesKubeconfigPath != "" { values = append(values, shellExportValue{Name: "KUBECONFIG", Value: info.KubernetesKubeconfigPath}) } + if info.RegistryReady && info.RegistryURL != "" { + values = append(values, shellExportValue{Name: "REGISTRY", Value: info.RegistryURL}) + } return values } @@ -210,8 +214,8 @@ func provision(ctx context.Context, manager *vmstart.Manager, cfg config.Config, } } - createOptions := spindsnapshot.CreateOptions{Kind: project.Kind} - if project.Kind { + createOptions := spindsnapshot.CreateOptions{K8s: project.K8s} + if project.K8s != "" { createOptions.KubeconfigPath = provisioningKubeconfigPath(cfg, project) } if err := manager.SnapshotCreateWithOptions(ctx, project.SnapshotName, project.ProvisioningName, createOptions); err != nil { @@ -343,12 +347,16 @@ func loadProject(start string) (project, error) { if err := storeName("image", image); err != nil { return project{}, err } + k8s := strings.TrimSpace(cfg.K8s) + if k8s != "" && k8s != "kind" && k8s != "k3d" { + return project{}, fmt.Errorf("k8s %q: supported values are kind or k3d", k8s) + } return project{ Root: root, ConfigPath: configPath, Name: name, Image: image, - Kind: cfg.Kind, + K8s: k8s, Setup: cfg.Setup, VMName: name, ProvisioningName: name + "-provisioning", diff --git a/internal/spind/up/run_test.go b/internal/spind/up/run_test.go index d738eb9..40b5d9c 100644 --- a/internal/spind/up/run_test.go +++ b/internal/spind/up/run_test.go @@ -15,7 +15,7 @@ import ( func TestLoadProjectUsesYAMLNameBeforeBasename(t *testing.T) { root := t.TempDir() - writeFile(t, filepath.Join(root, "spind.yaml"), "name: custom\nimage: docker\nkind: true\nsetup:\n - echo setup\n") + writeFile(t, filepath.Join(root, "spind.yaml"), "name: custom\nimage: docker\nk8s: kind\nsetup:\n - echo setup\n") chdir(t, root) project, err := loadProject(".") @@ -25,7 +25,7 @@ func TestLoadProjectUsesYAMLNameBeforeBasename(t *testing.T) { if project.Name != "custom" || project.VMName != "custom" || project.ProvisioningName != "custom-provisioning" || project.SnapshotName != "custom-provisioned" { t.Fatalf("project names = %#v", project) } - if project.Image != "docker" || !project.Kind || len(project.Setup) != 1 || project.Setup[0] != "echo setup" { + if project.Image != "docker" || project.K8s != "kind" || len(project.Setup) != 1 || project.Setup[0] != "echo setup" { t.Fatalf("project config = %#v", project) } } diff --git a/internal/spind/vm/start/kind_ready.go b/internal/spind/vm/start/kind_ready.go index 30459a4..1dbfbcf 100644 --- a/internal/spind/vm/start/kind_ready.go +++ b/internal/spind/vm/start/kind_ready.go @@ -1,7 +1,9 @@ package start import ( + "bytes" "context" + "encoding/json" "errors" "fmt" "net" @@ -15,6 +17,7 @@ import ( spindkind "github.com/suin/spind/internal/spind/kind" spindsnapshot "github.com/suin/spind/internal/spind/snapshot" spindvm "github.com/suin/spind/internal/spind/vmstore" + "gopkg.in/yaml.v3" ) func kubernetesSupport(metadata spindvm.Metadata) string { @@ -32,10 +35,11 @@ func kubernetesStatus(metadata spindvm.Metadata, state spindvm.State) string { } func prepareKindSnapshot(ctx context.Context, tmpDir string, vmDir string, state spindvm.State, options spindsnapshot.CreateOptions) (*spindkind.Metadata, error) { - if !options.Kind { + if options.K8s == "" { return nil, nil } return spindkind.PrepareSnapshot(ctx, tmpDir, state.DockerSocketPath, state.DockerAvailable && state.DockerAPIReady, spindkind.SnapshotOptions{ + Distribution: options.K8s, KubeconfigPath: options.KubeconfigPath, Context: options.Context, }) @@ -91,6 +95,7 @@ func (m *Manager) configureKubernetesEndpoint(ctx context.Context, name string, return state } state.KubernetesReady = true + state = m.configureRegistryEndpoint(ctx, name, vmDir, metadata, state, kindMetadata) return state } @@ -115,11 +120,19 @@ func allocateKubernetesPort(preferred int) (int, error) { } func startKubernetesRelay(ctx context.Context, name string, vmDir string, metadata spindvm.Metadata, state spindvm.State) (int, error) { + return startTCPRelay(ctx, name, vmDir, metadata, state, state.KubernetesRelayLogPath, state.KubernetesAPIServerPort, state.KubernetesAPIServerTargetPort, "Kubernetes") +} + +func startRegistryRelay(ctx context.Context, name string, vmDir string, metadata spindvm.Metadata, state spindvm.State) (int, error) { + return startTCPRelay(ctx, name, vmDir, metadata, state, state.RegistryRelayLogPath, state.RegistryPort, state.RegistryTargetPort, "registry") +} + +func startTCPRelay(ctx context.Context, name string, vmDir string, metadata spindvm.Metadata, state spindvm.State, logPath string, listenPort int, targetPort int, service string) (int, error) { args := []string{ "kubernetes-relay", name, - "--listen-port", strconv.Itoa(state.KubernetesAPIServerPort), - "--target-port", strconv.Itoa(state.KubernetesAPIServerTargetPort), + "--listen-port", strconv.Itoa(listenPort), + "--target-port", strconv.Itoa(targetPort), "--guest-port", fmt.Sprintf("%d", state.DockerTCPForwardGuestPort), } switch metadata.Backend { @@ -144,9 +157,9 @@ func startKubernetesRelay(ctx context.Context, name string, vmDir string, metada if err != nil { return 0, fmt.Errorf("resolve spind executable: %w", err) } - logFile, err := os.OpenFile(state.KubernetesRelayLogPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) if err != nil { - return 0, fmt.Errorf("open Kubernetes relay log: %w", err) + return 0, fmt.Errorf("open %s relay log: %w", service, err) } defer logFile.Close() cmd := exec.CommandContext(ctx, executable, args...) @@ -154,12 +167,12 @@ func startKubernetesRelay(ctx context.Context, name string, vmDir string, metada cmd.Stderr = logFile cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} if err := cmd.Start(); err != nil { - return 0, fmt.Errorf("start Kubernetes relay: %w", err) + return 0, fmt.Errorf("start %s relay: %w", service, err) } pid := cmd.Process.Pid if err := cmd.Process.Release(); err != nil { _ = signalProcess(pid, syscall.SIGTERM) - return 0, fmt.Errorf("release Kubernetes relay process: %w", err) + return 0, fmt.Errorf("release %s relay process: %w", service, err) } return pid, nil } @@ -178,6 +191,131 @@ func cleanupKubernetesEndpoint(state spindvm.State) { _ = signalProcess(state.KubernetesRelayPID, syscall.SIGTERM) _ = waitForExit(state.KubernetesRelayPID, 2*time.Second) } + if state.RegistryRelayPID != 0 && processAlive(state.RegistryRelayPID) { + _ = signalProcess(state.RegistryRelayPID, syscall.SIGTERM) + _ = waitForExit(state.RegistryRelayPID, 2*time.Second) + } +} + +func (m *Manager) configureRegistryEndpoint(ctx context.Context, name string, vmDir string, metadata spindvm.Metadata, state spindvm.State, kindMetadata spindkind.Metadata) spindvm.State { + if kindMetadata.Distribution != spindkind.DistributionK3d || kindMetadata.RegistryTargetPort == 0 { + return state + } + state.RegistryReady = false + state.RegistryLastError = "" + state.RegistryRelayLogPath = filepath.Join(vmDir, "registry-relay.log") + state.RegistryTargetPort = kindMetadata.RegistryTargetPort + state.RegistryHostFromCluster = kindMetadata.RegistryHostFromCluster + state.RegistryURL = fmt.Sprintf("localhost:%d", state.RegistryTargetPort) + port, err := allocateKubernetesPort(firstPositive(state.RegistryPort, state.RegistryTargetPort)) + if err != nil { + state.RegistryLastError = err.Error() + return state + } + state.RegistryPort = port + pid, err := startRegistryRelay(ctx, name, vmDir, metadata, state) + if err != nil { + state.RegistryLastError = err.Error() + return state + } + state.RegistryRelayPID = pid + if err := waitForTCPPort(ctx, "127.0.0.1", port, 5*time.Second); err != nil { + state.RegistryLastError = err.Error() + return state + } + if err := updateLocalRegistryHosting(ctx, state.KubernetesKubeconfigPath, state.RegistryURL, kindMetadata.RegistryHostFromCluster); err != nil { + state.RegistryLastError = err.Error() + return state + } + state.RegistryLocalHostingUpdated = true + state.RegistryReady = true + return state +} + +func firstPositive(values ...int) int { + for _, value := range values { + if value > 0 { + return value + } + } + return 0 +} + +type configMapDocument struct { + APIVersion string `json:"apiVersion" yaml:"apiVersion"` + Kind string `json:"kind" yaml:"kind"` + Metadata map[string]string `json:"metadata" yaml:"metadata"` + Data map[string]string `json:"data" yaml:"data"` +} + +type localRegistryHostingV1 struct { + Host string `yaml:"host"` + HostFromClusterNetwork string `yaml:"hostFromClusterNetwork,omitempty"` + HostFromContainerRuntime string `yaml:"hostFromContainerRuntime,omitempty"` + Help string `yaml:"help,omitempty"` +} + +func updateLocalRegistryHosting(ctx context.Context, kubeconfigPath string, registryURL string, hostFromCluster string) error { + hosting, err := readLocalRegistryHosting(ctx, kubeconfigPath) + if err != nil { + hosting = localRegistryHostingV1{ + HostFromClusterNetwork: hostFromCluster, + HostFromContainerRuntime: hostFromCluster, + Help: "https://k3d.io/stable/usage/registries/#using-a-local-registry", + } + } + hosting.Host = registryURL + if hosting.HostFromClusterNetwork == "" { + hosting.HostFromClusterNetwork = hostFromCluster + } + if hosting.HostFromContainerRuntime == "" { + hosting.HostFromContainerRuntime = hostFromCluster + } + data, err := yaml.Marshal(hosting) + if err != nil { + return fmt.Errorf("marshal local registry hosting: %w", err) + } + cm := configMapDocument{ + APIVersion: "v1", + Kind: "ConfigMap", + Metadata: map[string]string{ + "name": "local-registry-hosting", + "namespace": "kube-public", + }, + Data: map[string]string{"localRegistryHosting.v1": string(data)}, + } + manifest, err := yaml.Marshal(cm) + if err != nil { + return fmt.Errorf("marshal local registry hosting ConfigMap: %w", err) + } + cmd := exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeconfigPath, "apply", "-f", "-") + cmd.Stdin = bytes.NewReader(manifest) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("apply local registry hosting ConfigMap: %w: %s", err, string(output)) + } + return nil +} + +func readLocalRegistryHosting(ctx context.Context, kubeconfigPath string) (localRegistryHostingV1, error) { + cmd := exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeconfigPath, "--namespace", "kube-public", "get", "configmap", "local-registry-hosting", "-o", "json") + output, err := cmd.CombinedOutput() + if err != nil { + return localRegistryHostingV1{}, fmt.Errorf("read local registry hosting ConfigMap: %w: %s", err, string(output)) + } + var cm configMapDocument + if err := json.Unmarshal(output, &cm); err != nil { + return localRegistryHostingV1{}, fmt.Errorf("parse local registry hosting ConfigMap: %w", err) + } + raw := cm.Data["localRegistryHosting.v1"] + if raw == "" { + return localRegistryHostingV1{}, errors.New("local registry hosting ConfigMap has no localRegistryHosting.v1 data") + } + var hosting localRegistryHostingV1 + if err := yaml.Unmarshal([]byte(raw), &hosting); err != nil { + return localRegistryHostingV1{}, fmt.Errorf("parse localRegistryHosting.v1: %w", err) + } + return hosting, nil } func waitForTCPPort(ctx context.Context, host string, port int, timeout time.Duration) error { diff --git a/internal/spind/vm/start/manager.go b/internal/spind/vm/start/manager.go index c741f67..35cba48 100644 --- a/internal/spind/vm/start/manager.go +++ b/internal/spind/vm/start/manager.go @@ -404,6 +404,7 @@ func (m *Manager) CreateFromSnapshot(ctx context.Context, name string, snapshotN ExecPort: snapshot.ExecPort, RestoreStatePath: snapshotRestoreStatePath(vmDir, snapshot.Backend), KindReady: snapshot.KindReady, + K8sDistribution: snapshot.K8sDistribution, } if snapshot.KindReady { metadata.KubeconfigPath = filepath.Join(vmDir, "kubeconfig") @@ -655,6 +656,15 @@ func (m *Manager) VMStatus(name string) (spindvm.Info, error) { KubernetesAPIServerTargetPort: state.KubernetesAPIServerTargetPort, KubernetesRelayPID: state.KubernetesRelayPID, KubernetesRelayLogPath: state.KubernetesRelayLogPath, + RegistryReady: state.RegistryReady, + RegistryLastError: state.RegistryLastError, + RegistryURL: state.RegistryURL, + RegistryPort: state.RegistryPort, + RegistryTargetPort: state.RegistryTargetPort, + RegistryRelayPID: state.RegistryRelayPID, + RegistryRelayLogPath: state.RegistryRelayLogPath, + RegistryHostFromCluster: state.RegistryHostFromCluster, + RegistryLocalHostingUpdated: state.RegistryLocalHostingUpdated, SerialLogPath: serialLogPath, BackendLogPath: backendLogPath, EventLogPath: eventLogPath, @@ -853,13 +863,14 @@ func vmLogPaths(vmDir string, backend string, state spindvm.State) []string { state.EventLogPath, state.HostShareVirtioFSLogPath, state.KubernetesRelayLogPath, + state.RegistryRelayLogPath, filepath.Join(vmDir, "cloud-hypervisor.log"), filepath.Join(vmDir, "serial.log"), filepath.Join(vmDir, "cloud-hypervisor-event.log"), filepath.Join(vmDir, "virtiofsd.log"), ) default: - candidates = append(candidates, state.KubernetesRelayLogPath, filepath.Join(vmDir, vmLogName)) + candidates = append(candidates, state.KubernetesRelayLogPath, state.RegistryRelayLogPath, filepath.Join(vmDir, vmLogName)) } seen := map[string]bool{} paths := make([]string, 0, len(candidates)) diff --git a/internal/spind/vm/start/run.go b/internal/spind/vm/start/run.go index 0bdf22b..30a668c 100644 --- a/internal/spind/vm/start/run.go +++ b/internal/spind/vm/start/run.go @@ -41,6 +41,7 @@ func Run(ctx context.Context, cfg config.Config, options Options, stdout io.Writ output.PrintHostShareStartLine(stdout, stderr, info) output.PrintDockerStartLine(stdout, stderr, info) output.PrintKubernetesStartLine(stdout, stderr, info) + output.PrintRegistryStartLine(stdout, stderr, info) } return 0 } diff --git a/internal/spind/vm/start/snapshot.go b/internal/spind/vm/start/snapshot.go index c417ee5..7d5b2c5 100644 --- a/internal/spind/vm/start/snapshot.go +++ b/internal/spind/vm/start/snapshot.go @@ -31,6 +31,9 @@ func (m *Manager) SnapshotCreateWithOptions(ctx context.Context, snapshotName st if err := validateStoreName(vmName); err != nil { return fmt.Errorf("VM %q: %w", vmName, err) } + if options.K8s != "" && options.K8s != spindkind.DistributionKind && options.K8s != spindkind.DistributionK3d { + return fmt.Errorf("unsupported Kubernetes distribution %q", options.K8s) + } vmDir := filepath.Join(m.VMStore, vmName) var metadata spindvm.Metadata @@ -163,6 +166,7 @@ func (m *Manager) createCloudHypervisorSnapshot(ctx context.Context, snapshotNam } if kindMetadata != nil { snapshot.KindReady = true + snapshot.K8sDistribution = kindMetadata.Distribution } if err := writeJSON(filepath.Join(tmpDir, snapshotMetadataName), snapshot, 0o644); err != nil { return fmt.Errorf("write snapshot metadata: %w", err) @@ -247,6 +251,7 @@ func (m *Manager) createVirtualizationFrameworkSnapshot(ctx context.Context, sna snapshot.NetworkMAC = config.NetworkMAC if kindMetadata != nil { snapshot.KindReady = true + snapshot.K8sDistribution = kindMetadata.Distribution } if err := writeJSON(filepath.Join(tmpDir, snapshotMetadataName), snapshot, 0o644); err != nil { return fmt.Errorf("write snapshot metadata: %w", err) diff --git a/internal/spind/vm/status/run.go b/internal/spind/vm/status/run.go index 7110a89..9f93482 100644 --- a/internal/spind/vm/status/run.go +++ b/internal/spind/vm/status/run.go @@ -141,6 +141,15 @@ func Info(cfg config.Config, name string) (vmstore.Info, error) { KubernetesAPIServerTargetPort: state.KubernetesAPIServerTargetPort, KubernetesRelayPID: state.KubernetesRelayPID, KubernetesRelayLogPath: state.KubernetesRelayLogPath, + RegistryReady: state.RegistryReady, + RegistryLastError: state.RegistryLastError, + RegistryURL: state.RegistryURL, + RegistryPort: state.RegistryPort, + RegistryTargetPort: state.RegistryTargetPort, + RegistryRelayPID: state.RegistryRelayPID, + RegistryRelayLogPath: state.RegistryRelayLogPath, + RegistryHostFromCluster: state.RegistryHostFromCluster, + RegistryLocalHostingUpdated: state.RegistryLocalHostingUpdated, SerialLogPath: serialLogPath, BackendLogPath: backendLogPath, EventLogPath: eventLogPath, @@ -233,13 +242,14 @@ func logPaths(vmDir string, backend string, state vmstore.State) []string { state.EventLogPath, state.HostShareVirtioFSLogPath, state.KubernetesRelayLogPath, + state.RegistryRelayLogPath, filepath.Join(vmDir, "cloud-hypervisor.log"), filepath.Join(vmDir, "serial.log"), filepath.Join(vmDir, "cloud-hypervisor-event.log"), filepath.Join(vmDir, "virtiofsd.log"), ) default: - candidates = append(candidates, state.KubernetesRelayLogPath, filepath.Join(vmDir, vmstore.LogName)) + candidates = append(candidates, state.KubernetesRelayLogPath, state.RegistryRelayLogPath, filepath.Join(vmDir, vmstore.LogName)) } seen := map[string]bool{} paths := make([]string, 0, len(candidates)) diff --git a/internal/spind/vmstore/types.go b/internal/spind/vmstore/types.go index b542861..0871cd7 100644 --- a/internal/spind/vmstore/types.go +++ b/internal/spind/vmstore/types.go @@ -30,6 +30,7 @@ type Metadata struct { MemoryMiB int `json:"memoryMiB,omitempty"` ExecPort uint32 `json:"execPort,omitempty"` KindReady bool `json:"kindReady,omitempty"` + K8sDistribution string `json:"k8sDistribution,omitempty"` KubeconfigPath string `json:"kubeconfigPath,omitempty"` } @@ -91,6 +92,15 @@ type State struct { KubernetesAPIServerTargetPort int `json:"kubernetesApiServerTargetPort,omitempty"` KubernetesRelayPID int `json:"kubernetesRelayPid,omitempty"` KubernetesRelayLogPath string `json:"kubernetesRelayLogPath,omitempty"` + RegistryReady bool `json:"registryReady,omitempty"` + RegistryLastError string `json:"registryLastError,omitempty"` + RegistryURL string `json:"registryUrl,omitempty"` + RegistryPort int `json:"registryPort,omitempty"` + RegistryTargetPort int `json:"registryTargetPort,omitempty"` + RegistryRelayPID int `json:"registryRelayPid,omitempty"` + RegistryRelayLogPath string `json:"registryRelayLogPath,omitempty"` + RegistryHostFromCluster string `json:"registryHostFromCluster,omitempty"` + RegistryLocalHostingUpdated bool `json:"registryLocalHostingUpdated,omitempty"` StartedAt time.Time `json:"startedAt,omitempty"` LastStartDurationMS int64 `json:"lastStartDurationMs,omitempty"` UpdatedAt time.Time `json:"updatedAt"` @@ -166,6 +176,15 @@ type Info struct { KubernetesAPIServerTargetPort int KubernetesRelayPID int KubernetesRelayLogPath string + RegistryReady bool + RegistryLastError string + RegistryURL string + RegistryPort int + RegistryTargetPort int + RegistryRelayPID int + RegistryRelayLogPath string + RegistryHostFromCluster string + RegistryLocalHostingUpdated bool SerialLogPath string BackendLogPath string EventLogPath string diff --git a/spind.yaml b/spind.yaml index 1357050..b949a49 100644 --- a/spind.yaml +++ b/spind.yaml @@ -1,6 +1,6 @@ name: sample image: docker -kind: true +k8s: kind setup: - "kind create cluster" - 'api_server_port="$(docker port kind-control-plane 6443/tcp | awk -F: ''END { print $NF }'')" && kubectl config set-cluster kind-kind --server "https://127.0.0.1:$api_server_port"' diff --git a/tests/e2e/helpers.ts b/tests/e2e/helpers.ts index 4d9220e..4693955 100644 --- a/tests/e2e/helpers.ts +++ b/tests/e2e/helpers.ts @@ -14,10 +14,12 @@ const createVerbose$ = create$({ verbose: e2eEnv.execaVerbose }); const CLOUD_HYPERVISOR_BACKEND = "cloud-hypervisor"; export const DOCKER_HOST_PREREQUISITES = "docker-host"; export const KIND_READY_PREREQUISITES = "kind-ready"; +export const K3D_READY_PREREQUISITES = "k3d-ready"; export type E2EPrerequisites = | typeof DOCKER_HOST_PREREQUISITES | typeof KIND_READY_PREREQUISITES + | typeof K3D_READY_PREREQUISITES | "vm"; export class E2EEnv { @@ -180,7 +182,8 @@ async function requireE2EPrerequisites( if ( prerequisites === DOCKER_HOST_PREREQUISITES || - prerequisites === KIND_READY_PREREQUISITES + prerequisites === KIND_READY_PREREQUISITES || + prerequisites === K3D_READY_PREREQUISITES ) { await requireCommand("docker"); } @@ -188,6 +191,10 @@ async function requireE2EPrerequisites( await requireCommand("kind"); await requireCommand("kubectl"); } + if (prerequisites === K3D_READY_PREREQUISITES) { + await requireCommand("k3d"); + await requireCommand("kubectl"); + } } function detectBackend(): string { diff --git a/tests/e2e/k3d-registry.test.ts b/tests/e2e/k3d-registry.test.ts new file mode 100644 index 0000000..55519f8 --- /dev/null +++ b/tests/e2e/k3d-registry.test.ts @@ -0,0 +1,74 @@ +import { expect, setDefaultTimeout, test } from "bun:test"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { K3D_READY_PREREQUISITES, createE2EEnv } from "./helpers"; + +setDefaultTimeout(45 * 60_000); + +test("k3d registry snapshot restores a pullable registry", async () => { + await using env = await createE2EEnv(K3D_READY_PREREQUISITES); + const projectName = `e2e-k3d-${env.id}`; + const registryName = `${projectName}-registry`; + const imageName = "spind-e2e"; + const imageTag = "dev"; + const projectRoot = path.join(env.root, "project"); + await mkdir(projectRoot, { recursive: true }); + await writeFile( + path.join(projectRoot, "spind.yaml"), + [ + `name: ${projectName}`, + "image: docker", + "k8s: k3d", + "setup:", + yamlListItem( + `k3d cluster create ${projectName} --registry-create ${registryName} --kubeconfig-update-default=false --timeout 180s`, + ), + yamlListItem(`k3d kubeconfig get ${projectName} > "$KUBECONFIG"`), + yamlListItem( + `kubectl --context k3d-${projectName} wait node --all --for=condition=Ready --timeout=180s`, + ), + yamlListItem( + `registry_port="$(docker port ${registryName} 5000/tcp | awk -F: 'END { print $NF }')" && docker pull busybox:latest && docker tag busybox:latest "localhost:$registry_port/${imageName}:${imageTag}" && docker push "localhost:$registry_port/${imageName}:${imageTag}"`, + ), + "", + ].join("\n"), + ); + + const $ = env.project$(projectRoot); + const cleanup$ = env.cleanup$(); + try { + await $`spind up`; + + const vmList = await $`spind vm list ${projectName} --json`; + const vm = JSON.parse(vmList.stdout) as { + dockerEndpointUri?: string; + kubernetes?: { ready?: string; kubeconfig?: string }; + registry?: { ready?: boolean; url?: string }; + }; + expect(typeof vm.dockerEndpointUri).toBe("string"); + expect(vm.registry?.ready).toBe(true); + expect(vm.registry?.url).toMatch(/^localhost:\d+$/u); + + const registryURL = vm.registry?.url; + if (registryURL === undefined) { + throw new Error( + `missing registry or Docker endpoint in VM info: ${vmList.stdout}`, + ); + } + + const docker$ = env.docker$(projectName); + await docker$`docker pull ${registryURL}/${imageName}:${imageTag}`; + + const hosting = + await $`kubectl --kubeconfig ${path.join(env.spindHome, "vms", projectName, "kubeconfig")} --namespace kube-public get configmap local-registry-hosting -o jsonpath={.data.localRegistryHosting\\.v1}`; + expect(hosting.stdout).toContain(`host: ${registryURL}`); + } finally { + await cleanup$`spind vm delete ${projectName} --force`; + await cleanup$`spind vm delete ${projectName}-provisioning --force`; + await cleanup$`spind snapshot delete ${projectName}-provisioned`; + } +}); + +function yamlListItem(value: string): string { + return ` - ${JSON.stringify(value)}`; +} diff --git a/tests/e2e/kind-ready.test.ts b/tests/e2e/kind-ready.test.ts index 88709f0..1702a9b 100644 --- a/tests/e2e/kind-ready.test.ts +++ b/tests/e2e/kind-ready.test.ts @@ -38,7 +38,7 @@ test("kind-ready snapshot restores a VM with Ready nodes", async () => { await $`kubectl --kubeconfig ${kubeconfig} --context ${context} wait node --all --for=condition=Ready --timeout=180s`; // Create snapshot - await $`spind snapshot create ${snapshot} --vm ${baseVM} --kind`; + await $`spind snapshot create ${snapshot} --vm ${baseVM} --k8s=kind`; // Create VM from snapshot await $`spind vm create ${workVM} --snapshot ${snapshot}`; diff --git a/tests/e2e/up.test.ts b/tests/e2e/up.test.ts index baac669..1da1774 100644 --- a/tests/e2e/up.test.ts +++ b/tests/e2e/up.test.ts @@ -16,7 +16,7 @@ test("spind up provisions a project snapshot and starts the project VM", async ( [ `name: ${projectName}`, "image: docker", - "kind: true", + "k8s: kind", "setup:", yamlListItem(`kind create cluster --name ${projectName}`), yamlListItem( From e962eb93c3522696dbe4fad8467537fd6564b628 Mon Sep 17 00:00:00 2001 From: suin Date: Fri, 19 Jun 2026 06:31:54 +0900 Subject: [PATCH 2/6] Allow Tilt sample context --- README.md | 2 ++ examples/tilt/Tiltfile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 3b03561..4ecb77d 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,8 @@ Tilt can auto-detect a k3d local registry through the `kube-public/local-registr A minimal Tiltfile looks like this. ```python +allow_k8s_contexts("spind-tilt-sample") + docker_build("tilt-sample", ".") k8s_yaml("k8s.yaml") diff --git a/examples/tilt/Tiltfile b/examples/tilt/Tiltfile index 1c0421a..98f334b 100644 --- a/examples/tilt/Tiltfile +++ b/examples/tilt/Tiltfile @@ -1,3 +1,5 @@ +allow_k8s_contexts("spind-tilt-sample") + docker_build("tilt-sample", ".") k8s_yaml("k8s.yaml") From de6194a5359eb3be119bab328c1b2ff09804f223 Mon Sep 17 00:00:00 2001 From: suin Date: Fri, 19 Jun 2026 06:40:08 +0900 Subject: [PATCH 3/6] Use direct guest IP for k3d relays --- README.md | 2 +- examples/tilt/Tiltfile | 2 +- internal/spind/vm/start/kind_ready.go | 69 ++++++++++++++----------- internal/spind/vm/start/manager_test.go | 53 +++++++++++++++++++ 4 files changed, 95 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 4ecb77d..1fad1b8 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ docker_build("tilt-sample", ".") k8s_yaml("k8s.yaml") -k8s_resource("tilt-sample", port_forwards=8080) +k8s_resource("tilt-sample", port_forwards="8080:80") ``` Run Tilt after the spind environment is ready. diff --git a/examples/tilt/Tiltfile b/examples/tilt/Tiltfile index 98f334b..7300a61 100644 --- a/examples/tilt/Tiltfile +++ b/examples/tilt/Tiltfile @@ -4,4 +4,4 @@ docker_build("tilt-sample", ".") k8s_yaml("k8s.yaml") -k8s_resource("tilt-sample", port_forwards=8080) +k8s_resource("tilt-sample", port_forwards="8080:80") diff --git a/internal/spind/vm/start/kind_ready.go b/internal/spind/vm/start/kind_ready.go index 1dbfbcf..e4b3840 100644 --- a/internal/spind/vm/start/kind_ready.go +++ b/internal/spind/vm/start/kind_ready.go @@ -80,7 +80,7 @@ func (m *Manager) configureKubernetesEndpoint(ctx context.Context, name string, state.KubernetesLastError = err.Error() return state } - pid, err := startKubernetesRelay(ctx, name, vmDir, metadata, state) + pid, err := startKubernetesRelay(ctx, name, vmDir, metadata, state, kindMetadata) if err != nil { state.KubernetesLastError = err.Error() return state @@ -119,39 +119,18 @@ func allocateKubernetesPort(preferred int) (int, error) { return listener.Addr().(*net.TCPAddr).Port, nil } -func startKubernetesRelay(ctx context.Context, name string, vmDir string, metadata spindvm.Metadata, state spindvm.State) (int, error) { - return startTCPRelay(ctx, name, vmDir, metadata, state, state.KubernetesRelayLogPath, state.KubernetesAPIServerPort, state.KubernetesAPIServerTargetPort, "Kubernetes") +func startKubernetesRelay(ctx context.Context, name string, vmDir string, metadata spindvm.Metadata, state spindvm.State, kindMetadata spindkind.Metadata) (int, error) { + return startTCPRelay(ctx, name, vmDir, metadata, state, state.KubernetesRelayLogPath, state.KubernetesAPIServerPort, state.KubernetesAPIServerTargetPort, "Kubernetes", kindMetadata.Distribution == spindkind.DistributionK3d) } func startRegistryRelay(ctx context.Context, name string, vmDir string, metadata spindvm.Metadata, state spindvm.State) (int, error) { - return startTCPRelay(ctx, name, vmDir, metadata, state, state.RegistryRelayLogPath, state.RegistryPort, state.RegistryTargetPort, "registry") + return startTCPRelay(ctx, name, vmDir, metadata, state, state.RegistryRelayLogPath, state.RegistryPort, state.RegistryTargetPort, "registry", true) } -func startTCPRelay(ctx context.Context, name string, vmDir string, metadata spindvm.Metadata, state spindvm.State, logPath string, listenPort int, targetPort int, service string) (int, error) { - args := []string{ - "kubernetes-relay", - name, - "--listen-port", strconv.Itoa(listenPort), - "--target-port", strconv.Itoa(targetPort), - "--guest-port", fmt.Sprintf("%d", state.DockerTCPForwardGuestPort), - } - switch metadata.Backend { - case BackendVirtualizationFramework: - if state.ExecSocketPath == "" { - return 0, errors.New("exec socket path is missing") - } - args = append(args, - "--ssh-socket", state.ExecSocketPath, - "--ssh-key", filepath.Join(vmDir, vmSSHPrivateKeyName), - "--ssh-user", metadata.ExecUser, - ) - case BackendCloudHypervisor: - if state.CloudHypervisorVsockSocketPath == "" { - return 0, errors.New("Cloud Hypervisor vsock socket path is missing") - } - args = append(args, "--vsock", state.CloudHypervisorVsockSocketPath) - default: - return 0, fmt.Errorf("unsupported backend %q", metadata.Backend) +func startTCPRelay(ctx context.Context, name string, vmDir string, metadata spindvm.Metadata, state spindvm.State, logPath string, listenPort int, targetPort int, service string, preferGuestIP bool) (int, error) { + args, err := tcpRelayArgs(name, vmDir, metadata, state, listenPort, targetPort, preferGuestIP) + if err != nil { + return 0, err } executable, err := os.Executable() if err != nil { @@ -177,6 +156,38 @@ func startTCPRelay(ctx context.Context, name string, vmDir string, metadata spin return pid, nil } +func tcpRelayArgs(name string, vmDir string, metadata spindvm.Metadata, state spindvm.State, listenPort int, targetPort int, preferGuestIP bool) ([]string, error) { + args := []string{ + "kubernetes-relay", + name, + "--listen-port", strconv.Itoa(listenPort), + "--target-port", strconv.Itoa(targetPort), + "--guest-port", fmt.Sprintf("%d", state.DockerTCPForwardGuestPort), + } + switch metadata.Backend { + case BackendVirtualizationFramework: + if preferGuestIP && state.DockerGuestIPAddress != "" { + return append(args, "--guest-ip", state.DockerGuestIPAddress), nil + } + if state.ExecSocketPath == "" { + return nil, errors.New("exec socket path is missing") + } + args = append(args, + "--ssh-socket", state.ExecSocketPath, + "--ssh-key", filepath.Join(vmDir, vmSSHPrivateKeyName), + "--ssh-user", metadata.ExecUser, + ) + case BackendCloudHypervisor: + if state.CloudHypervisorVsockSocketPath == "" { + return nil, errors.New("Cloud Hypervisor vsock socket path is missing") + } + args = append(args, "--vsock", state.CloudHypervisorVsockSocketPath) + default: + return nil, fmt.Errorf("unsupported backend %q", metadata.Backend) + } + return args, nil +} + func kubectlCheckGeneratedKubeconfig(ctx context.Context, kubeconfigPath string) error { cmd := exec.CommandContext(ctx, "kubectl", "--kubeconfig", kubeconfigPath, "get", "nodes") output, err := cmd.CombinedOutput() diff --git a/internal/spind/vm/start/manager_test.go b/internal/spind/vm/start/manager_test.go index 91ab971..98ee546 100644 --- a/internal/spind/vm/start/manager_test.go +++ b/internal/spind/vm/start/manager_test.go @@ -17,6 +17,7 @@ import ( "os/signal" "path/filepath" "runtime" + "slices" "strconv" "strings" "syscall" @@ -1862,6 +1863,58 @@ func TestUnknownVMFails(t *testing.T) { } } +func TestTCPRelayArgsUsesGuestIPWhenPreferred(t *testing.T) { + vmDir := t.TempDir() + args, err := tcpRelayArgs("work", vmDir, spindvm.Metadata{ + Backend: BackendVirtualizationFramework, + ExecUser: "spind", + }, spindvm.State{ + ExecSocketPath: filepath.Join(vmDir, "exec.sock"), + DockerGuestIPAddress: "192.168.64.202", + }, 49321, 40123, true) + if err != nil { + t.Fatal(err) + } + want := []string{ + "kubernetes-relay", + "work", + "--listen-port", "49321", + "--target-port", "40123", + "--guest-port", "0", + "--guest-ip", "192.168.64.202", + } + if !slices.Equal(args, want) { + t.Fatalf("args = %#v, want %#v", args, want) + } +} + +func TestTCPRelayArgsKeepsSSHWhenGuestIPIsNotPreferred(t *testing.T) { + vmDir := t.TempDir() + args, err := tcpRelayArgs("work", vmDir, spindvm.Metadata{ + Backend: BackendVirtualizationFramework, + ExecUser: "spind", + }, spindvm.State{ + ExecSocketPath: filepath.Join(vmDir, "exec.sock"), + DockerGuestIPAddress: "192.168.64.202", + }, 49321, 40123, false) + if err != nil { + t.Fatal(err) + } + want := []string{ + "kubernetes-relay", + "work", + "--listen-port", "49321", + "--target-port", "40123", + "--guest-port", "0", + "--ssh-socket", filepath.Join(vmDir, "exec.sock"), + "--ssh-key", filepath.Join(vmDir, vmSSHPrivateKeyName), + "--ssh-user", "spind", + } + if !slices.Equal(args, want) { + t.Fatalf("args = %#v, want %#v", args, want) + } +} + func TestFakeRunnerProcess(t *testing.T) { if os.Getenv("SPIND_FAKE_RUNNER") != "1" { return From 0bc8c1616a3539f1c9c448c4bb52fccd5ede3ce3 Mon Sep 17 00:00:00 2001 From: suin Date: Fri, 19 Jun 2026 10:02:19 +0900 Subject: [PATCH 4/6] Fix DNS after Docker snapshot restore --- internal/spind/vm/start/cloud_hypervisor.go | 9 ++++- internal/spind/vm/start/manager_test.go | 37 +++++++++++++++++++++ tests/e2e/docker-host.test.ts | 21 ++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/internal/spind/vm/start/cloud_hypervisor.go b/internal/spind/vm/start/cloud_hypervisor.go index 2c7b2c8..aaf2651 100644 --- a/internal/spind/vm/start/cloud_hypervisor.go +++ b/internal/spind/vm/start/cloud_hypervisor.go @@ -280,7 +280,7 @@ func (m *Manager) startCloudHypervisorRestore(ctx context.Context, name string, } args := []string{ - "--restore", fmt.Sprintf("source_url=file://%s,memory_restore_mode=ondemand", restoreDir), + "--restore", fmt.Sprintf("source_url=file://%s,memory_restore_mode=%s", restoreDir, cloudHypervisorMemoryRestoreMode(config)), "--api-socket", fmt.Sprintf("path=%s", config.APISocketPath), "--log-file", config.VMMLogPath, "--event-monitor", fmt.Sprintf("path=%s", config.EventLogPath), @@ -522,6 +522,13 @@ func cloudHypervisorRequiresSharedMemory(config cloudhypervisor.Config) bool { return config.NetBackend == cloudHypervisorNetworkBackendPasst && config.NetSocketPath != "" } +func cloudHypervisorMemoryRestoreMode(config cloudhypervisor.Config) string { + if config.NetBackend == cloudHypervisorNetworkBackendPasst && config.NetSocketPath != "" { + return "copy" + } + return "ondemand" +} + func disableCloudHypervisorDockerNetwork(config *cloudhypervisor.Config) { config.NetBackend = "" config.NetSocketPath = "" diff --git a/internal/spind/vm/start/manager_test.go b/internal/spind/vm/start/manager_test.go index 98ee546..f9a712e 100644 --- a/internal/spind/vm/start/manager_test.go +++ b/internal/spind/vm/start/manager_test.go @@ -225,6 +225,43 @@ func TestCloudHypervisorArgsIncludeDockerPasstNetwork(t *testing.T) { } } +func TestCloudHypervisorMemoryRestoreMode(t *testing.T) { + tests := []struct { + name string + config cloudhypervisor.Config + want string + }{ + { + name: "passt vhost-user", + config: cloudhypervisor.Config{ + NetBackend: cloudHypervisorNetworkBackendPasst, + NetSocketPath: "passt.sock", + }, + want: "copy", + }, + { + name: "tap", + config: cloudhypervisor.Config{ + NetBackend: cloudHypervisorNetworkBackendTap, + NetTapName: "tap0", + }, + want: "ondemand", + }, + { + name: "no network", + want: "ondemand", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := cloudHypervisorMemoryRestoreMode(tt.config); got != tt.want { + t.Fatalf("cloudHypervisorMemoryRestoreMode() = %q, want %q", got, tt.want) + } + }) + } +} + func TestCreateVirtualizationFrameworkWritesMultipleDisks(t *testing.T) { manager := newTestManager(t) imageDir := filepath.Join(manager.ImageStore, "docker") diff --git a/tests/e2e/docker-host.test.ts b/tests/e2e/docker-host.test.ts index 23dc381..5e1e146 100644 --- a/tests/e2e/docker-host.test.ts +++ b/tests/e2e/docker-host.test.ts @@ -26,6 +26,27 @@ test("Docker host endpoint works", async () => { } }); +test("Docker host snapshot restore keeps DNS usable", async () => { + await using env = await createE2EEnv("docker-host"); + const { $, baseVM, snapshot, workVM } = env; + + await $`spind vm create ${baseVM} --image docker`; + await $`spind vm start ${baseVM}`; + + const baseDocker = env.docker$(baseVM); + await dockerPull(baseDocker, "busybox:latest"); + await $`spind vm exec ${baseVM} -- getent hosts example.com`; + await baseDocker`docker run --rm busybox nslookup example.com`; + + await $`spind snapshot create ${snapshot} --vm ${baseVM}`; + await $`spind vm create ${workVM} --snapshot ${snapshot}`; + await $`spind vm start ${workVM}`; + + const restoredDocker = env.docker$(workVM); + await $`spind vm exec ${workVM} -- getent hosts example.com`; + await restoredDocker`docker run --rm busybox nslookup example.com`; +}); + type Env = Awaited>; async function checkDockerHost({ From a26c84f856d1df636cf852d758935001fd933404 Mon Sep 17 00:00:00 2001 From: suin Date: Fri, 19 Jun 2026 10:30:46 +0900 Subject: [PATCH 5/6] Add manual macOS VZ test declaration --- .github/PULL_REQUEST_TEMPLATE.md | 25 +++++++++++++++++++++++++ .github/workflows/ci.yml | 9 --------- 2 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..f5fa29f --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,25 @@ +## Summary + + + +## Checks + +- [ ] `task ci` + +## macOS VZ test + +GitHub-hosted macOS runner では VZ の実テストを安定して実行できません。 +必要な場合は、物理 Apple Silicon Mac 上で手動確認してください。 + +- [ ] Not run +- [ ] Run on physical Apple Silicon macOS + +Command: + +```sh +task ci:macos +``` + +Result / notes: + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f5389f..f9e787d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,12 +14,3 @@ jobs: with: enable-cache: "true" - run: devbox run -- task ci - - ci-macos: - runs-on: macos-15-intel - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: jetify-com/devbox-install-action@8c6a66ed6273138b1915457069de78cb52fe3bd7 # v0.15.0 - with: - enable-cache: "true" - - run: devbox run -- task ci:macos From 636ed185dc1be78294c669a7af242c197ab85210 Mon Sep 17 00:00:00 2001 From: suin Date: Fri, 19 Jun 2026 10:35:16 +0900 Subject: [PATCH 6/6] Format PR template --- .github/PULL_REQUEST_TEMPLATE.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f5fa29f..ba9305c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,8 +8,7 @@ ## macOS VZ test -GitHub-hosted macOS runner では VZ の実テストを安定して実行できません。 -必要な場合は、物理 Apple Silicon Mac 上で手動確認してください。 +GitHub-hosted macOS runner では VZ の実テストを安定して実行できません。必要な場合は、物理 Apple Silicon Mac 上で手動確認してください。 - [ ] Not run - [ ] Run on physical Apple Silicon macOS