dev-pod-api-build/internal/k8s/pods_test.go
2026-04-16 04:16:36 +00:00

592 lines
17 KiB
Go

package k8s
import (
"context"
"errors"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
fakedynamic "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/kubernetes/fake"
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
)
// newTestClient creates a test client with fake k8s and dynamic clients,
// and pre-seeds the VPN gateway secret.
func newTestClient() *Client {
fakeClient := fake.NewSimpleClientset()
scheme := runtime.NewScheme()
fakeDyn := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme,
map[schema.GroupVersionResource]string{
traefikIngressRouteGVR: "IngressRouteList",
traefikMiddlewareGVR: "MiddlewareList",
},
)
client := NewClientWithClientset(fakeClient, fakeDyn, testCfg)
// Seed VPN gateway secret
ctx := context.Background()
vpnSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: testCfg.VPNGatewaySecret,
Namespace: testCfg.VPNGatewayNS,
},
Data: map[string][]byte{
"VPN_GATEWAY_KEY": []byte("test-vpn-key-123"),
},
}
_, _ = fakeClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: testCfg.VPNGatewayNS},
}, metav1.CreateOptions{})
_, _ = fakeClient.CoreV1().Secrets(testCfg.VPNGatewayNS).Create(ctx, vpnSecret, metav1.CreateOptions{})
return client
}
func defaultCreateOpts() CreatePodOpts {
return CreatePodOpts{
User: "alice",
Pod: "main",
Tools: "go@1.25,rust@1.94",
Task: "build a web server",
CPUReq: "2",
CPULimit: "4",
MemReq: "4Gi",
MemLimit: "8Gi",
MaxConcurrentPods: 3,
}
}
func TestFetchVPNKey(t *testing.T) {
t.Run("success", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
key, err := client.FetchVPNKey(ctx)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if key != "test-vpn-key-123" {
t.Errorf("expected test-vpn-key-123, got %s", key)
}
})
t.Run("secret_not_found", func(t *testing.T) {
fakeClient := fake.NewSimpleClientset()
fakeDyn := fakedynamic.NewSimpleDynamicClient(runtime.NewScheme())
client := NewClientWithClientset(fakeClient, fakeDyn, testCfg)
ctx := context.Background()
_, err := client.FetchVPNKey(ctx)
if err == nil {
t.Fatal("expected error for missing secret")
}
})
t.Run("key_field_missing", func(t *testing.T) {
fakeClient := fake.NewSimpleClientset()
fakeDyn := fakedynamic.NewSimpleDynamicClient(runtime.NewScheme())
client := NewClientWithClientset(fakeClient, fakeDyn, testCfg)
ctx := context.Background()
// Create secret without vpn-key field
_, _ = fakeClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: testCfg.VPNGatewayNS},
}, metav1.CreateOptions{})
_, _ = fakeClient.CoreV1().Secrets(testCfg.VPNGatewayNS).Create(ctx, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: testCfg.VPNGatewaySecret,
Namespace: testCfg.VPNGatewayNS,
},
Data: map[string][]byte{"other-key": []byte("value")},
}, metav1.CreateOptions{})
_, err := client.FetchVPNKey(ctx)
if err == nil {
t.Fatal("expected error for missing vpn-key field")
}
})
}
func TestCreatePod(t *testing.T) {
t.Run("success", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
result, err := client.CreatePod(ctx, opts)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.User != "alice" {
t.Errorf("expected user alice, got %s", result.User)
}
if result.Name != "main" {
t.Errorf("expected pod name main, got %s", result.Name)
}
if result.Status != "Pending" {
t.Errorf("expected status Pending, got %s", result.Status)
}
if result.URL != "https://spinoff.dev/@alice/main/" {
t.Errorf("expected URL https://spinoff.dev/@alice/main/, got %s", result.URL)
}
if result.Tools != "go@1.25,rust@1.94" {
t.Errorf("expected tools go@1.25,rust@1.94, got %s", result.Tools)
}
// Verify k8s resources were created
ns := model.NamespaceName("alice")
// Namespace
_, err = client.Clientset.CoreV1().Namespaces().Get(ctx, ns, metav1.GetOptions{})
if err != nil {
t.Errorf("namespace not created: %v", err)
}
// Pod
_, err = client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-main", metav1.GetOptions{})
if err != nil {
t.Errorf("pod not created: %v", err)
}
// Service
_, err = client.Clientset.CoreV1().Services(ns).Get(ctx, "dev-pod-main-svc", metav1.GetOptions{})
if err != nil {
t.Errorf("service not created: %v", err)
}
// Per-pod PVC
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
if err != nil {
t.Errorf("per-pod pvc not created: %v", err)
}
// Secrets
_, err = client.Clientset.CoreV1().Secrets(ns).Get(ctx, "dev-secrets", metav1.GetOptions{})
if err != nil {
t.Errorf("dev-secrets not created: %v", err)
}
_, err = client.Clientset.CoreV1().Secrets(ns).Get(ctx, "ai-proxy-secrets", metav1.GetOptions{})
if err != nil {
t.Errorf("ai-proxy-secrets not created: %v", err)
}
// ConfigMap
_, err = client.Clientset.CoreV1().ConfigMaps(ns).Get(ctx, "ai-proxy-config", metav1.GetOptions{})
if err != nil {
t.Errorf("ai-proxy-config not created: %v", err)
}
// NetworkPolicy
_, err = client.Clientset.NetworkingV1().NetworkPolicies(ns).Get(ctx, "dev-pod-network-policy", metav1.GetOptions{})
if err != nil {
t.Errorf("network policy not created: %v", err)
}
// IngressRoute (via dynamic client)
_, err = client.Dynamic.Resource(traefikIngressRouteGVR).Namespace(ns).Get(ctx, "dev-pod-main-ingress", metav1.GetOptions{})
if err != nil {
t.Errorf("ingress route not created: %v", err)
}
// Middlewares
_, err = client.Dynamic.Resource(traefikMiddlewareGVR).Namespace(ns).Get(ctx, "spinoff-basic-auth", metav1.GetOptions{})
if err != nil {
t.Errorf("basic-auth middleware not created: %v", err)
}
_, err = client.Dynamic.Resource(traefikMiddlewareGVR).Namespace(ns).Get(ctx, "strip-dev-main-ralphex-prefix", metav1.GetOptions{})
if err != nil {
t.Errorf("strip-prefix middleware not created: %v", err)
}
})
t.Run("quota_exceeded", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
// Create 2 pods to fill quota of 2
opts := defaultCreateOpts()
opts.MaxConcurrentPods = 2
opts.Pod = "pod1"
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("first pod failed: %v", err)
}
opts.Pod = "pod2"
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("second pod failed: %v", err)
}
// Third pod should fail
opts.Pod = "pod3"
_, err := client.CreatePod(ctx, opts)
if !errors.Is(err, ErrQuotaExceeded) {
t.Errorf("expected ErrQuotaExceeded, got: %v", err)
}
})
t.Run("duplicate_pod", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("first create failed: %v", err)
}
_, err := client.CreatePod(ctx, opts)
if !errors.Is(err, ErrPodAlreadyExists) {
t.Errorf("expected ErrPodAlreadyExists, got: %v", err)
}
})
t.Run("shared_resources_idempotent", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
// Create first pod
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("first pod failed: %v", err)
}
// Create second pod for same user
opts.Pod = "secondary"
result, err := client.CreatePod(ctx, opts)
if err != nil {
t.Fatalf("second pod failed: %v", err)
}
if result.Name != "secondary" {
t.Errorf("expected pod name secondary, got %s", result.Name)
}
// Both pods should exist
ns := model.NamespaceName("alice")
pods, err := client.Clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{})
if err != nil {
t.Fatalf("failed to list pods: %v", err)
}
if len(pods.Items) != 2 {
t.Errorf("expected 2 pods, got %d", len(pods.Items))
}
// Each pod should have its own PVC
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
if err != nil {
t.Errorf("workspace-main PVC not created: %v", err)
}
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-secondary", metav1.GetOptions{})
if err != nil {
t.Errorf("workspace-secondary PVC not created: %v", err)
}
})
t.Run("secrets_contain_dummy_and_real_keys", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("create failed: %v", err)
}
ns := model.NamespaceName("alice")
// dev-secrets should have dummy keys (fake clientset preserves StringData)
devSec, _ := client.Clientset.CoreV1().Secrets(ns).Get(ctx, "dev-secrets", metav1.GetOptions{})
if devSec.StringData["ANTHROPIC_API_KEY"] != "sk-devpod" {
t.Errorf("dev-secrets should have dummy anthropic key, got %s", devSec.StringData["ANTHROPIC_API_KEY"])
}
if devSec.StringData["VPN_GATEWAY_KEY"] != "test-vpn-key-123" {
t.Errorf("dev-secrets should have VPN key, got %s", devSec.StringData["VPN_GATEWAY_KEY"])
}
// ai-proxy-secrets should have real keys
aiSec, _ := client.Clientset.CoreV1().Secrets(ns).Get(ctx, "ai-proxy-secrets", metav1.GetOptions{})
if aiSec.StringData["anthropic-key"] != "br_test_anthropic" {
t.Errorf("ai-proxy-secrets should have real anthropic key, got %s", aiSec.StringData["anthropic-key"])
}
if aiSec.StringData["openai-key"] != "br_test_openai" {
t.Errorf("ai-proxy-secrets should have real openai key, got %s", aiSec.StringData["openai-key"])
}
})
}
func TestDeletePod(t *testing.T) {
t.Run("deletes_existing_pod", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("setup: create pod failed: %v", err)
}
err := client.DeletePod(ctx, "alice", "main")
if err != nil {
t.Fatalf("delete failed: %v", err)
}
ns := model.NamespaceName("alice")
// Pod should be gone
_, err = client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-main", metav1.GetOptions{})
if err == nil {
t.Error("pod should have been deleted")
}
// Service should be gone
_, err = client.Clientset.CoreV1().Services(ns).Get(ctx, "dev-pod-main-svc", metav1.GetOptions{})
if err == nil {
t.Error("service should have been deleted")
}
// IngressRoute should be gone
_, err = client.Dynamic.Resource(traefikIngressRouteGVR).Namespace(ns).Get(ctx, "dev-pod-main-ingress", metav1.GetOptions{})
if err == nil {
t.Error("ingress route should have been deleted")
}
// Per-pod PVC should be gone
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
if err == nil {
t.Error("per-pod PVC should have been deleted")
}
// Namespace should still exist
_, err = client.Clientset.CoreV1().Namespaces().Get(ctx, ns, metav1.GetOptions{})
if err != nil {
t.Errorf("namespace should still exist: %v", err)
}
})
t.Run("not_found", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
// Create namespace so the "get pod" doesn't fail on namespace
_ = client.EnsureNamespace(ctx, "alice")
err := client.DeletePod(ctx, "alice", "nonexistent")
if !errors.Is(err, ErrPodNotFound) {
t.Errorf("expected ErrPodNotFound, got: %v", err)
}
})
t.Run("keeps_other_pods", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
// Create two pods
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("setup: create pod1 failed: %v", err)
}
opts.Pod = "secondary"
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("setup: create pod2 failed: %v", err)
}
// Delete only the first
if err := client.DeletePod(ctx, "alice", "main"); err != nil {
t.Fatalf("delete failed: %v", err)
}
ns := model.NamespaceName("alice")
// First pod gone
_, err := client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-main", metav1.GetOptions{})
if err == nil {
t.Error("first pod should be deleted")
}
// First pod's PVC gone
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
if err == nil {
t.Error("first pod's PVC should be deleted")
}
// Second pod still exists
_, err = client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-secondary", metav1.GetOptions{})
if err != nil {
t.Errorf("second pod should still exist: %v", err)
}
// Second pod's PVC still exists
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-secondary", metav1.GetOptions{})
if err != nil {
t.Errorf("second pod's PVC should still exist: %v", err)
}
})
}
func TestDeleteAllPods(t *testing.T) {
t.Run("deletes_entire_namespace", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("setup: create pod failed: %v", err)
}
if err := client.DeleteAllPods(ctx, "alice"); err != nil {
t.Fatalf("delete all failed: %v", err)
}
_, err := client.Clientset.CoreV1().Namespaces().Get(ctx, "dev-alice", metav1.GetOptions{})
if err == nil {
t.Error("namespace should have been deleted")
}
})
t.Run("idempotent_on_nonexistent", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
if err := client.DeleteAllPods(ctx, "nonexistent"); err != nil {
t.Fatalf("expected no error, got: %v", err)
}
})
}
func TestListPods(t *testing.T) {
t.Run("empty_namespace", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
pods, err := client.ListPods(ctx, "alice")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(pods) != 0 {
t.Errorf("expected 0 pods, got %d", len(pods))
}
})
t.Run("multiple_pods", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("create pod1 failed: %v", err)
}
opts.Pod = "secondary"
opts.Tools = "python@3.12"
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("create pod2 failed: %v", err)
}
pods, err := client.ListPods(ctx, "alice")
if err != nil {
t.Fatalf("list failed: %v", err)
}
if len(pods) != 2 {
t.Fatalf("expected 2 pods, got %d", len(pods))
}
// Check pod details are populated
podNames := make(map[string]bool)
for _, p := range pods {
podNames[p.Name] = true
if p.User != "alice" {
t.Errorf("expected user alice, got %s", p.User)
}
if p.URL == "" {
t.Error("expected non-empty URL")
}
}
if !podNames["main"] || !podNames["secondary"] {
t.Errorf("expected pods main and secondary, got %v", podNames)
}
})
t.Run("multiple_users_isolated", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
// Alice's pod
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("create alice pod failed: %v", err)
}
// Bob's pod
opts.User = "bob"
opts.Pod = "dev"
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("create bob pod failed: %v", err)
}
// Alice should see only her pod
alicePods, err := client.ListPods(ctx, "alice")
if err != nil {
t.Fatalf("list alice pods failed: %v", err)
}
if len(alicePods) != 1 {
t.Errorf("expected 1 pod for alice, got %d", len(alicePods))
}
// Bob should see only his pod
bobPods, err := client.ListPods(ctx, "bob")
if err != nil {
t.Fatalf("list bob pods failed: %v", err)
}
if len(bobPods) != 1 {
t.Errorf("expected 1 pod for bob, got %d", len(bobPods))
}
})
}
func TestGetPod(t *testing.T) {
t.Run("success", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("setup: create pod failed: %v", err)
}
pod, err := client.GetPod(ctx, "alice", "main")
if err != nil {
t.Fatalf("get pod failed: %v", err)
}
if pod.User != "alice" {
t.Errorf("expected user alice, got %s", pod.User)
}
if pod.Name != "main" {
t.Errorf("expected name main, got %s", pod.Name)
}
if pod.Tools != "go@1.25,rust@1.94" {
t.Errorf("expected tools go@1.25,rust@1.94, got %s", pod.Tools)
}
if pod.Task != "build a web server" {
t.Errorf("expected task, got %s", pod.Task)
}
if pod.URL != "https://spinoff.dev/@alice/main/" {
t.Errorf("expected URL, got %s", pod.URL)
}
if pod.CPUReq != "2" {
t.Errorf("expected cpu req 2, got %s", pod.CPUReq)
}
})
t.Run("not_found", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
_ = client.EnsureNamespace(ctx, "alice")
_, err := client.GetPod(ctx, "alice", "nonexistent")
if !errors.Is(err, ErrPodNotFound) {
t.Errorf("expected ErrPodNotFound, got: %v", err)
}
})
}