build source

This commit is contained in:
build 2026-04-16 04:16:36 +00:00
commit ee1fec43ed
4171 changed files with 1351288 additions and 0 deletions

BIN
internal/k8s/._client.go Normal file

Binary file not shown.

BIN
internal/k8s/._cluster.go Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
internal/k8s/._metrics.go Normal file

Binary file not shown.

BIN
internal/k8s/._namespace.go Normal file

Binary file not shown.

Binary file not shown.

BIN
internal/k8s/._pods.go Normal file

Binary file not shown.

BIN
internal/k8s/._pods_test.go Normal file

Binary file not shown.

BIN
internal/k8s/._runners.go Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
internal/k8s/._templates.go Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

102
internal/k8s/client.go Normal file
View file

@ -0,0 +1,102 @@
package k8s
import (
"fmt"
"log/slog"
"os"
"path/filepath"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
// Client wraps a Kubernetes clientset with config used for template generation.
type Client struct {
Clientset kubernetes.Interface
Dynamic dynamic.Interface
Config Config
}
// Config holds cluster-specific settings for pod template generation.
type Config struct {
Domain string // e.g. "spinoff.dev"
Registry string // e.g. "10.22.0.56:30500"
GoldenImage string // e.g. "dev-golden:v2"
VPNGatewayNS string // e.g. "claw-system"
VPNGatewaySecret string // e.g. "vpn-gateway-secrets"
AnthropicKey string // baseroute anthropic key
OpenAIKey string // baseroute openai key
ForgejoURL string // e.g. "http://forgejo.dev-infra.svc:3000"
DevPodAPIURL string // e.g. "http://dev-pod-api.dev-infra.svc:8080"
}
// ConfigFromEnv reads Config from environment variables with defaults.
func ConfigFromEnv() Config {
return Config{
Domain: envOrDefault("DOMAIN", "spinoff.dev"),
Registry: envOrDefault("REGISTRY", "10.22.0.56:30500"),
GoldenImage: envOrDefault("GOLDEN_IMAGE", "dev-golden:v2"),
VPNGatewayNS: envOrDefault("VPN_GATEWAY_NS", "claw-system"),
VPNGatewaySecret: envOrDefault("VPN_GATEWAY_SECRET", "vpn-gateway-secrets"),
AnthropicKey: os.Getenv("DEFAULT_ANTHROPIC_KEY"),
OpenAIKey: os.Getenv("DEFAULT_OPENAI_KEY"),
ForgejoURL: envOrDefault("FORGEJO_URL", "http://forgejo.dev-infra.svc:3000"),
DevPodAPIURL: envOrDefault("DEV_POD_API_URL", "http://dev-pod-api.dev-infra.svc:8080"),
}
}
// NewClient creates a Kubernetes client using in-cluster config,
// falling back to kubeconfig for local development.
func NewClient(cfg Config) (*Client, error) {
restCfg, err := rest.InClusterConfig()
if err != nil {
slog.Info("in-cluster config not available, falling back to kubeconfig")
restCfg, err = kubeconfigFromDefault()
if err != nil {
return nil, fmt.Errorf("create k8s config: %w", err)
}
}
clientset, err := kubernetes.NewForConfig(restCfg)
if err != nil {
return nil, fmt.Errorf("create k8s clientset: %w", err)
}
dyn, err := dynamic.NewForConfig(restCfg)
if err != nil {
return nil, fmt.Errorf("create dynamic client: %w", err)
}
return &Client{
Clientset: clientset,
Dynamic: dyn,
Config: cfg,
}, nil
}
// NewClientWithClientset creates a Client with pre-configured clients (for testing).
func NewClientWithClientset(clientset kubernetes.Interface, dyn dynamic.Interface, cfg Config) *Client {
return &Client{
Clientset: clientset,
Dynamic: dyn,
Config: cfg,
}
}
func kubeconfigFromDefault() (*rest.Config, error) {
home, err := os.UserHomeDir()
if err != nil {
return nil, fmt.Errorf("get home dir: %w", err)
}
kubeconfig := filepath.Join(home, ".kube", "config")
return clientcmd.BuildConfigFromFlags("", kubeconfig)
}
func envOrDefault(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}

166
internal/k8s/cluster.go Normal file
View file

@ -0,0 +1,166 @@
package k8s
import (
"context"
"encoding/json"
"fmt"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
)
// nodeMetricsList mirrors the metrics API response for nodes.
type nodeMetricsList struct {
Items []nodeMetrics `json:"items"`
}
type nodeMetrics struct {
Metadata struct {
Name string `json:"name"`
} `json:"metadata"`
Usage map[string]string `json:"usage"`
}
// GetClusterStatus returns node list with capacity, allocatable, and usage.
func (c *Client) GetClusterStatus(ctx context.Context) (*model.ClusterStatus, error) {
nodes, err := c.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("list nodes: %w", err)
}
// Try to fetch node metrics (may not be available)
metricsMap := c.fetchNodeMetrics(ctx)
var totalCPUCap, totalCPUAlloc, totalMemCap, totalMemAlloc int64
status := &model.ClusterStatus{
Nodes: make([]model.NodeStatus, 0, len(nodes.Items)),
}
for _, node := range nodes.Items {
ns := model.NodeStatus{
Name: node.Name,
Status: nodeConditionStatus(node),
CPUCapacity: node.Status.Capacity.Cpu().String(),
CPUAllocatable: node.Status.Allocatable.Cpu().String(),
MemCapacity: node.Status.Capacity.Memory().String(),
MemAllocatable: node.Status.Allocatable.Memory().String(),
}
totalCPUCap += node.Status.Capacity.Cpu().MilliValue()
totalCPUAlloc += node.Status.Allocatable.Cpu().MilliValue()
totalMemCap += node.Status.Capacity.Memory().Value()
totalMemAlloc += node.Status.Allocatable.Memory().Value()
if m, ok := metricsMap[node.Name]; ok {
ns.CPUUsage = m.cpuUsage
ns.MemUsage = m.memUsage
}
status.Nodes = append(status.Nodes, ns)
}
status.Total = model.ResourceSummary{
CPUCapacity: resource.NewMilliQuantity(totalCPUCap, resource.DecimalSI).String(),
CPUAllocatable: resource.NewMilliQuantity(totalCPUAlloc, resource.DecimalSI).String(),
MemCapacity: resource.NewQuantity(totalMemCap, resource.BinarySI).String(),
MemAllocatable: resource.NewQuantity(totalMemAlloc, resource.BinarySI).String(),
}
return status, nil
}
type nodeUsage struct {
cpuUsage string
memUsage string
}
func (c *Client) fetchNodeMetrics(ctx context.Context) map[string]nodeUsage {
result := make(map[string]nodeUsage)
restClient := c.Clientset.Discovery().RESTClient()
if restClient == nil {
return result
}
data, err := restClient.Get().
AbsPath("/apis/metrics.k8s.io/v1beta1/nodes").
DoRaw(ctx)
if err != nil {
return result
}
var metrics nodeMetricsList
if err := json.Unmarshal(data, &metrics); err != nil {
return result
}
for _, m := range metrics.Items {
nu := nodeUsage{}
if cpu, ok := m.Usage["cpu"]; ok {
nu.cpuUsage = cpu
}
if mem, ok := m.Usage["memory"]; ok {
nu.memUsage = mem
}
result[m.Metadata.Name] = nu
}
return result
}
func nodeConditionStatus(node corev1.Node) string {
for _, cond := range node.Status.Conditions {
if cond.Type == corev1.NodeReady {
if cond.Status == corev1.ConditionTrue {
return "Ready"
}
return "NotReady"
}
}
return "Unknown"
}
// cacheServices maps friendly names to PVC names in the dev-infra namespace.
var cacheServices = []struct {
name string
pvcName string
}{
{"verdaccio", "verdaccio-storage"},
{"athens", "athens-storage"},
{"cargo-proxy", "cargo-proxy-cache"},
}
// GetCacheStats returns PVC status for cache services in dev-infra.
func (c *Client) GetCacheStats(ctx context.Context) ([]model.CacheStat, error) {
stats := make([]model.CacheStat, 0, len(cacheServices))
for _, svc := range cacheServices {
pvc, err := c.Clientset.CoreV1().PersistentVolumeClaims("dev-infra").Get(ctx, svc.pvcName, metav1.GetOptions{})
if err != nil {
stats = append(stats, model.CacheStat{
Name: svc.name,
PVCName: svc.pvcName,
Status: "NotFound",
})
continue
}
capacity := ""
if pvc.Status.Capacity != nil {
if storage, ok := pvc.Status.Capacity[corev1.ResourceStorage]; ok {
capacity = storage.String()
}
}
stats = append(stats, model.CacheStat{
Name: svc.name,
PVCName: svc.pvcName,
Capacity: capacity,
Status: string(pvc.Status.Phase),
})
}
return stats, nil
}

View file

@ -0,0 +1,224 @@
package k8s
import (
"context"
"testing"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
func TestGetClusterStatus_Nodes(t *testing.T) {
cs := fake.NewSimpleClientset(
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{Type: corev1.NodeReady, Status: corev1.ConditionTrue},
},
Capacity: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("8"),
corev1.ResourceMemory: resource.MustParse("32Gi"),
},
Allocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("7800m"),
corev1.ResourceMemory: resource.MustParse("30Gi"),
},
},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{Name: "node-2"},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{Type: corev1.NodeReady, Status: corev1.ConditionFalse},
},
Capacity: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("4"),
corev1.ResourceMemory: resource.MustParse("16Gi"),
},
Allocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("3800m"),
corev1.ResourceMemory: resource.MustParse("14Gi"),
},
},
},
)
client := NewClientWithClientset(cs, nil, Config{})
status, err := client.GetClusterStatus(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(status.Nodes) != 2 {
t.Fatalf("expected 2 nodes, got %d", len(status.Nodes))
}
n1 := status.Nodes[0]
if n1.Name != "node-1" {
t.Fatalf("expected node-1, got %s", n1.Name)
}
if n1.Status != "Ready" {
t.Fatalf("expected Ready, got %s", n1.Status)
}
if n1.CPUCapacity != "8" {
t.Fatalf("expected CPU capacity 8, got %s", n1.CPUCapacity)
}
n2 := status.Nodes[1]
if n2.Status != "NotReady" {
t.Fatalf("expected NotReady, got %s", n2.Status)
}
// Check totals: 8 + 4 = 12 CPU, 7800m + 3800m = 11600m allocatable
if status.Total.CPUCapacity != "12" {
t.Fatalf("expected total CPU capacity 12, got %s", status.Total.CPUCapacity)
}
}
func TestGetClusterStatus_EmptyCluster(t *testing.T) {
cs := fake.NewSimpleClientset()
client := NewClientWithClientset(cs, nil, Config{})
status, err := client.GetClusterStatus(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(status.Nodes) != 0 {
t.Fatalf("expected 0 nodes, got %d", len(status.Nodes))
}
}
func TestGetClusterStatus_NodeWithNoReadyCondition(t *testing.T) {
cs := fake.NewSimpleClientset(
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{Name: "node-unknown"},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{Type: corev1.NodeMemoryPressure, Status: corev1.ConditionFalse},
},
Capacity: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("4"),
corev1.ResourceMemory: resource.MustParse("8Gi"),
},
Allocatable: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("4"),
corev1.ResourceMemory: resource.MustParse("8Gi"),
},
},
},
)
client := NewClientWithClientset(cs, nil, Config{})
status, err := client.GetClusterStatus(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if status.Nodes[0].Status != "Unknown" {
t.Fatalf("expected Unknown, got %s", status.Nodes[0].Status)
}
}
func TestGetCacheStats_AllBound(t *testing.T) {
cs := fake.NewSimpleClientset(
&corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Name: "verdaccio-storage", Namespace: "dev-infra"},
Status: corev1.PersistentVolumeClaimStatus{
Phase: corev1.ClaimBound,
Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("10Gi")},
},
},
&corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Name: "athens-storage", Namespace: "dev-infra"},
Status: corev1.PersistentVolumeClaimStatus{
Phase: corev1.ClaimBound,
Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("10Gi")},
},
},
&corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Name: "cargo-proxy-cache", Namespace: "dev-infra"},
Status: corev1.PersistentVolumeClaimStatus{
Phase: corev1.ClaimBound,
Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("10Gi")},
},
},
)
client := NewClientWithClientset(cs, nil, Config{})
stats, err := client.GetCacheStats(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(stats) != 3 {
t.Fatalf("expected 3 cache stats, got %d", len(stats))
}
for _, s := range stats {
if s.Status != "Bound" {
t.Fatalf("expected Bound for %s, got %s", s.Name, s.Status)
}
if s.Capacity != "10Gi" {
t.Fatalf("expected 10Gi capacity for %s, got %s", s.Name, s.Capacity)
}
}
}
func TestGetCacheStats_MissingPVC(t *testing.T) {
// Only verdaccio exists
cs := fake.NewSimpleClientset(
&corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Name: "verdaccio-storage", Namespace: "dev-infra"},
Status: corev1.PersistentVolumeClaimStatus{
Phase: corev1.ClaimBound,
Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("10Gi")},
},
},
)
client := NewClientWithClientset(cs, nil, Config{})
stats, err := client.GetCacheStats(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(stats) != 3 {
t.Fatalf("expected 3 entries, got %d", len(stats))
}
for _, s := range stats {
switch s.Name {
case "verdaccio":
if s.Status != "Bound" {
t.Fatalf("expected Bound for verdaccio, got %s", s.Status)
}
case "athens", "cargo-proxy":
if s.Status != "NotFound" {
t.Fatalf("expected NotFound for %s, got %s", s.Name, s.Status)
}
}
}
}
func TestGetCacheStats_EmptyCluster(t *testing.T) {
cs := fake.NewSimpleClientset()
client := NewClientWithClientset(cs, nil, Config{})
stats, err := client.GetCacheStats(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(stats) != 3 {
t.Fatalf("expected 3 entries, got %d", len(stats))
}
for _, s := range stats {
if s.Status != "NotFound" {
t.Fatalf("expected NotFound for %s, got %s", s.Name, s.Status)
}
}
}

View file

@ -0,0 +1,369 @@
package k8s
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"k8s.io/apimachinery/pkg/util/yaml"
)
func toInt64(v interface{}) int64 {
switch n := v.(type) {
case int64:
return n
case float64:
return int64(n)
case int:
return int64(n)
default:
return 0
}
}
func forgejoYAMLPath() string {
_, thisFile, _, _ := runtime.Caller(0)
return filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "..", "k8s", "dev-infra", "forgejo.yaml")
}
func readForgejoDocuments(t *testing.T) []map[string]interface{} {
t.Helper()
data, err := os.ReadFile(forgejoYAMLPath())
if err != nil {
t.Fatalf("reading forgejo.yaml: %v", err)
}
var docs []map[string]interface{}
parts := strings.Split(string(data), "\n---")
for _, part := range parts {
trimmed := strings.TrimSpace(part)
if trimmed == "" {
continue
}
// Strip leading comment lines (file-level comments before first doc)
lines := strings.Split(trimmed, "\n")
var contentLines []string
for _, line := range lines {
stripped := strings.TrimSpace(line)
if len(contentLines) == 0 && (stripped == "" || strings.HasPrefix(stripped, "#")) {
continue
}
contentLines = append(contentLines, line)
}
content := strings.Join(contentLines, "\n")
if content == "" {
continue
}
var doc map[string]interface{}
if err := yaml.NewYAMLOrJSONDecoder(strings.NewReader(content), 4096).Decode(&doc); err != nil {
t.Fatalf("decoding YAML document: %v\n---\n%s", err, content[:min(200, len(content))])
}
docs = append(docs, doc)
}
return docs
}
func findDoc(docs []map[string]interface{}, kind, name string) map[string]interface{} {
for _, doc := range docs {
if doc["kind"] == kind {
meta, ok := doc["metadata"].(map[string]interface{})
if ok && meta["name"] == name {
return doc
}
}
}
return nil
}
func TestForgejoYAMLExists(t *testing.T) {
path := forgejoYAMLPath()
if _, err := os.Stat(path); os.IsNotExist(err) {
t.Fatalf("forgejo.yaml does not exist at %s", path)
}
}
func TestForgejoYAMLDocumentCount(t *testing.T) {
docs := readForgejoDocuments(t)
// Expected: Secret (admin), ConfigMap, PVC, Deployment, Service, IngressRoute, Middleware (basic-auth), Secret (htpasswd), Middleware (trailing-slash)
if len(docs) < 9 {
kinds := make([]string, len(docs))
for i, d := range docs {
kinds[i] = d["kind"].(string)
}
t.Errorf("expected at least 9 YAML documents, got %d: %v", len(docs), kinds)
}
}
func TestForgejoAdminSecret(t *testing.T) {
docs := readForgejoDocuments(t)
doc := findDoc(docs, "Secret", "forgejo-admin-secret")
if doc == nil {
t.Fatal("missing forgejo-admin-secret Secret")
}
meta := doc["metadata"].(map[string]interface{})
if meta["namespace"] != "dev-infra" {
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
}
}
func TestForgejoConfigMap(t *testing.T) {
docs := readForgejoDocuments(t)
doc := findDoc(docs, "ConfigMap", "forgejo-config")
if doc == nil {
t.Fatal("missing forgejo-config ConfigMap")
}
meta := doc["metadata"].(map[string]interface{})
if meta["namespace"] != "dev-infra" {
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
}
data := doc["data"].(map[string]interface{})
appIni, ok := data["app.ini"].(string)
if !ok {
t.Fatal("missing app.ini in ConfigMap data")
}
requiredSettings := []string{
"ROOT_URL = https://spinoff.dev/forgejo/",
"HTTP_PORT = 3000",
"DB_TYPE = sqlite3",
"ENABLED = true",
"DISABLE_REGISTRATION = true",
"OFFLINE_MODE = true",
"DEFAULT_BRANCH = main",
}
for _, s := range requiredSettings {
if !strings.Contains(appIni, s) {
t.Errorf("app.ini missing required setting: %s", s)
}
}
}
func TestForgejoPVC(t *testing.T) {
docs := readForgejoDocuments(t)
doc := findDoc(docs, "PersistentVolumeClaim", "forgejo-storage")
if doc == nil {
t.Fatal("missing forgejo-storage PVC")
}
meta := doc["metadata"].(map[string]interface{})
if meta["namespace"] != "dev-infra" {
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
}
spec := doc["spec"].(map[string]interface{})
storageClass, ok := spec["storageClassName"].(string)
if !ok || storageClass != "longhorn" {
t.Errorf("expected storageClassName longhorn, got %v", storageClass)
}
resources := spec["resources"].(map[string]interface{})
requests := resources["requests"].(map[string]interface{})
if requests["storage"] != "5Gi" {
t.Errorf("expected 5Gi storage, got %v", requests["storage"])
}
}
func TestForgejoDeployment(t *testing.T) {
docs := readForgejoDocuments(t)
doc := findDoc(docs, "Deployment", "forgejo")
if doc == nil {
t.Fatal("missing forgejo Deployment")
}
meta := doc["metadata"].(map[string]interface{})
if meta["namespace"] != "dev-infra" {
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
}
labels := meta["labels"].(map[string]interface{})
if labels["app"] != "forgejo" {
t.Errorf("expected label app=forgejo, got %v", labels["app"])
}
spec := doc["spec"].(map[string]interface{})
strategy := spec["strategy"].(map[string]interface{})
if strategy["type"] != "Recreate" {
t.Errorf("expected Recreate strategy, got %v", strategy["type"])
}
template := spec["template"].(map[string]interface{})
podSpec := template["spec"].(map[string]interface{})
t.Run("init_containers", func(t *testing.T) {
initContainers, ok := podSpec["initContainers"].([]interface{})
if !ok || len(initContainers) < 2 {
t.Fatal("expected at least 2 init containers (setup-config and create-admin)")
}
setupC := initContainers[0].(map[string]interface{})
if setupC["name"] != "setup-config" {
t.Errorf("expected first init container name setup-config, got %v", setupC["name"])
}
initC := initContainers[1].(map[string]interface{})
if initC["name"] != "create-admin" {
t.Errorf("expected second init container name create-admin, got %v", initC["name"])
}
image, _ := initC["image"].(string)
if !strings.HasPrefix(image, "codeberg.org/forgejo/forgejo:") {
t.Errorf("expected forgejo image, got %v", image)
}
})
t.Run("containers", func(t *testing.T) {
containers := podSpec["containers"].([]interface{})
if len(containers) != 2 {
t.Fatalf("expected 2 containers (forgejo + ipip-sidecar), got %d", len(containers))
}
forgejo := containers[0].(map[string]interface{})
if forgejo["name"] != "forgejo" {
t.Errorf("expected container name forgejo, got %v", forgejo["name"])
}
image, _ := forgejo["image"].(string)
if !strings.HasPrefix(image, "codeberg.org/forgejo/forgejo:") {
t.Errorf("expected forgejo image, got %v", image)
}
ports := forgejo["ports"].([]interface{})
if len(ports) != 1 {
t.Errorf("expected 1 port, got %d", len(ports))
}
port := ports[0].(map[string]interface{})
containerPort := toInt64(port["containerPort"])
if containerPort != 3000 {
t.Errorf("expected port 3000, got %v", port["containerPort"])
}
ipip := containers[1].(map[string]interface{})
if ipip["name"] != "ipip-sidecar" {
t.Errorf("expected container name ipip-sidecar, got %v", ipip["name"])
}
})
t.Run("volumes", func(t *testing.T) {
volumes := podSpec["volumes"].([]interface{})
if len(volumes) < 2 {
t.Fatalf("expected at least 2 volumes (storage + config), got %d", len(volumes))
}
volumeNames := make(map[string]bool)
for _, v := range volumes {
vol := v.(map[string]interface{})
volumeNames[vol["name"].(string)] = true
}
if !volumeNames["storage"] {
t.Error("missing storage volume")
}
if !volumeNames["config"] {
t.Error("missing config volume")
}
})
}
func TestForgejoService(t *testing.T) {
docs := readForgejoDocuments(t)
doc := findDoc(docs, "Service", "forgejo")
if doc == nil {
t.Fatal("missing forgejo Service")
}
meta := doc["metadata"].(map[string]interface{})
if meta["namespace"] != "dev-infra" {
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
}
spec := doc["spec"].(map[string]interface{})
if spec["type"] != "ClusterIP" {
t.Errorf("expected ClusterIP, got %v", spec["type"])
}
ports := spec["ports"].([]interface{})
if len(ports) != 1 {
t.Fatalf("expected 1 port, got %d", len(ports))
}
port := ports[0].(map[string]interface{})
portNum := toInt64(port["port"])
if portNum != 3000 {
t.Errorf("expected port 3000, got %v", port["port"])
}
selector := spec["selector"].(map[string]interface{})
if selector["app"] != "forgejo" {
t.Errorf("expected selector app=forgejo, got %v", selector["app"])
}
}
func TestForgejoIngressRoute(t *testing.T) {
docs := readForgejoDocuments(t)
doc := findDoc(docs, "IngressRoute", "forgejo-ingress")
if doc == nil {
t.Fatal("missing forgejo-ingress IngressRoute")
}
meta := doc["metadata"].(map[string]interface{})
if meta["namespace"] != "dev-infra" {
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
}
spec := doc["spec"].(map[string]interface{})
entryPoints := spec["entryPoints"].([]interface{})
if len(entryPoints) != 2 {
t.Errorf("expected 2 entryPoints (web, websecure), got %d", len(entryPoints))
}
routes := spec["routes"].([]interface{})
if len(routes) < 2 {
t.Fatalf("expected at least 2 routes (main + trailing-slash redirect), got %d", len(routes))
}
mainRoute := routes[0].(map[string]interface{})
matchRule, _ := mainRoute["match"].(string)
if !strings.Contains(matchRule, "PathPrefix(`/forgejo/`)") {
t.Errorf("expected PathPrefix /forgejo/ in main route, got %v", matchRule)
}
middlewares := mainRoute["middlewares"].([]interface{})
if len(middlewares) < 1 {
t.Fatal("expected at least 1 middleware on main route")
}
mw := middlewares[0].(map[string]interface{})
if mw["name"] != "forgejo-basic-auth" {
t.Errorf("expected forgejo-basic-auth middleware, got %v", mw["name"])
}
}
func TestForgejoBasicAuthMiddleware(t *testing.T) {
docs := readForgejoDocuments(t)
doc := findDoc(docs, "Middleware", "forgejo-basic-auth")
if doc == nil {
t.Fatal("missing forgejo-basic-auth Middleware")
}
meta := doc["metadata"].(map[string]interface{})
if meta["namespace"] != "dev-infra" {
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
}
spec := doc["spec"].(map[string]interface{})
basicAuth := spec["basicAuth"].(map[string]interface{})
if basicAuth["secret"] != "forgejo-basic-auth-secret" {
t.Errorf("expected secret forgejo-basic-auth-secret, got %v", basicAuth["secret"])
}
}
func TestForgejoTrailingSlashMiddleware(t *testing.T) {
docs := readForgejoDocuments(t)
doc := findDoc(docs, "Middleware", "forgejo-add-trailing-slash")
if doc == nil {
t.Fatal("missing forgejo-add-trailing-slash Middleware")
}
spec := doc["spec"].(map[string]interface{})
redirect := spec["redirectRegex"].(map[string]interface{})
if redirect["permanent"] != true {
t.Error("expected permanent redirect")
}
}

135
internal/k8s/metrics.go Normal file
View file

@ -0,0 +1,135 @@
package k8s
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"strings"
"time"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// UsageRecorder records resource usage samples into the store.
type UsageRecorder interface {
RecordResourceSample(ctx context.Context, userID, podName string, cpuMillicores, memBytes float64) error
}
// ResourceSampler periodically samples pod resource usage and records it.
type ResourceSampler struct {
client *Client
recorder UsageRecorder
logger *slog.Logger
interval time.Duration
}
// NewResourceSampler creates a new resource sampler with a 60-second interval.
func NewResourceSampler(client *Client, recorder UsageRecorder, logger *slog.Logger) *ResourceSampler {
return &ResourceSampler{
client: client,
recorder: recorder,
logger: logger,
interval: 60 * time.Second,
}
}
// Start runs the sampling loop until the context is cancelled.
func (s *ResourceSampler) Start(ctx context.Context) {
ticker := time.NewTicker(s.interval)
defer ticker.Stop()
s.logger.Info("resource sampler started", "interval", s.interval)
for {
select {
case <-ctx.Done():
s.logger.Info("resource sampler stopped")
return
case <-ticker.C:
s.sample(ctx)
}
}
}
// podMetricsList mirrors the k8s metrics API response.
type podMetricsList struct {
Items []podMetrics `json:"items"`
}
type podMetrics struct {
Metadata podMetricsMetadata `json:"metadata"`
Containers []containerMetrics `json:"containers"`
}
type podMetricsMetadata struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
Labels map[string]string `json:"labels"`
}
type containerMetrics struct {
Name string `json:"name"`
Usage map[string]string `json:"usage"`
}
func (s *ResourceSampler) sample(ctx context.Context) {
// List dev-pod namespaces
nsList, err := s.client.Clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{
LabelSelector: "managed-by=dev-pod-api",
})
if err != nil {
s.logger.Error("failed to list namespaces for sampling", "error", err)
return
}
for _, ns := range nsList.Items {
user := ns.Labels["user"]
if user == "" {
continue
}
// Query metrics API for this namespace
data, err := s.client.Clientset.Discovery().RESTClient().Get().
AbsPath(fmt.Sprintf("/apis/metrics.k8s.io/v1beta1/namespaces/%s/pods", ns.Name)).
DoRaw(ctx)
if err != nil {
s.logger.Warn("failed to get pod metrics", "namespace", ns.Name, "error", err)
continue
}
var metrics podMetricsList
if err := json.Unmarshal(data, &metrics); err != nil {
s.logger.Warn("failed to parse pod metrics", "namespace", ns.Name, "error", err)
continue
}
for _, pm := range metrics.Items {
if !strings.HasPrefix(pm.Metadata.Name, "dev-pod-") {
continue
}
podName := strings.TrimPrefix(pm.Metadata.Name, "dev-pod-")
var totalCPU, totalMem float64
for _, cm := range pm.Containers {
if cpuStr, ok := cm.Usage["cpu"]; ok {
q, err := resource.ParseQuantity(cpuStr)
if err == nil {
totalCPU += float64(q.MilliValue())
}
}
if memStr, ok := cm.Usage["memory"]; ok {
q, err := resource.ParseQuantity(memStr)
if err == nil {
totalMem += float64(q.Value())
}
}
}
if err := s.recorder.RecordResourceSample(ctx, user, podName, totalCPU, totalMem); err != nil {
s.logger.Warn("failed to record sample", "user", user, "pod", podName, "error", err)
}
}
}
}

49
internal/k8s/namespace.go Normal file
View file

@ -0,0 +1,49 @@
package k8s
import (
"context"
"fmt"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
)
// EnsureNamespace creates a namespace for the user if it doesn't already exist.
// Returns nil if the namespace already exists (idempotent).
func (c *Client) EnsureNamespace(ctx context.Context, user string) error {
ns := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: model.NamespaceName(user),
Labels: map[string]string{
"app": "dev-pod",
"managed-by": "dev-pod-api",
"user": user,
},
},
}
_, err := c.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
if err != nil {
if errors.IsAlreadyExists(err) {
return nil
}
return fmt.Errorf("create namespace %s: %w", ns.Name, err)
}
return nil
}
// DeleteNamespace deletes the user's namespace and all resources in it.
func (c *Client) DeleteNamespace(ctx context.Context, user string) error {
nsName := model.NamespaceName(user)
err := c.Clientset.CoreV1().Namespaces().Delete(ctx, nsName, metav1.DeleteOptions{})
if err != nil {
if errors.IsNotFound(err) {
return nil
}
return fmt.Errorf("delete namespace %s: %w", nsName, err)
}
return nil
}

View file

@ -0,0 +1,87 @@
package k8s
import (
"context"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
fakedynamic "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/kubernetes/fake"
)
func TestEnsureNamespace(t *testing.T) {
t.Run("creates_new_namespace", func(t *testing.T) {
client := NewClientWithClientset(fake.NewSimpleClientset(), fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), testCfg)
ctx := context.Background()
err := client.EnsureNamespace(ctx, "alice")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ns, err := client.Clientset.CoreV1().Namespaces().Get(ctx, "dev-alice", metav1.GetOptions{})
if err != nil {
t.Fatalf("failed to get namespace: %v", err)
}
if ns.Name != "dev-alice" {
t.Errorf("expected namespace dev-alice, got %s", ns.Name)
}
if ns.Labels["app"] != "dev-pod" {
t.Errorf("expected label app=dev-pod, got %s", ns.Labels["app"])
}
if ns.Labels["managed-by"] != "dev-pod-api" {
t.Errorf("expected label managed-by=dev-pod-api, got %s", ns.Labels["managed-by"])
}
if ns.Labels["user"] != "alice" {
t.Errorf("expected label user=alice, got %s", ns.Labels["user"])
}
})
t.Run("idempotent_on_existing_namespace", func(t *testing.T) {
client := NewClientWithClientset(fake.NewSimpleClientset(), fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), testCfg)
ctx := context.Background()
// Create twice - second call should not error
err := client.EnsureNamespace(ctx, "bob")
if err != nil {
t.Fatalf("first create failed: %v", err)
}
err = client.EnsureNamespace(ctx, "bob")
if err != nil {
t.Fatalf("second create should be idempotent, got: %v", err)
}
})
}
func TestDeleteNamespace(t *testing.T) {
t.Run("deletes_existing_namespace", func(t *testing.T) {
client := NewClientWithClientset(fake.NewSimpleClientset(), fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), testCfg)
ctx := context.Background()
err := client.EnsureNamespace(ctx, "alice")
if err != nil {
t.Fatalf("setup failed: %v", err)
}
err = client.DeleteNamespace(ctx, "alice")
if err != nil {
t.Fatalf("delete failed: %v", err)
}
_, err = client.Clientset.CoreV1().Namespaces().Get(ctx, "dev-alice", metav1.GetOptions{})
if err == nil {
t.Error("expected namespace to be deleted")
}
})
t.Run("no_error_on_nonexistent_namespace", func(t *testing.T) {
client := NewClientWithClientset(fake.NewSimpleClientset(), fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), testCfg)
ctx := context.Background()
err := client.DeleteNamespace(ctx, "nonexistent")
if err != nil {
t.Fatalf("expected no error for nonexistent namespace, got: %v", err)
}
})
}

501
internal/k8s/pods.go Normal file
View file

@ -0,0 +1,501 @@
package k8s
import (
"context"
"errors"
"fmt"
"log/slog"
"time"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
)
var (
ErrPodAlreadyExists = errors.New("pod already exists")
ErrPodNotFound = errors.New("pod not found")
ErrQuotaExceeded = errors.New("quota exceeded")
)
var traefikIngressRouteGVR = schema.GroupVersionResource{
Group: "traefik.io",
Version: "v1alpha1",
Resource: "ingressroutes",
}
var traefikMiddlewareGVR = schema.GroupVersionResource{
Group: "traefik.io",
Version: "v1alpha1",
Resource: "middlewares",
}
// CreatePodOpts holds parameters for creating a complete pod with all resources.
type CreatePodOpts struct {
User string
Pod string
Tools string
Task string
CPUReq string
CPULimit string
MemReq string
MemLimit string
MaxConcurrentPods int // 0 = no limit
MaxCPUPerPod int // 0 = no limit
MaxRAMGBPerPod int // 0 = no limit
ForgejoToken string
TailscaleKey string
}
// FetchVPNKey reads the VPN gateway key from the configured secret in the gateway namespace.
func (c *Client) FetchVPNKey(ctx context.Context) (string, error) {
secret, err := c.Clientset.CoreV1().Secrets(c.Config.VPNGatewayNS).Get(
ctx, c.Config.VPNGatewaySecret, metav1.GetOptions{},
)
if err != nil {
return "", fmt.Errorf("get vpn gateway secret %s/%s: %w",
c.Config.VPNGatewayNS, c.Config.VPNGatewaySecret, err)
}
key, ok := secret.Data["VPN_GATEWAY_KEY"]
if !ok {
return "", fmt.Errorf("VPN_GATEWAY_KEY field not found in secret %s/%s",
c.Config.VPNGatewayNS, c.Config.VPNGatewaySecret)
}
return string(key), nil
}
// ValidatePodQuota checks per-pod CPU and RAM limits against the user's quota.
func ValidatePodQuota(cpuLimit string, maxCPUPerPod int, memLimit string, maxRAMGBPerPod int) error {
if maxCPUPerPod > 0 && cpuLimit != "" {
cpu, err := resource.ParseQuantity(cpuLimit)
if err != nil {
return fmt.Errorf("parse cpu_limit %q: %w", cpuLimit, err)
}
maxCPU := resource.MustParse(fmt.Sprintf("%d", maxCPUPerPod))
if cpu.Cmp(maxCPU) > 0 {
return fmt.Errorf("%w: cpu_limit %s exceeds max %d", ErrQuotaExceeded, cpuLimit, maxCPUPerPod)
}
}
if maxRAMGBPerPod > 0 && memLimit != "" {
mem, err := resource.ParseQuantity(memLimit)
if err != nil {
return fmt.Errorf("parse mem_limit %q: %w", memLimit, err)
}
maxMem := resource.MustParse(fmt.Sprintf("%dGi", maxRAMGBPerPod))
if mem.Cmp(maxMem) > 0 {
return fmt.Errorf("%w: mem_limit %s exceeds max %dGi", ErrQuotaExceeded, memLimit, maxRAMGBPerPod)
}
}
return nil
}
// CreatePod creates a complete dev pod with all associated k8s resources.
// Shared resources (namespace, PVC, secrets, configmap, network policy) are created idempotently.
// Pod-specific resources (pod, service, ingress) are cleaned up on partial failure.
func (c *Client) CreatePod(ctx context.Context, opts CreatePodOpts) (*model.Pod, error) {
ns := model.NamespaceName(opts.User)
// Check quota: count existing pods for this user
if opts.MaxConcurrentPods > 0 {
existingPods, err := c.Clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
LabelSelector: "app=dev-pod",
})
if err != nil && !k8serrors.IsNotFound(err) {
return nil, fmt.Errorf("list existing pods: %w", err)
}
if existingPods != nil && len(existingPods.Items) >= opts.MaxConcurrentPods {
return nil, ErrQuotaExceeded
}
}
// Check per-pod resource quota
if err := ValidatePodQuota(opts.CPULimit, opts.MaxCPUPerPod, opts.MemLimit, opts.MaxRAMGBPerPod); err != nil {
return nil, err
}
// Check for duplicate pod
podFullName := model.PodFullName(opts.Pod)
_, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, podFullName, metav1.GetOptions{})
if err == nil {
return nil, ErrPodAlreadyExists
}
if !k8serrors.IsNotFound(err) {
return nil, fmt.Errorf("check existing pod: %w", err)
}
// Fetch VPN key
vpnKey, err := c.FetchVPNKey(ctx)
if err != nil {
return nil, fmt.Errorf("fetch vpn key: %w", err)
}
// Create shared resources (idempotent)
if err := c.EnsureNamespace(ctx, opts.User); err != nil {
return nil, fmt.Errorf("ensure namespace: %w", err)
}
if err := c.ensureSharedResources(ctx, opts.User, vpnKey, opts.ForgejoToken, opts.TailscaleKey); err != nil {
return nil, err
}
// Create per-pod PVC
pvc := PVCTemplate(opts.User, opts.Pod)
if _, err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Create(ctx, pvc, metav1.CreateOptions{}); err != nil {
if !k8serrors.IsAlreadyExists(err) {
return nil, fmt.Errorf("create per-pod pvc: %w", err)
}
}
// Create pod-specific resources with cleanup on failure
podOpts := PodOpts{
User: opts.User,
Pod: opts.Pod,
Tools: opts.Tools,
Task: opts.Task,
CPUReq: opts.CPUReq,
CPULimit: opts.CPULimit,
MemReq: opts.MemReq,
MemLimit: opts.MemLimit,
VPNKey: vpnKey,
AnthropicKey: c.Config.AnthropicKey,
OpenAIKey: c.Config.OpenAIKey,
ForgejoToken: opts.ForgejoToken,
TailscaleKey: opts.TailscaleKey,
}
var cleanups []func()
doCleanup := func() {
for i := len(cleanups) - 1; i >= 0; i-- {
cleanups[i]()
}
}
// Register PVC cleanup
cleanups = append(cleanups, func() {
if delErr := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, pvc.Name, metav1.DeleteOptions{}); delErr != nil {
slog.Warn("cleanup: failed to delete pvc", "pvc", pvc.Name, "error", delErr)
}
})
// Create Pod
pod := PodTemplate(c.Config, podOpts)
if _, err := c.Clientset.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil {
return nil, fmt.Errorf("create pod: %w", err)
}
cleanups = append(cleanups, func() {
if delErr := c.Clientset.CoreV1().Pods(ns).Delete(ctx, pod.Name, metav1.DeleteOptions{}); delErr != nil {
slog.Warn("cleanup: failed to delete pod", "pod", pod.Name, "error", delErr)
}
})
// Create Service
svc := ServiceTemplate(opts.User, opts.Pod)
if _, err := c.Clientset.CoreV1().Services(ns).Create(ctx, svc, metav1.CreateOptions{}); err != nil {
doCleanup()
return nil, fmt.Errorf("create service: %w", err)
}
cleanups = append(cleanups, func() {
if delErr := c.Clientset.CoreV1().Services(ns).Delete(ctx, svc.Name, metav1.DeleteOptions{}); delErr != nil {
slog.Warn("cleanup: failed to delete service", "service", svc.Name, "error", delErr)
}
})
// Create IngressRoute + Middlewares
ingressObjs := IngressTemplate(opts.User, opts.Pod, c.Config.Domain)
for _, obj := range ingressObjs {
gvr, err := traefikGVR(obj.GetKind())
if err != nil {
doCleanup()
return nil, fmt.Errorf("resolve GVR for %s: %w", obj.GetKind(), err)
}
_, err = c.Dynamic.Resource(gvr).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{})
if err != nil {
if k8serrors.IsAlreadyExists(err) {
continue // shared resource already exists, don't register cleanup
}
doCleanup()
return nil, fmt.Errorf("create %s %s: %w", obj.GetKind(), obj.GetName(), err)
}
name := obj.GetName()
kind := obj.GetKind()
cleanups = append(cleanups, func() {
gvr, gvrErr := traefikGVR(kind)
if gvrErr != nil {
slog.Warn("cleanup: unknown traefik kind", "kind", kind, "error", gvrErr)
return
}
if delErr := c.Dynamic.Resource(gvr).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}); delErr != nil {
slog.Warn("cleanup: failed to delete "+kind, "name", name, "error", delErr)
}
})
}
return &model.Pod{
User: opts.User,
Name: opts.Pod,
Tools: opts.Tools,
CPUReq: opts.CPUReq,
CPULimit: opts.CPULimit,
MemReq: opts.MemReq,
MemLimit: opts.MemLimit,
Task: opts.Task,
Status: "Pending",
URL: model.PodURL(c.Config.Domain, opts.User, opts.Pod),
}, nil
}
// DeletePod removes a single pod and its associated service and ingress resources.
// Keeps the namespace and shared resources if other pods exist.
func (c *Client) DeletePod(ctx context.Context, user, pod string) error {
ns := model.NamespaceName(user)
podFullName := model.PodFullName(pod)
// Verify pod exists
_, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, podFullName, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
return ErrPodNotFound
}
return fmt.Errorf("get pod: %w", err)
}
// Delete pod
if err := c.Clientset.CoreV1().Pods(ns).Delete(ctx, podFullName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
return fmt.Errorf("delete pod: %w", err)
}
// Delete service
svcName := model.ServiceName(pod)
if err := c.Clientset.CoreV1().Services(ns).Delete(ctx, svcName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
return fmt.Errorf("delete service: %w", err)
}
// Delete ingress route and middlewares
ingressName := fmt.Sprintf("dev-pod-%s-ingress", pod)
if err := c.Dynamic.Resource(traefikIngressRouteGVR).Namespace(ns).Delete(ctx, ingressName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
return fmt.Errorf("delete ingress route: %w", err)
}
// Delete middlewares: basic-auth is shared, only delete pod-specific strip-prefix
stripName := fmt.Sprintf("strip-dev-%s-ralphex-prefix", pod)
if err := c.Dynamic.Resource(traefikMiddlewareGVR).Namespace(ns).Delete(ctx, stripName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
return fmt.Errorf("delete strip-prefix middleware: %w", err)
}
// Delete per-pod PVC
pvcName := PVCName(pod)
if err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
return fmt.Errorf("delete per-pod pvc: %w", err)
}
return nil
}
// DeleteAllPods removes all pods for a user by deleting the entire namespace.
func (c *Client) DeleteAllPods(ctx context.Context, user string) error {
return c.DeleteNamespace(ctx, user)
}
// ListPods returns all dev pods for a user with status and age.
func (c *Client) ListPods(ctx context.Context, user string) ([]model.Pod, error) {
ns := model.NamespaceName(user)
podList, err := c.Clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
LabelSelector: "app=dev-pod",
})
if err != nil {
if k8serrors.IsNotFound(err) {
return nil, nil
}
return nil, fmt.Errorf("list pods: %w", err)
}
result := make([]model.Pod, 0, len(podList.Items))
for _, p := range podList.Items {
podName := p.Labels["podname"]
cpuReq, cpuLimit, memReq, memLimit := containerResources(p.Spec.Containers)
result = append(result, model.Pod{
User: user,
Name: podName,
Tools: extractEnvVar(p, "dev", "DEV_TOOLS"),
CPUReq: cpuReq,
CPULimit: cpuLimit,
MemReq: memReq,
MemLimit: memLimit,
Task: extractEnvVar(p, "dev", "TASK_DESCRIPTION"),
Status: string(p.Status.Phase),
Age: formatAge(p.CreationTimestamp.Time),
URL: model.PodURL(c.Config.Domain, user, podName),
CreatedAt: p.CreationTimestamp.Time,
Labels: p.Labels,
})
}
return result, nil
}
// GetPod returns detailed info about a single dev pod.
func (c *Client) GetPod(ctx context.Context, user, pod string) (*model.Pod, error) {
ns := model.NamespaceName(user)
podFullName := model.PodFullName(pod)
p, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, podFullName, metav1.GetOptions{})
if err != nil {
if k8serrors.IsNotFound(err) {
return nil, ErrPodNotFound
}
return nil, fmt.Errorf("get pod: %w", err)
}
cpuReq, cpuLimit, memReq, memLimit := containerResources(p.Spec.Containers)
return &model.Pod{
User: user,
Name: pod,
Tools: extractEnvVar(*p, "dev", "DEV_TOOLS"),
CPUReq: cpuReq,
CPULimit: cpuLimit,
MemReq: memReq,
MemLimit: memLimit,
Task: extractEnvVar(*p, "dev", "TASK_DESCRIPTION"),
Status: string(p.Status.Phase),
Age: formatAge(p.CreationTimestamp.Time),
URL: model.PodURL(c.Config.Domain, user, pod),
CreatedAt: p.CreationTimestamp.Time,
Labels: p.Labels,
}, nil
}
// ensureSharedResources creates namespace-level resources idempotently.
func (c *Client) ensureSharedResources(ctx context.Context, user, vpnKey, forgejoToken, tailscaleKey string) error {
ns := model.NamespaceName(user)
// Secrets
devSec, aiSec := SecretsTemplate(user, vpnKey, c.Config.AnthropicKey, c.Config.OpenAIKey, forgejoToken, tailscaleKey)
if _, err := c.Clientset.CoreV1().Secrets(ns).Create(ctx, devSec, metav1.CreateOptions{}); err != nil {
if k8serrors.IsAlreadyExists(err) {
existing, getErr := c.Clientset.CoreV1().Secrets(ns).Get(ctx, devSec.Name, metav1.GetOptions{})
if getErr != nil {
return fmt.Errorf("get dev-secrets for update: %w", getErr)
}
devSec.ResourceVersion = existing.ResourceVersion
if _, err = c.Clientset.CoreV1().Secrets(ns).Update(ctx, devSec, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("update dev-secrets: %w", err)
}
} else {
return fmt.Errorf("create dev-secrets: %w", err)
}
}
if _, err := c.Clientset.CoreV1().Secrets(ns).Create(ctx, aiSec, metav1.CreateOptions{}); err != nil {
if k8serrors.IsAlreadyExists(err) {
existing, getErr := c.Clientset.CoreV1().Secrets(ns).Get(ctx, aiSec.Name, metav1.GetOptions{})
if getErr != nil {
return fmt.Errorf("get ai-proxy-secrets for update: %w", getErr)
}
aiSec.ResourceVersion = existing.ResourceVersion
if _, err = c.Clientset.CoreV1().Secrets(ns).Update(ctx, aiSec, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("update ai-proxy-secrets: %w", err)
}
} else {
return fmt.Errorf("create ai-proxy-secrets: %w", err)
}
}
// Basic auth secret (for Traefik basicAuth middleware)
basicAuthSec := BasicAuthSecretTemplate(user)
if _, err := c.Clientset.CoreV1().Secrets(ns).Create(ctx, basicAuthSec, metav1.CreateOptions{}); err != nil && !k8serrors.IsAlreadyExists(err) {
return fmt.Errorf("create basic-auth secret: %w", err)
}
// ConfigMap
cm := AIProxyConfigMapTemplate(user)
if _, err := c.Clientset.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{}); err != nil {
if k8serrors.IsAlreadyExists(err) {
existing, getErr := c.Clientset.CoreV1().ConfigMaps(ns).Get(ctx, cm.Name, metav1.GetOptions{})
if getErr != nil {
return fmt.Errorf("get ai-proxy-config for update: %w", getErr)
}
cm.ResourceVersion = existing.ResourceVersion
if _, err = c.Clientset.CoreV1().ConfigMaps(ns).Update(ctx, cm, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("update ai-proxy-config: %w", err)
}
} else {
return fmt.Errorf("create ai-proxy-config: %w", err)
}
}
// NetworkPolicy
np := NetworkPolicyTemplate(user)
if _, err := c.Clientset.NetworkingV1().NetworkPolicies(ns).Create(ctx, np, metav1.CreateOptions{}); err != nil && !k8serrors.IsAlreadyExists(err) {
return fmt.Errorf("create network policy: %w", err)
}
return nil
}
// traefikGVR returns the GroupVersionResource for a Traefik CRD kind.
func traefikGVR(kind string) (schema.GroupVersionResource, error) {
switch kind {
case "IngressRoute":
return traefikIngressRouteGVR, nil
case "Middleware":
return traefikMiddlewareGVR, nil
default:
return schema.GroupVersionResource{}, fmt.Errorf("unsupported traefik kind: %s", kind)
}
}
// containerResources safely extracts CPU/memory request/limit strings from the first container.
func containerResources(containers []corev1.Container) (cpuReq, cpuLimit, memReq, memLimit string) {
if len(containers) == 0 {
return "0", "0", "0", "0"
}
c := containers[0]
return c.Resources.Requests.Cpu().String(),
c.Resources.Limits.Cpu().String(),
formatMemory(c.Resources.Requests.Memory()),
formatMemory(c.Resources.Limits.Memory())
}
// extractEnvVar finds an environment variable value from a named container in a pod.
func extractEnvVar(pod corev1.Pod, containerName, envName string) string {
for _, c := range pod.Spec.Containers {
if c.Name == containerName {
for _, e := range c.Env {
if e.Name == envName {
return e.Value
}
}
}
}
return ""
}
// formatAge returns a human-readable duration since the given time.
func formatAge(created time.Time) string {
if created.IsZero() {
return ""
}
d := time.Since(created)
switch {
case d < time.Minute:
return fmt.Sprintf("%ds", int(d.Seconds()))
case d < time.Hour:
return fmt.Sprintf("%dm", int(d.Minutes()))
case d < 24*time.Hour:
return fmt.Sprintf("%dh", int(d.Hours()))
default:
return fmt.Sprintf("%dd", int(d.Hours()/24))
}
}
// formatMemory returns a human-readable memory quantity.
func formatMemory(q *resource.Quantity) string {
if q == nil || q.IsZero() {
return "0"
}
return q.String()
}

592
internal/k8s/pods_test.go Normal file
View file

@ -0,0 +1,592 @@
package k8s
import (
"context"
"errors"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
fakedynamic "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/kubernetes/fake"
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
)
// newTestClient creates a test client with fake k8s and dynamic clients,
// and pre-seeds the VPN gateway secret.
func newTestClient() *Client {
fakeClient := fake.NewSimpleClientset()
scheme := runtime.NewScheme()
fakeDyn := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme,
map[schema.GroupVersionResource]string{
traefikIngressRouteGVR: "IngressRouteList",
traefikMiddlewareGVR: "MiddlewareList",
},
)
client := NewClientWithClientset(fakeClient, fakeDyn, testCfg)
// Seed VPN gateway secret
ctx := context.Background()
vpnSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: testCfg.VPNGatewaySecret,
Namespace: testCfg.VPNGatewayNS,
},
Data: map[string][]byte{
"VPN_GATEWAY_KEY": []byte("test-vpn-key-123"),
},
}
_, _ = fakeClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: testCfg.VPNGatewayNS},
}, metav1.CreateOptions{})
_, _ = fakeClient.CoreV1().Secrets(testCfg.VPNGatewayNS).Create(ctx, vpnSecret, metav1.CreateOptions{})
return client
}
func defaultCreateOpts() CreatePodOpts {
return CreatePodOpts{
User: "alice",
Pod: "main",
Tools: "go@1.25,rust@1.94",
Task: "build a web server",
CPUReq: "2",
CPULimit: "4",
MemReq: "4Gi",
MemLimit: "8Gi",
MaxConcurrentPods: 3,
}
}
func TestFetchVPNKey(t *testing.T) {
t.Run("success", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
key, err := client.FetchVPNKey(ctx)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if key != "test-vpn-key-123" {
t.Errorf("expected test-vpn-key-123, got %s", key)
}
})
t.Run("secret_not_found", func(t *testing.T) {
fakeClient := fake.NewSimpleClientset()
fakeDyn := fakedynamic.NewSimpleDynamicClient(runtime.NewScheme())
client := NewClientWithClientset(fakeClient, fakeDyn, testCfg)
ctx := context.Background()
_, err := client.FetchVPNKey(ctx)
if err == nil {
t.Fatal("expected error for missing secret")
}
})
t.Run("key_field_missing", func(t *testing.T) {
fakeClient := fake.NewSimpleClientset()
fakeDyn := fakedynamic.NewSimpleDynamicClient(runtime.NewScheme())
client := NewClientWithClientset(fakeClient, fakeDyn, testCfg)
ctx := context.Background()
// Create secret without vpn-key field
_, _ = fakeClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{Name: testCfg.VPNGatewayNS},
}, metav1.CreateOptions{})
_, _ = fakeClient.CoreV1().Secrets(testCfg.VPNGatewayNS).Create(ctx, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: testCfg.VPNGatewaySecret,
Namespace: testCfg.VPNGatewayNS,
},
Data: map[string][]byte{"other-key": []byte("value")},
}, metav1.CreateOptions{})
_, err := client.FetchVPNKey(ctx)
if err == nil {
t.Fatal("expected error for missing vpn-key field")
}
})
}
func TestCreatePod(t *testing.T) {
t.Run("success", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
result, err := client.CreatePod(ctx, opts)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.User != "alice" {
t.Errorf("expected user alice, got %s", result.User)
}
if result.Name != "main" {
t.Errorf("expected pod name main, got %s", result.Name)
}
if result.Status != "Pending" {
t.Errorf("expected status Pending, got %s", result.Status)
}
if result.URL != "https://spinoff.dev/@alice/main/" {
t.Errorf("expected URL https://spinoff.dev/@alice/main/, got %s", result.URL)
}
if result.Tools != "go@1.25,rust@1.94" {
t.Errorf("expected tools go@1.25,rust@1.94, got %s", result.Tools)
}
// Verify k8s resources were created
ns := model.NamespaceName("alice")
// Namespace
_, err = client.Clientset.CoreV1().Namespaces().Get(ctx, ns, metav1.GetOptions{})
if err != nil {
t.Errorf("namespace not created: %v", err)
}
// Pod
_, err = client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-main", metav1.GetOptions{})
if err != nil {
t.Errorf("pod not created: %v", err)
}
// Service
_, err = client.Clientset.CoreV1().Services(ns).Get(ctx, "dev-pod-main-svc", metav1.GetOptions{})
if err != nil {
t.Errorf("service not created: %v", err)
}
// Per-pod PVC
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
if err != nil {
t.Errorf("per-pod pvc not created: %v", err)
}
// Secrets
_, err = client.Clientset.CoreV1().Secrets(ns).Get(ctx, "dev-secrets", metav1.GetOptions{})
if err != nil {
t.Errorf("dev-secrets not created: %v", err)
}
_, err = client.Clientset.CoreV1().Secrets(ns).Get(ctx, "ai-proxy-secrets", metav1.GetOptions{})
if err != nil {
t.Errorf("ai-proxy-secrets not created: %v", err)
}
// ConfigMap
_, err = client.Clientset.CoreV1().ConfigMaps(ns).Get(ctx, "ai-proxy-config", metav1.GetOptions{})
if err != nil {
t.Errorf("ai-proxy-config not created: %v", err)
}
// NetworkPolicy
_, err = client.Clientset.NetworkingV1().NetworkPolicies(ns).Get(ctx, "dev-pod-network-policy", metav1.GetOptions{})
if err != nil {
t.Errorf("network policy not created: %v", err)
}
// IngressRoute (via dynamic client)
_, err = client.Dynamic.Resource(traefikIngressRouteGVR).Namespace(ns).Get(ctx, "dev-pod-main-ingress", metav1.GetOptions{})
if err != nil {
t.Errorf("ingress route not created: %v", err)
}
// Middlewares
_, err = client.Dynamic.Resource(traefikMiddlewareGVR).Namespace(ns).Get(ctx, "spinoff-basic-auth", metav1.GetOptions{})
if err != nil {
t.Errorf("basic-auth middleware not created: %v", err)
}
_, err = client.Dynamic.Resource(traefikMiddlewareGVR).Namespace(ns).Get(ctx, "strip-dev-main-ralphex-prefix", metav1.GetOptions{})
if err != nil {
t.Errorf("strip-prefix middleware not created: %v", err)
}
})
t.Run("quota_exceeded", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
// Create 2 pods to fill quota of 2
opts := defaultCreateOpts()
opts.MaxConcurrentPods = 2
opts.Pod = "pod1"
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("first pod failed: %v", err)
}
opts.Pod = "pod2"
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("second pod failed: %v", err)
}
// Third pod should fail
opts.Pod = "pod3"
_, err := client.CreatePod(ctx, opts)
if !errors.Is(err, ErrQuotaExceeded) {
t.Errorf("expected ErrQuotaExceeded, got: %v", err)
}
})
t.Run("duplicate_pod", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("first create failed: %v", err)
}
_, err := client.CreatePod(ctx, opts)
if !errors.Is(err, ErrPodAlreadyExists) {
t.Errorf("expected ErrPodAlreadyExists, got: %v", err)
}
})
t.Run("shared_resources_idempotent", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
// Create first pod
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("first pod failed: %v", err)
}
// Create second pod for same user
opts.Pod = "secondary"
result, err := client.CreatePod(ctx, opts)
if err != nil {
t.Fatalf("second pod failed: %v", err)
}
if result.Name != "secondary" {
t.Errorf("expected pod name secondary, got %s", result.Name)
}
// Both pods should exist
ns := model.NamespaceName("alice")
pods, err := client.Clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{})
if err != nil {
t.Fatalf("failed to list pods: %v", err)
}
if len(pods.Items) != 2 {
t.Errorf("expected 2 pods, got %d", len(pods.Items))
}
// Each pod should have its own PVC
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
if err != nil {
t.Errorf("workspace-main PVC not created: %v", err)
}
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-secondary", metav1.GetOptions{})
if err != nil {
t.Errorf("workspace-secondary PVC not created: %v", err)
}
})
t.Run("secrets_contain_dummy_and_real_keys", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("create failed: %v", err)
}
ns := model.NamespaceName("alice")
// dev-secrets should have dummy keys (fake clientset preserves StringData)
devSec, _ := client.Clientset.CoreV1().Secrets(ns).Get(ctx, "dev-secrets", metav1.GetOptions{})
if devSec.StringData["ANTHROPIC_API_KEY"] != "sk-devpod" {
t.Errorf("dev-secrets should have dummy anthropic key, got %s", devSec.StringData["ANTHROPIC_API_KEY"])
}
if devSec.StringData["VPN_GATEWAY_KEY"] != "test-vpn-key-123" {
t.Errorf("dev-secrets should have VPN key, got %s", devSec.StringData["VPN_GATEWAY_KEY"])
}
// ai-proxy-secrets should have real keys
aiSec, _ := client.Clientset.CoreV1().Secrets(ns).Get(ctx, "ai-proxy-secrets", metav1.GetOptions{})
if aiSec.StringData["anthropic-key"] != "br_test_anthropic" {
t.Errorf("ai-proxy-secrets should have real anthropic key, got %s", aiSec.StringData["anthropic-key"])
}
if aiSec.StringData["openai-key"] != "br_test_openai" {
t.Errorf("ai-proxy-secrets should have real openai key, got %s", aiSec.StringData["openai-key"])
}
})
}
func TestDeletePod(t *testing.T) {
t.Run("deletes_existing_pod", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("setup: create pod failed: %v", err)
}
err := client.DeletePod(ctx, "alice", "main")
if err != nil {
t.Fatalf("delete failed: %v", err)
}
ns := model.NamespaceName("alice")
// Pod should be gone
_, err = client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-main", metav1.GetOptions{})
if err == nil {
t.Error("pod should have been deleted")
}
// Service should be gone
_, err = client.Clientset.CoreV1().Services(ns).Get(ctx, "dev-pod-main-svc", metav1.GetOptions{})
if err == nil {
t.Error("service should have been deleted")
}
// IngressRoute should be gone
_, err = client.Dynamic.Resource(traefikIngressRouteGVR).Namespace(ns).Get(ctx, "dev-pod-main-ingress", metav1.GetOptions{})
if err == nil {
t.Error("ingress route should have been deleted")
}
// Per-pod PVC should be gone
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
if err == nil {
t.Error("per-pod PVC should have been deleted")
}
// Namespace should still exist
_, err = client.Clientset.CoreV1().Namespaces().Get(ctx, ns, metav1.GetOptions{})
if err != nil {
t.Errorf("namespace should still exist: %v", err)
}
})
t.Run("not_found", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
// Create namespace so the "get pod" doesn't fail on namespace
_ = client.EnsureNamespace(ctx, "alice")
err := client.DeletePod(ctx, "alice", "nonexistent")
if !errors.Is(err, ErrPodNotFound) {
t.Errorf("expected ErrPodNotFound, got: %v", err)
}
})
t.Run("keeps_other_pods", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
// Create two pods
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("setup: create pod1 failed: %v", err)
}
opts.Pod = "secondary"
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("setup: create pod2 failed: %v", err)
}
// Delete only the first
if err := client.DeletePod(ctx, "alice", "main"); err != nil {
t.Fatalf("delete failed: %v", err)
}
ns := model.NamespaceName("alice")
// First pod gone
_, err := client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-main", metav1.GetOptions{})
if err == nil {
t.Error("first pod should be deleted")
}
// First pod's PVC gone
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
if err == nil {
t.Error("first pod's PVC should be deleted")
}
// Second pod still exists
_, err = client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-secondary", metav1.GetOptions{})
if err != nil {
t.Errorf("second pod should still exist: %v", err)
}
// Second pod's PVC still exists
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-secondary", metav1.GetOptions{})
if err != nil {
t.Errorf("second pod's PVC should still exist: %v", err)
}
})
}
func TestDeleteAllPods(t *testing.T) {
t.Run("deletes_entire_namespace", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("setup: create pod failed: %v", err)
}
if err := client.DeleteAllPods(ctx, "alice"); err != nil {
t.Fatalf("delete all failed: %v", err)
}
_, err := client.Clientset.CoreV1().Namespaces().Get(ctx, "dev-alice", metav1.GetOptions{})
if err == nil {
t.Error("namespace should have been deleted")
}
})
t.Run("idempotent_on_nonexistent", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
if err := client.DeleteAllPods(ctx, "nonexistent"); err != nil {
t.Fatalf("expected no error, got: %v", err)
}
})
}
func TestListPods(t *testing.T) {
t.Run("empty_namespace", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
pods, err := client.ListPods(ctx, "alice")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(pods) != 0 {
t.Errorf("expected 0 pods, got %d", len(pods))
}
})
t.Run("multiple_pods", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("create pod1 failed: %v", err)
}
opts.Pod = "secondary"
opts.Tools = "python@3.12"
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("create pod2 failed: %v", err)
}
pods, err := client.ListPods(ctx, "alice")
if err != nil {
t.Fatalf("list failed: %v", err)
}
if len(pods) != 2 {
t.Fatalf("expected 2 pods, got %d", len(pods))
}
// Check pod details are populated
podNames := make(map[string]bool)
for _, p := range pods {
podNames[p.Name] = true
if p.User != "alice" {
t.Errorf("expected user alice, got %s", p.User)
}
if p.URL == "" {
t.Error("expected non-empty URL")
}
}
if !podNames["main"] || !podNames["secondary"] {
t.Errorf("expected pods main and secondary, got %v", podNames)
}
})
t.Run("multiple_users_isolated", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
// Alice's pod
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("create alice pod failed: %v", err)
}
// Bob's pod
opts.User = "bob"
opts.Pod = "dev"
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("create bob pod failed: %v", err)
}
// Alice should see only her pod
alicePods, err := client.ListPods(ctx, "alice")
if err != nil {
t.Fatalf("list alice pods failed: %v", err)
}
if len(alicePods) != 1 {
t.Errorf("expected 1 pod for alice, got %d", len(alicePods))
}
// Bob should see only his pod
bobPods, err := client.ListPods(ctx, "bob")
if err != nil {
t.Fatalf("list bob pods failed: %v", err)
}
if len(bobPods) != 1 {
t.Errorf("expected 1 pod for bob, got %d", len(bobPods))
}
})
}
func TestGetPod(t *testing.T) {
t.Run("success", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
opts := defaultCreateOpts()
if _, err := client.CreatePod(ctx, opts); err != nil {
t.Fatalf("setup: create pod failed: %v", err)
}
pod, err := client.GetPod(ctx, "alice", "main")
if err != nil {
t.Fatalf("get pod failed: %v", err)
}
if pod.User != "alice" {
t.Errorf("expected user alice, got %s", pod.User)
}
if pod.Name != "main" {
t.Errorf("expected name main, got %s", pod.Name)
}
if pod.Tools != "go@1.25,rust@1.94" {
t.Errorf("expected tools go@1.25,rust@1.94, got %s", pod.Tools)
}
if pod.Task != "build a web server" {
t.Errorf("expected task, got %s", pod.Task)
}
if pod.URL != "https://spinoff.dev/@alice/main/" {
t.Errorf("expected URL, got %s", pod.URL)
}
if pod.CPUReq != "2" {
t.Errorf("expected cpu req 2, got %s", pod.CPUReq)
}
})
t.Run("not_found", func(t *testing.T) {
client := newTestClient()
ctx := context.Background()
_ = client.EnsureNamespace(ctx, "alice")
_, err := client.GetPod(ctx, "alice", "nonexistent")
if !errors.Is(err, ErrPodNotFound) {
t.Errorf("expected ErrPodNotFound, got: %v", err)
}
})
}

248
internal/k8s/runners.go Normal file
View file

@ -0,0 +1,248 @@
package k8s
import (
"context"
"fmt"
"log/slog"
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
)
// CreateRunnerPodOpts holds parameters for creating a runner pod.
type CreateRunnerPodOpts struct {
User string
RunnerID string
Tools string
Task string
RepoURL string
Branch string
CPUReq string
MemReq string
ForgejoRunnerToken string
ForgejoURL string
}
// CreateRunnerPod creates an ephemeral runner pod in the user's namespace.
func (c *Client) CreateRunnerPod(ctx context.Context, opts CreateRunnerPodOpts) (string, error) {
ns := model.NamespaceName(opts.User)
podName := model.RunnerPodName(opts.RunnerID)
cpuReq := opts.CPUReq
if cpuReq == "" {
cpuReq = "2"
}
memReq := opts.MemReq
if memReq == "" {
memReq = "4Gi"
}
forgejoURL := opts.ForgejoURL
if forgejoURL == "" {
forgejoURL = c.Config.ForgejoURL
}
devPodAPIURL := c.Config.DevPodAPIURL
if devPodAPIURL == "" {
devPodAPIURL = "http://dev-pod-api.dev-infra.svc:8080"
}
privileged := true
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: ns,
Labels: map[string]string{
"app": "dev-pod-runner",
"runner-id": opts.RunnerID,
"user": opts.User,
},
},
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyNever,
Containers: []corev1.Container{
{
Name: "runner",
Image: fmt.Sprintf("%s/%s", c.Config.Registry, c.Config.GoldenImage),
Env: []corev1.EnvVar{
{Name: "RUNNER_MODE", Value: "true"},
{Name: "RUNNER_ID", Value: opts.RunnerID},
{Name: "DEV_POD_API_URL", Value: devPodAPIURL},
{Name: "FORGEJO_URL", Value: forgejoURL},
{Name: "FORGEJO_RUNNER_TOKEN", Value: opts.ForgejoRunnerToken},
{Name: "RUNNER_REPO", Value: opts.RepoURL},
{Name: "RUNNER_BRANCH", Value: opts.Branch},
{Name: "RUNNER_TASK", Value: opts.Task},
{Name: "DEV_TOOLS", Value: opts.Tools},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(cpuReq),
corev1.ResourceMemory: resource.MustParse(memReq),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(cpuReq),
corev1.ResourceMemory: resource.MustParse(memReq),
},
},
},
{
Name: "ipip-sidecar",
Image: fmt.Sprintf("%s/claw-ipip-tunnel:dev", c.Config.Registry),
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{
Privileged: &privileged,
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"NET_ADMIN"},
},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("8Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("32Mi"),
},
},
Env: []corev1.EnvVar{
{Name: "POD_ID", Value: podName},
{Name: "VPN_GATEWAY_HOST", Value: fmt.Sprintf("vpn-gateway.%s.svc", c.Config.VPNGatewayNS)},
{
Name: "VPN_GATEWAY_KEY",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "dev-secrets"},
Key: "VPN_GATEWAY_KEY",
},
},
},
},
},
},
},
}
if _, err := c.Clientset.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil {
return "", fmt.Errorf("create runner pod: %w", err)
}
return podName, nil
}
// DeleteRunnerPod deletes a runner pod and its scratch PVC from the user's namespace.
func (c *Client) DeleteRunnerPod(ctx context.Context, user, podName string) error {
ns := model.NamespaceName(user)
if err := c.Clientset.CoreV1().Pods(ns).Delete(ctx, podName, metav1.DeleteOptions{}); err != nil {
slog.Warn("delete runner pod", "pod", podName, "ns", ns, "error", err)
}
pvcName := "workspace-" + podName
if err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil {
slog.Warn("delete runner pvc", "pvc", pvcName, "ns", ns, "error", err)
}
return nil
}
// RunnerCleanupStore is the subset of store operations needed by the cleanup goroutine.
type RunnerCleanupStore interface {
ListRunners(ctx context.Context, userFilter string, statusFilter string) ([]model.Runner, error)
UpdateRunnerStatus(ctx context.Context, id string, newStatus model.RunnerStatus, forgejoRunnerID string) error
DeleteRunner(ctx context.Context, id string) error
GetStaleRunners(ctx context.Context, ttl time.Duration) ([]model.Runner, error)
}
// RunnerCleaner polls for completed/stale runners and cleans up their k8s resources.
type RunnerCleaner struct {
k8s *Client
store RunnerCleanupStore
logger *slog.Logger
ttl time.Duration
interval time.Duration
}
// NewRunnerCleaner creates a runner cleanup goroutine.
func NewRunnerCleaner(k8s *Client, st RunnerCleanupStore, logger *slog.Logger) *RunnerCleaner {
return &RunnerCleaner{
k8s: k8s,
store: st,
logger: logger,
ttl: 2 * time.Hour,
interval: 30 * time.Second,
}
}
// Start runs the cleanup loop until the context is cancelled.
func (rc *RunnerCleaner) Start(ctx context.Context) {
rc.logger.Info("runner cleaner started", "ttl", rc.ttl, "interval", rc.interval)
ticker := time.NewTicker(rc.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
rc.logger.Info("runner cleaner stopped")
return
case <-ticker.C:
rc.cleanupCompleted(ctx)
rc.cleanupStale(ctx)
}
}
}
func (rc *RunnerCleaner) cleanupCompleted(ctx context.Context) {
for _, status := range []string{"completed", "failed"} {
runners, err := rc.store.ListRunners(ctx, "", status)
if err != nil {
rc.logger.Error("list runners for cleanup", "status", status, "error", err)
continue
}
for _, r := range runners {
rc.logger.Info("cleaning up runner", "id", r.ID, "status", r.Status, "user", r.User)
if err := rc.store.UpdateRunnerStatus(ctx, r.ID, model.RunnerStatusCleanupPending, ""); err != nil {
rc.logger.Error("mark runner cleanup_pending", "id", r.ID, "error", err)
continue
}
if r.PodName != "" {
if err := rc.k8s.DeleteRunnerPod(ctx, r.User, r.PodName); err != nil {
rc.logger.Error("delete runner pod", "id", r.ID, "pod", r.PodName, "error", err)
}
}
if err := rc.store.DeleteRunner(ctx, r.ID); err != nil {
rc.logger.Error("delete runner record", "id", r.ID, "error", err)
}
}
}
}
func (rc *RunnerCleaner) cleanupStale(ctx context.Context) {
stale, err := rc.store.GetStaleRunners(ctx, rc.ttl)
if err != nil {
rc.logger.Error("get stale runners", "error", err)
return
}
for _, r := range stale {
rc.logger.Warn("force-destroying stale runner", "id", r.ID, "status", r.Status,
"age", time.Since(r.CreatedAt).Round(time.Second))
if r.PodName != "" {
if err := rc.k8s.DeleteRunnerPod(ctx, r.User, r.PodName); err != nil {
rc.logger.Error("delete stale runner pod", "id", r.ID, "error", err)
}
}
if err := rc.store.DeleteRunner(ctx, r.ID); err != nil {
rc.logger.Error("delete stale runner record", "id", r.ID, "error", err)
}
}
}

View file

@ -0,0 +1,194 @@
package k8s
import (
"context"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
)
func defaultRunnerOpts() CreateRunnerPodOpts {
return CreateRunnerPodOpts{
User: "alice",
RunnerID: "runner-abc12345",
Tools: "go,rust",
Task: "implement feature",
RepoURL: "alice/myrepo",
Branch: "main",
CPUReq: "2",
MemReq: "4Gi",
ForgejoRunnerToken: "test-runner-token",
}
}
func TestCreateRunnerPod(t *testing.T) {
client := newTestClient()
ctx := context.Background()
if err := client.EnsureNamespace(ctx, "alice"); err != nil {
t.Fatalf("ensure namespace: %v", err)
}
opts := defaultRunnerOpts()
podName, err := client.CreateRunnerPod(ctx, opts)
if err != nil {
t.Fatalf("create runner pod: %v", err)
}
expectedPodName := model.RunnerPodName("runner-abc12345")
if podName != expectedPodName {
t.Errorf("expected pod name %q, got %q", expectedPodName, podName)
}
ns := model.NamespaceName("alice")
pod, err := client.Clientset.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{})
if err != nil {
t.Fatalf("get runner pod: %v", err)
}
if pod.Labels["app"] != "dev-pod-runner" {
t.Errorf("expected app label dev-pod-runner, got %q", pod.Labels["app"])
}
if pod.Labels["runner-id"] != "runner-abc12345" {
t.Errorf("expected runner-id label runner-abc12345, got %q", pod.Labels["runner-id"])
}
if pod.Labels["user"] != "alice" {
t.Errorf("expected user label alice, got %q", pod.Labels["user"])
}
if len(pod.Spec.Containers) != 2 {
t.Fatalf("expected 2 containers (runner + ipip-sidecar), got %d", len(pod.Spec.Containers))
}
container := pod.Spec.Containers[0]
if container.Name != "runner" {
t.Errorf("expected container name runner, got %q", container.Name)
}
sidecar := pod.Spec.Containers[1]
if sidecar.Name != "ipip-sidecar" {
t.Errorf("expected sidecar name ipip-sidecar, got %q", sidecar.Name)
}
if sidecar.SecurityContext == nil || !*sidecar.SecurityContext.Privileged {
t.Error("expected ipip-sidecar to be privileged")
}
envMap := make(map[string]string)
for _, e := range container.Env {
envMap[e.Name] = e.Value
}
if envMap["RUNNER_MODE"] != "true" {
t.Error("expected RUNNER_MODE=true")
}
if envMap["RUNNER_ID"] != "runner-abc12345" {
t.Errorf("expected RUNNER_ID=runner-abc12345, got %q", envMap["RUNNER_ID"])
}
if envMap["DEV_POD_API_URL"] == "" {
t.Error("expected DEV_POD_API_URL to be set")
}
if envMap["FORGEJO_RUNNER_TOKEN"] != "test-runner-token" {
t.Errorf("expected FORGEJO_RUNNER_TOKEN=test-runner-token, got %q", envMap["FORGEJO_RUNNER_TOKEN"])
}
if envMap["RUNNER_REPO"] != "alice/myrepo" {
t.Errorf("expected RUNNER_REPO=alice/myrepo, got %q", envMap["RUNNER_REPO"])
}
if envMap["RUNNER_BRANCH"] != "main" {
t.Errorf("expected RUNNER_BRANCH=main, got %q", envMap["RUNNER_BRANCH"])
}
if envMap["DEV_TOOLS"] != "go,rust" {
t.Errorf("expected DEV_TOOLS=go,rust, got %q", envMap["DEV_TOOLS"])
}
cpuReq := container.Resources.Requests.Cpu()
if cpuReq.String() != "2" {
t.Errorf("expected cpu request 2, got %s", cpuReq.String())
}
memReq := container.Resources.Requests.Memory()
if memReq.String() != "4Gi" {
t.Errorf("expected memory request 4Gi, got %s", memReq.String())
}
}
func TestCreateRunnerPod_DefaultResources(t *testing.T) {
client := newTestClient()
ctx := context.Background()
if err := client.EnsureNamespace(ctx, "alice"); err != nil {
t.Fatalf("ensure namespace: %v", err)
}
opts := CreateRunnerPodOpts{
User: "alice",
RunnerID: "runner-defaults",
RepoURL: "alice/repo",
}
podName, err := client.CreateRunnerPod(ctx, opts)
if err != nil {
t.Fatalf("create runner pod: %v", err)
}
ns := model.NamespaceName("alice")
pod, err := client.Clientset.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{})
if err != nil {
t.Fatalf("get runner pod: %v", err)
}
container := pod.Spec.Containers[0]
cpuReq := container.Resources.Requests.Cpu()
if cpuReq.String() != "2" {
t.Errorf("expected default cpu request 2, got %s", cpuReq.String())
}
memReq := container.Resources.Requests.Memory()
if memReq.String() != "4Gi" {
t.Errorf("expected default memory request 4Gi, got %s", memReq.String())
}
}
func TestDeleteRunnerPod(t *testing.T) {
client := newTestClient()
ctx := context.Background()
if err := client.EnsureNamespace(ctx, "alice"); err != nil {
t.Fatalf("ensure namespace: %v", err)
}
opts := defaultRunnerOpts()
podName, err := client.CreateRunnerPod(ctx, opts)
if err != nil {
t.Fatalf("create runner pod: %v", err)
}
if err := client.DeleteRunnerPod(ctx, "alice", podName); err != nil {
t.Fatalf("delete runner pod: %v", err)
}
ns := model.NamespaceName("alice")
pods, err := client.Clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
LabelSelector: "app=dev-pod-runner",
})
if err != nil {
t.Fatalf("list pods: %v", err)
}
if len(pods.Items) != 0 {
t.Errorf("expected 0 runner pods after delete, got %d", len(pods.Items))
}
}
func TestDeleteRunnerPod_NonExistent(t *testing.T) {
client := newTestClient()
ctx := context.Background()
err := client.DeleteRunnerPod(ctx, "alice", "nonexistent-pod")
if err != nil {
t.Fatalf("delete nonexistent pod should not fail, got: %v", err)
}
}
func TestRunnerPodName(t *testing.T) {
name := model.RunnerPodName("runner-abc123")
if name != "dev-pod-runner-abc123" {
t.Errorf("expected dev-pod-runner-abc123, got %s", name)
}
}

433
internal/k8s/sessionhost.go Normal file
View file

@ -0,0 +1,433 @@
package k8s
import (
"context"
"fmt"
"time"
corev1 "k8s.io/api/core/v1"
netv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/intstr"
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
)
// SessionHostSpec captures the defaults + per-install knobs. The values
// are chosen to match web-tui's k8s/session-host manifests.
type SessionHostSpec struct {
// Image is the dev-container image (golden image). Defaults to the
// value baked into the ConfigFromEnv loader.
Image string
// PortWatchImage is the sidecar image built from web-tui/cmd/port-watch.
PortWatchImage string
// ApexDomain is the root domain used to form per-user hostnames
// (e.g. "spinoff.dev" → "*.alice.spinoff.dev").
ApexDomain string
// ClusterIssuer is the cert-manager ClusterIssuer used to obtain the
// per-user wildcard cert via DNS-01 (e.g. "letsencrypt-cloudflare-dns01").
ClusterIssuer string
// GatewayService is the in-cluster Service name that the per-user
// ingress forwards <port>.<user>.<apex>/* traffic to.
GatewayService string
// GatewayServiceNamespace is where GatewayService lives (e.g. "web-tui").
GatewayServiceNamespace string
}
// EnsureSessionHost is idempotent: guarantees the user has a
// long-lived session-host pod + supporting PVCs, Service, Ingress, and
// per-user wildcard cert. Re-running on an already-provisioned user is
// a no-op that reports current status.
func (c *Client) EnsureSessionHost(ctx context.Context, user string, spec SessionHostSpec) (SessionHostStatus, error) {
ns := model.NamespaceName(user)
if err := c.EnsureNamespace(ctx, user); err != nil {
return SessionHostStatus{}, err
}
if err := c.ensureSessionHostPVCs(ctx, ns); err != nil {
return SessionHostStatus{}, err
}
if err := c.ensureSessionHostServiceAccount(ctx, ns); err != nil {
return SessionHostStatus{}, err
}
pod, err := c.ensureSessionHostPod(ctx, ns, user, spec)
if err != nil {
return SessionHostStatus{}, err
}
if err := c.ensureSessionHostService(ctx, ns); err != nil {
return SessionHostStatus{}, err
}
if err := c.ensureSessionHostCertificate(ctx, ns, user, spec); err != nil {
return SessionHostStatus{}, err
}
if err := c.ensureSessionHostIngress(ctx, ns, user, spec); err != nil {
return SessionHostStatus{}, err
}
return c.readSessionHostStatus(ctx, ns, user, pod)
}
// SessionHostStatusForUser returns the current status of the user's
// session-host pod without creating anything new.
func (c *Client) SessionHostStatusForUser(ctx context.Context, user string) (SessionHostStatus, error) {
ns := model.NamespaceName(user)
pod, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, "session-host", metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
return SessionHostStatus{Ready: false}, nil
}
return SessionHostStatus{}, fmt.Errorf("get session-host pod: %w", err)
}
return c.readSessionHostStatus(ctx, ns, user, pod)
}
// ---- helpers ----
const (
sessionHostPodName = "session-host"
sessionHostServiceName = "session-host"
sessionHostSA = "session-host"
workspacePVC = "workspace"
dotfilesPVC = "dotfiles"
)
func (c *Client) ensureSessionHostPVCs(ctx context.Context, ns string) error {
pvcs := []struct {
name string
size string
}{
{workspacePVC, "20Gi"},
{dotfilesPVC, "1Gi"},
}
for _, p := range pvcs {
want := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Name: p.name},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
Resources: corev1.VolumeResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: resource.MustParse(p.size),
},
},
},
}
_, err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Create(ctx, want, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("create pvc %s/%s: %w", ns, p.name, err)
}
}
return nil
}
func (c *Client) ensureSessionHostServiceAccount(ctx context.Context, ns string) error {
sa := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{Name: sessionHostSA},
}
_, err := c.Clientset.CoreV1().ServiceAccounts(ns).Create(ctx, sa, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("create sa %s/%s: %w", ns, sessionHostSA, err)
}
return nil
}
func (c *Client) ensureSessionHostPod(ctx context.Context, ns, user string, spec SessionHostSpec) (*corev1.Pod, error) {
existing, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, sessionHostPodName, metav1.GetOptions{})
if err == nil {
return existing, nil
}
if !errors.IsNotFound(err) {
return nil, fmt.Errorf("get pod %s/%s: %w", ns, sessionHostPodName, err)
}
pod := sessionHostPodManifest(user, spec)
pod, err = c.Clientset.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{})
if err != nil {
if errors.IsAlreadyExists(err) {
return c.Clientset.CoreV1().Pods(ns).Get(ctx, sessionHostPodName, metav1.GetOptions{})
}
return nil, fmt.Errorf("create pod %s/%s: %w", ns, sessionHostPodName, err)
}
return pod, nil
}
func sessionHostPodManifest(user string, spec SessionHostSpec) *corev1.Pod {
one := int64(1000)
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: sessionHostPodName,
Labels: map[string]string{
"app.kubernetes.io/name": "session-host",
"app.kubernetes.io/component": "dev-pod",
"web-tui.spinoff.dev/user": user,
"managed-by": "dev-pod-api",
},
},
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyAlways,
ServiceAccountName: sessionHostSA,
SecurityContext: &corev1.PodSecurityContext{FSGroup: &one},
Containers: []corev1.Container{
{
Name: "dev",
Image: spec.Image,
ImagePullPolicy: corev1.PullIfNotPresent,
Command: []string{"/usr/local/bin/dev-entrypoint"},
TTY: true,
Stdin: true,
Env: []corev1.EnvVar{
{Name: "HOME", Value: "/home/dev"},
{Name: "USER", Value: "dev"},
{Name: "WEBTUI_USER", Value: user},
},
VolumeMounts: []corev1.VolumeMount{
{Name: "workspace", MountPath: "/home/dev/workspace"},
{Name: "dotfiles", MountPath: "/home/dev/.config"},
{Name: "tmp", MountPath: "/tmp"},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("250m"),
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
},
{
Name: "port-watch",
Image: spec.PortWatchImage,
ImagePullPolicy: corev1.PullIfNotPresent,
Args: []string{"-listen=:9100", "-uploads=/home/dev/uploads"},
Ports: []corev1.ContainerPort{
{ContainerPort: 9100, Name: "portwatch"},
},
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromInt32(9100)},
},
PeriodSeconds: 5,
},
VolumeMounts: []corev1.VolumeMount{
{Name: "workspace", MountPath: "/home/dev/workspace", ReadOnly: true},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("20m"),
corev1.ResourceMemory: resource.MustParse("32Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("64Mi"),
},
},
},
},
Volumes: []corev1.Volume{
{Name: "workspace", VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: workspacePVC}}},
{Name: "dotfiles", VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: dotfilesPVC}}},
{Name: "tmp", VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: ptrQuantity("1Gi")}}},
},
},
}
}
func (c *Client) ensureSessionHostService(ctx context.Context, ns string) error {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{Name: sessionHostServiceName},
Spec: corev1.ServiceSpec{
Selector: map[string]string{"app.kubernetes.io/name": "session-host"},
Ports: []corev1.ServicePort{
{Name: "portwatch", Port: 9100, TargetPort: intstr.FromInt32(9100)},
},
},
}
_, err := c.Clientset.CoreV1().Services(ns).Create(ctx, svc, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("create svc: %w", err)
}
return nil
}
func (c *Client) ensureSessionHostCertificate(ctx context.Context, ns, user string, spec SessionHostSpec) error {
name := "wildcard-" + user
gvr := schema.GroupVersionResource{Group: "cert-manager.io", Version: "v1", Resource: "certificates"}
wildcardHost := "*." + user + "." + spec.ApexDomain
obj := &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "cert-manager.io/v1",
"kind": "Certificate",
"metadata": map[string]any{
"name": name,
},
"spec": map[string]any{
"secretName": name + "-tls",
"dnsNames": []string{wildcardHost},
"duration": "2160h",
"renewBefore": "720h",
"privateKey": map[string]any{"algorithm": "ECDSA", "size": int64(256)},
"issuerRef": map[string]any{
"name": spec.ClusterIssuer,
"kind": "ClusterIssuer",
"group": "cert-manager.io",
},
},
}}
_, err := c.Dynamic.Resource(gvr).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("create certificate: %w", err)
}
return nil
}
func (c *Client) ensureSessionHostIngress(ctx context.Context, ns, user string, spec SessionHostSpec) error {
host := "*." + user + "." + spec.ApexDomain
pathType := netv1.PathTypePrefix
className := "traefik"
ing := &netv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "preview-" + user,
Annotations: map[string]string{
"cert-manager.io/cluster-issuer": spec.ClusterIssuer,
"traefik.ingress.kubernetes.io/router.tls": "true",
},
},
Spec: netv1.IngressSpec{
IngressClassName: &className,
TLS: []netv1.IngressTLS{
{Hosts: []string{host}, SecretName: "wildcard-" + user + "-tls"},
},
Rules: []netv1.IngressRule{
{
Host: host,
IngressRuleValue: netv1.IngressRuleValue{
HTTP: &netv1.HTTPIngressRuleValue{
Paths: []netv1.HTTPIngressPath{
{
Path: "/",
PathType: &pathType,
Backend: netv1.IngressBackend{
Service: &netv1.IngressServiceBackend{
Name: spec.GatewayService,
Port: netv1.ServiceBackendPort{Name: "http"},
},
},
},
},
},
},
},
},
},
}
_, err := c.Clientset.NetworkingV1().Ingresses(ns).Create(ctx, ing, metav1.CreateOptions{})
if err != nil && !errors.IsAlreadyExists(err) {
return fmt.Errorf("create ingress: %w", err)
}
return nil
}
func (c *Client) readSessionHostStatus(ctx context.Context, ns, user string, pod *corev1.Pod) (SessionHostStatus, error) {
ready := pod.Status.Phase == corev1.PodRunning && allContainersReady(pod)
certReady, certReason := c.readCertStatus(ctx, ns, "wildcard-"+user)
return SessionHostStatus{
Ready: ready,
PodName: pod.Name,
Namespace: ns,
CertReady: certReady,
CertPendingReason: certReason,
CreatedAt: pod.CreationTimestamp.Time,
}, nil
}
func (c *Client) readCertStatus(ctx context.Context, ns, name string) (bool, string) {
gvr := schema.GroupVersionResource{Group: "cert-manager.io", Version: "v1", Resource: "certificates"}
obj, err := c.Dynamic.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return false, "not-found"
}
conds, ok, _ := unstructuredNested(obj.Object, "status", "conditions")
if !ok {
return false, "no-status"
}
list, ok := conds.([]any)
if !ok {
return false, "bad-status"
}
for _, c := range list {
m, ok := c.(map[string]any)
if !ok {
continue
}
if m["type"] == "Ready" {
if m["status"] == "True" {
return true, ""
}
if reason, ok := m["reason"].(string); ok {
return false, reason
}
return false, "not-ready"
}
}
return false, "pending"
}
// SessionHostStatus is duplicated here so the k8s package doesn't need
// to import the api package (avoids an import cycle). The api package
// mirrors the fields.
type SessionHostStatus struct {
Ready bool
PodName string
Namespace string
CertReady bool
CertPendingReason string
AtchSessionCount int
CreatedAt time.Time
}
// ---- tiny helpers ----
func allContainersReady(pod *corev1.Pod) bool {
if len(pod.Status.ContainerStatuses) == 0 {
return false
}
for _, cs := range pod.Status.ContainerStatuses {
if !cs.Ready {
return false
}
}
return true
}
func unstructuredNested(obj map[string]any, keys ...string) (any, bool, error) {
cur := any(obj)
for _, k := range keys {
m, ok := cur.(map[string]any)
if !ok {
return nil, false, nil
}
v, ok := m[k]
if !ok {
return nil, false, nil
}
cur = v
}
return cur, true, nil
}
func ptrQuantity(s string) *resource.Quantity {
q := resource.MustParse(s)
return &q
}

599
internal/k8s/templates.go Normal file
View file

@ -0,0 +1,599 @@
package k8s
import (
"fmt"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/intstr"
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
)
// PodOpts holds all parameters needed to create a dev pod and its resources.
type PodOpts struct {
User string
Pod string
Tools string
Task string
CPUReq string
CPULimit string
MemReq string
MemLimit string
VPNKey string
AnthropicKey string
OpenAIKey string
ForgejoToken string
TailscaleKey string
}
// PVCName returns the per-pod PVC name for a given pod.
func PVCName(pod string) string {
return fmt.Sprintf("workspace-%s", pod)
}
// PVCTemplate returns a PersistentVolumeClaim for a specific pod's workspace.
func PVCTemplate(user, pod string) *corev1.PersistentVolumeClaim {
storageClass := "longhorn"
return &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: PVCName(pod),
Namespace: model.NamespaceName(user),
Labels: map[string]string{
"app": "dev-pod",
"podname": pod,
},
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
StorageClassName: &storageClass,
Resources: corev1.VolumeResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: resource.MustParse("20Gi"),
},
},
},
}
}
// PodTemplate returns a Pod spec matching the YAML pod-template.yaml.
func PodTemplate(cfg Config, opts PodOpts) *corev1.Pod {
ns := model.NamespaceName(opts.User)
podName := model.PodFullName(opts.Pod)
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: ns,
Labels: map[string]string{
"app": "dev-pod",
"podname": opts.Pod,
},
},
Spec: corev1.PodSpec{
HostAliases: []corev1.HostAlias{
{
IP: "127.0.0.1",
Hostnames: []string{"anthropic.internal", "openai.internal"},
},
},
Containers: []corev1.Container{
devContainer(cfg, opts),
aiProxyContainer(),
ipipSidecarContainer(cfg, opts),
},
Volumes: []corev1.Volume{
{
Name: "workspace",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: PVCName(opts.Pod),
},
},
},
{
Name: "ai-proxy-config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "ai-proxy-config",
},
DefaultMode: int32Ptr(0755),
},
},
},
{
Name: "ai-proxy-secrets",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "ai-proxy-secrets",
},
},
},
},
},
}
}
func devContainer(cfg Config, opts PodOpts) corev1.Container {
return corev1.Container{
Name: "dev",
Image: fmt.Sprintf("%s/%s", cfg.Registry, cfg.GoldenImage),
ImagePullPolicy: corev1.PullAlways,
Ports: []corev1.ContainerPort{
{ContainerPort: 7681, Name: "ttyd"},
{ContainerPort: 22, Name: "ssh"},
{ContainerPort: 3000, Name: "forgejo"},
{ContainerPort: 8080, Name: "vscode"},
{ContainerPort: 8090, Name: "ralphex-gerrit"},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(opts.CPUReq),
corev1.ResourceMemory: resource.MustParse(opts.MemReq),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(opts.CPULimit),
corev1.ResourceMemory: resource.MustParse(opts.MemLimit),
},
},
EnvFrom: []corev1.EnvFromSource{
{
SecretRef: &corev1.SecretEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "dev-secrets",
},
},
},
},
Env: []corev1.EnvVar{
{Name: "TASK_DESCRIPTION", Value: opts.Task},
{Name: "DEV_TOOLS", Value: opts.Tools},
{Name: "DEV_BASE_PATH", Value: fmt.Sprintf("/@%s/%s", opts.User, opts.Pod)},
{Name: "DEV_EXTERNAL_HOST", Value: cfg.Domain},
{Name: "TTYD_BASE_PATH", Value: fmt.Sprintf("/@%s/%s", opts.User, opts.Pod)},
{Name: "FORGEJO_URL", Value: cfg.ForgejoURL},
},
VolumeMounts: []corev1.VolumeMount{
{Name: "workspace", MountPath: "/home/dev/workspace"},
},
}
}
func aiProxyContainer() corev1.Container {
return corev1.Container{
Name: "ai-proxy",
Image: "nginx:alpine",
Command: []string{"/bin/sh", "/etc/nginx/templates/entrypoint.sh"},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("16Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("32Mi"),
},
},
VolumeMounts: []corev1.VolumeMount{
{Name: "ai-proxy-config", MountPath: "/etc/nginx/templates", ReadOnly: true},
{Name: "ai-proxy-secrets", MountPath: "/secrets", ReadOnly: true},
},
}
}
func ipipSidecarContainer(cfg Config, opts PodOpts) corev1.Container {
return corev1.Container{
Name: "ipip-sidecar",
Image: fmt.Sprintf("%s/claw-ipip-tunnel:dev", cfg.Registry),
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{
Privileged: boolPtr(true),
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"NET_ADMIN"},
},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("8Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("32Mi"),
},
},
Env: []corev1.EnvVar{
{Name: "POD_ID", Value: fmt.Sprintf("dev-%s-%s", opts.User, opts.Pod)},
{Name: "VPN_GATEWAY_HOST", Value: fmt.Sprintf("vpn-gateway.%s.svc", cfg.VPNGatewayNS)},
{
Name: "VPN_GATEWAY_KEY",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "dev-secrets"},
Key: "VPN_GATEWAY_KEY",
},
},
},
},
}
}
// ServiceTemplate returns a Service for the dev pod matching service.yaml.
func ServiceTemplate(user, pod string) *corev1.Service {
ns := model.NamespaceName(user)
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: model.ServiceName(pod),
Namespace: ns,
Labels: map[string]string{
"app": "dev-pod",
"podname": pod,
},
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Ports: []corev1.ServicePort{
{Name: "ttyd", Port: 7681, TargetPort: intstr.FromInt32(7681), Protocol: corev1.ProtocolTCP},
{Name: "ssh", Port: 22, TargetPort: intstr.FromInt32(22), Protocol: corev1.ProtocolTCP},
{Name: "forgejo", Port: 3000, TargetPort: intstr.FromInt32(3000), Protocol: corev1.ProtocolTCP},
{Name: "vscode", Port: 8080, TargetPort: intstr.FromInt32(8080), Protocol: corev1.ProtocolTCP},
{Name: "ralphex-gerrit", Port: 8090, TargetPort: intstr.FromInt32(8090), Protocol: corev1.ProtocolTCP},
},
Selector: map[string]string{
"app": "dev-pod",
"podname": pod,
},
},
}
}
// IngressTemplate returns Traefik IngressRoute as an unstructured object matching ingress.yaml.
// Returns the IngressRoute and two Middleware objects.
func IngressTemplate(user, pod, domain string) []*unstructured.Unstructured {
ns := model.NamespaceName(user)
svcName := model.ServiceName(pod)
ingressRoute := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "traefik.io/v1alpha1",
"kind": "IngressRoute",
"metadata": map[string]interface{}{
"name": fmt.Sprintf("dev-pod-%s-ingress", pod),
"namespace": ns,
},
"spec": map[string]interface{}{
"entryPoints": []interface{}{"web", "websecure"},
// No TLS section: Caddy terminates TLS and forwards HTTP to Traefik.
// With TLS here, Traefik v3 only routes on websecure, breaking Caddy -> port 80 flow.
// No explicit priority: Traefik v3 auto-calculates from rule length,
// ensuring these beat the OpenClaw catch-all (Host:* / PathPrefix:/).
"routes": []interface{}{
// ttyd — base route
map[string]interface{}{
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/`)", domain, user, pod),
"kind": "Rule",
"middlewares": []interface{}{
map[string]interface{}{"name": "spinoff-basic-auth"},
},
"services": []interface{}{
map[string]interface{}{"name": svcName, "port": int64(7681)},
},
},
// Forgejo
map[string]interface{}{
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/forgejo/`)", domain, user, pod),
"kind": "Rule",
"middlewares": []interface{}{
map[string]interface{}{"name": "spinoff-basic-auth"},
},
"services": []interface{}{
map[string]interface{}{"name": svcName, "port": int64(3000)},
},
},
// VS Code
map[string]interface{}{
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/vscode/`)", domain, user, pod),
"kind": "Rule",
"middlewares": []interface{}{
map[string]interface{}{"name": "spinoff-basic-auth"},
},
"services": []interface{}{
map[string]interface{}{"name": svcName, "port": int64(8080)},
},
},
// Gerrit code review (handles own base path, shares port 8090 with ralphex)
map[string]interface{}{
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/gerrit/`)", domain, user, pod),
"kind": "Rule",
"middlewares": []interface{}{
map[string]interface{}{"name": "spinoff-basic-auth"},
},
"services": []interface{}{
map[string]interface{}{"name": svcName, "port": int64(8090)},
},
},
// Ralphex dashboard (with prefix stripping)
map[string]interface{}{
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/ralphex/`)", domain, user, pod),
"kind": "Rule",
"middlewares": []interface{}{
map[string]interface{}{"name": "spinoff-basic-auth"},
map[string]interface{}{"name": fmt.Sprintf("strip-dev-%s-ralphex-prefix", pod)},
},
"services": []interface{}{
map[string]interface{}{"name": svcName, "port": int64(8090)},
},
},
},
},
},
}
basicAuthMiddleware := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "traefik.io/v1alpha1",
"kind": "Middleware",
"metadata": map[string]interface{}{
"name": "spinoff-basic-auth",
"namespace": ns,
},
"spec": map[string]interface{}{
"basicAuth": map[string]interface{}{
"secret": "spinoff-basic-auth",
},
},
},
}
stripPrefixMiddleware := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "traefik.io/v1alpha1",
"kind": "Middleware",
"metadata": map[string]interface{}{
"name": fmt.Sprintf("strip-dev-%s-ralphex-prefix", pod),
"namespace": ns,
},
"spec": map[string]interface{}{
"stripPrefix": map[string]interface{}{
"prefixes": []interface{}{
fmt.Sprintf("/@%s/%s/ralphex", user, pod),
},
},
},
},
}
return []*unstructured.Unstructured{ingressRoute, basicAuthMiddleware, stripPrefixMiddleware}
}
// SecretsTemplate returns dev-secrets and ai-proxy-secrets matching secrets-template.yaml.
func SecretsTemplate(user, vpnKey, anthropicKey, openaiKey, forgejoToken, tailscaleKey string) (*corev1.Secret, *corev1.Secret) {
ns := model.NamespaceName(user)
stringData := map[string]string{
"ANTHROPIC_API_KEY": "sk-devpod",
"ANTHROPIC_BASE_URL": "http://anthropic.internal",
"BASEROUTE_OPENAI_KEY": "sk-devpod",
"VPN_GATEWAY_KEY": vpnKey,
"FORGEJO_TOKEN": forgejoToken,
}
if tailscaleKey != "" {
stringData["TAILSCALE_AUTHKEY"] = tailscaleKey
}
devSecrets := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "dev-secrets",
Namespace: ns,
},
Type: corev1.SecretTypeOpaque,
StringData: stringData,
}
aiProxySecrets := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "ai-proxy-secrets",
Namespace: ns,
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{
"anthropic-key": anthropicKey,
"openai-key": openaiKey,
},
}
return devSecrets, aiProxySecrets
}
// BasicAuthSecretTemplate returns the spinoff-basic-auth Secret containing htpasswd data.
// Matches the secret created by dev-pod-create.sh for Traefik basicAuth middleware.
func BasicAuthSecretTemplate(user string) *corev1.Secret {
ns := model.NamespaceName(user)
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "spinoff-basic-auth",
Namespace: ns,
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{
"users": "admin:$2y$05$i2Wxz4GpO8.jnMXWk1len.xcP0.wPSL.ozfd/OgX8BCo9chO8F2WO\n",
},
}
}
// NetworkPolicyTemplate returns a NetworkPolicy matching network-policy.yaml.
func NetworkPolicyTemplate(user string) *networkingv1.NetworkPolicy {
ns := model.NamespaceName(user)
dnsPort := intstr.FromInt32(53)
return &networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: "dev-pod-network-policy",
Namespace: ns,
},
Spec: networkingv1.NetworkPolicySpec{
PodSelector: metav1.LabelSelector{},
PolicyTypes: []networkingv1.PolicyType{
networkingv1.PolicyTypeIngress,
networkingv1.PolicyTypeEgress,
},
Ingress: []networkingv1.NetworkPolicyIngressRule{
{
From: []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"kubernetes.io/metadata.name": "kube-system",
},
},
},
},
},
},
Egress: []networkingv1.NetworkPolicyEgressRule{
// VPN gateway in claw-system
{
To: []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"kubernetes.io/metadata.name": "claw-system",
},
},
PodSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "vpn-gateway",
},
},
},
},
},
// dev-infra namespace (cache services)
{
To: []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"kubernetes.io/metadata.name": "dev-infra",
},
},
},
},
},
// DNS resolution via kube-system
{
To: []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"kubernetes.io/metadata.name": "kube-system",
},
},
},
},
Ports: []networkingv1.NetworkPolicyPort{
{Protocol: protocolPtr(corev1.ProtocolUDP), Port: &dnsPort},
{Protocol: protocolPtr(corev1.ProtocolTCP), Port: &dnsPort},
},
},
},
},
}
}
// AIProxyConfigMapTemplate returns the ai-proxy-config ConfigMap matching ai-proxy-config.yaml.
func AIProxyConfigMapTemplate(user string) *corev1.ConfigMap {
ns := model.NamespaceName(user)
nginxConfTemplate := `events {
worker_connections 64;
}
http {
resolver %%RESOLVER%% valid=30s;
server {
listen 80;
server_name anthropic.internal;
client_max_body_size 50m;
location / {
set $upstream https://anthropic.baseroute.tech;
proxy_pass $upstream;
proxy_ssl_server_name on;
proxy_set_header Host anthropic.baseroute.tech;
proxy_set_header x-api-key "%%ANTHROPIC_KEY%%";
proxy_set_header Authorization "";
proxy_buffering off;
proxy_read_timeout 300s;
proxy_connect_timeout 10s;
}
}
server {
listen 80;
server_name openai.internal;
client_max_body_size 50m;
location / {
set $upstream https://openai.baseroute.tech;
proxy_pass $upstream;
proxy_ssl_server_name on;
proxy_set_header Host openai.baseroute.tech;
proxy_set_header Authorization "Bearer %%OPENAI_KEY%%";
proxy_set_header x-api-key "";
proxy_buffering off;
proxy_read_timeout 300s;
proxy_connect_timeout 10s;
}
}
}
`
entrypointSh := `#!/bin/sh
set -e
# Escape sed special chars (& \ |) in secret values to prevent replacement corruption
escape_sed() { printf '%s\n' "$1" | sed 's/[&\|/]/\\&/g'; }
ANTHROPIC_KEY=$(escape_sed "$(cat /secrets/anthropic-key)")
OPENAI_KEY=$(escape_sed "$(cat /secrets/openai-key)")
RESOLVER=$(awk '/^nameserver/{print $2; exit}' /etc/resolv.conf)
sed \
-e "s|%%ANTHROPIC_KEY%%|${ANTHROPIC_KEY}|g" \
-e "s|%%OPENAI_KEY%%|${OPENAI_KEY}|g" \
-e "s|%%RESOLVER%%|${RESOLVER}|g" \
/etc/nginx/templates/nginx.conf.template > /etc/nginx/nginx.conf
exec nginx -g 'daemon off;'
`
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "ai-proxy-config",
Namespace: ns,
},
Data: map[string]string{
"nginx.conf.template": nginxConfTemplate,
"entrypoint.sh": entrypointSh,
},
}
}
func boolPtr(b bool) *bool {
return &b
}
func int32Ptr(i int32) *int32 {
return &i
}
func protocolPtr(p corev1.Protocol) *corev1.Protocol {
return &p
}

View file

@ -0,0 +1,540 @@
package k8s
import (
"testing"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
)
var testCfg = Config{
Domain: "spinoff.dev",
Registry: "10.22.0.56:30500",
GoldenImage: "dev-golden:v2",
VPNGatewayNS: "claw-system",
VPNGatewaySecret: "vpn-gateway-secrets",
AnthropicKey: "br_test_anthropic",
OpenAIKey: "br_test_openai",
ForgejoURL: "http://forgejo.dev-infra.svc:3000",
}
var testOpts = PodOpts{
User: "alice",
Pod: "main",
Tools: "go@1.25,rust@1.94",
Task: "build a web server",
CPUReq: "2",
CPULimit: "4",
MemReq: "4Gi",
MemLimit: "8Gi",
VPNKey: "test-vpn-key",
AnthropicKey: "br_test_anthropic",
OpenAIKey: "br_test_openai",
}
func TestPVCTemplate(t *testing.T) {
pvc := PVCTemplate("alice", "main")
if pvc.Name != "workspace-main" {
t.Errorf("expected name workspace-main, got %s", pvc.Name)
}
if pvc.Namespace != "dev-alice" {
t.Errorf("expected namespace dev-alice, got %s", pvc.Namespace)
}
if len(pvc.Spec.AccessModes) != 1 || pvc.Spec.AccessModes[0] != corev1.ReadWriteOnce {
t.Errorf("expected ReadWriteOnce access mode")
}
if *pvc.Spec.StorageClassName != "longhorn" {
t.Errorf("expected longhorn storage class, got %s", *pvc.Spec.StorageClassName)
}
storage := pvc.Spec.Resources.Requests[corev1.ResourceStorage]
if storage.Cmp(resource.MustParse("20Gi")) != 0 {
t.Errorf("expected 20Gi storage, got %s", storage.String())
}
if pvc.Labels["app"] != "dev-pod" {
t.Errorf("expected label app=dev-pod, got %s", pvc.Labels["app"])
}
if pvc.Labels["podname"] != "main" {
t.Errorf("expected label podname=main, got %s", pvc.Labels["podname"])
}
}
func TestPVCTemplatePerPod(t *testing.T) {
tests := []struct {
user string
pod string
wantPVC string
}{
{"alice", "main", "workspace-main"},
{"bob", "build1", "workspace-build1"},
{"alice", "runner-42", "workspace-runner-42"},
}
for _, tt := range tests {
t.Run(tt.user+"/"+tt.pod, func(t *testing.T) {
pvc := PVCTemplate(tt.user, tt.pod)
if pvc.Name != tt.wantPVC {
t.Errorf("expected PVC name %s, got %s", tt.wantPVC, pvc.Name)
}
if pvc.Labels["podname"] != tt.pod {
t.Errorf("expected podname label %s, got %s", tt.pod, pvc.Labels["podname"])
}
})
}
}
func TestPVCName(t *testing.T) {
if got := PVCName("main"); got != "workspace-main" {
t.Errorf("expected workspace-main, got %s", got)
}
if got := PVCName("build1"); got != "workspace-build1" {
t.Errorf("expected workspace-build1, got %s", got)
}
}
func TestPodTemplate(t *testing.T) {
pod := PodTemplate(testCfg, testOpts)
t.Run("metadata", func(t *testing.T) {
if pod.Name != "dev-pod-main" {
t.Errorf("expected name dev-pod-main, got %s", pod.Name)
}
if pod.Namespace != "dev-alice" {
t.Errorf("expected namespace dev-alice, got %s", pod.Namespace)
}
if pod.Labels["app"] != "dev-pod" {
t.Errorf("expected label app=dev-pod, got %s", pod.Labels["app"])
}
if pod.Labels["podname"] != "main" {
t.Errorf("expected label podname=main, got %s", pod.Labels["podname"])
}
})
t.Run("host_aliases", func(t *testing.T) {
if len(pod.Spec.HostAliases) != 1 {
t.Fatalf("expected 1 host alias, got %d", len(pod.Spec.HostAliases))
}
ha := pod.Spec.HostAliases[0]
if ha.IP != "127.0.0.1" {
t.Errorf("expected IP 127.0.0.1, got %s", ha.IP)
}
if len(ha.Hostnames) != 2 {
t.Fatalf("expected 2 hostnames, got %d", len(ha.Hostnames))
}
if ha.Hostnames[0] != "anthropic.internal" || ha.Hostnames[1] != "openai.internal" {
t.Errorf("unexpected hostnames: %v", ha.Hostnames)
}
})
t.Run("containers_count", func(t *testing.T) {
if len(pod.Spec.Containers) != 3 {
t.Fatalf("expected 3 containers, got %d", len(pod.Spec.Containers))
}
})
t.Run("dev_container", func(t *testing.T) {
dev := pod.Spec.Containers[0]
if dev.Name != "dev" {
t.Errorf("expected container name dev, got %s", dev.Name)
}
if dev.Image != "10.22.0.56:30500/dev-golden:v2" {
t.Errorf("expected golden image, got %s", dev.Image)
}
if dev.ImagePullPolicy != corev1.PullAlways {
t.Errorf("expected PullAlways, got %s", dev.ImagePullPolicy)
}
if len(dev.Ports) != 5 {
t.Errorf("expected 5 ports, got %d", len(dev.Ports))
}
// Check resource requests
cpuReq := dev.Resources.Requests[corev1.ResourceCPU]
if cpuReq.Cmp(resource.MustParse("2")) != 0 {
t.Errorf("expected cpu req 2, got %s", cpuReq.String())
}
memLim := dev.Resources.Limits[corev1.ResourceMemory]
if memLim.Cmp(resource.MustParse("8Gi")) != 0 {
t.Errorf("expected mem limit 8Gi, got %s", memLim.String())
}
// Check env vars
envMap := make(map[string]string)
for _, e := range dev.Env {
envMap[e.Name] = e.Value
}
if envMap["TASK_DESCRIPTION"] != "build a web server" {
t.Errorf("unexpected TASK_DESCRIPTION: %s", envMap["TASK_DESCRIPTION"])
}
if envMap["DEV_TOOLS"] != "go@1.25,rust@1.94" {
t.Errorf("unexpected DEV_TOOLS: %s", envMap["DEV_TOOLS"])
}
if envMap["TTYD_BASE_PATH"] != "/@alice/main" {
t.Errorf("unexpected TTYD_BASE_PATH: %s", envMap["TTYD_BASE_PATH"])
}
if envMap["DEV_EXTERNAL_HOST"] != "spinoff.dev" {
t.Errorf("unexpected DEV_EXTERNAL_HOST: %s", envMap["DEV_EXTERNAL_HOST"])
}
if envMap["FORGEJO_URL"] != "http://forgejo.dev-infra.svc:3000" {
t.Errorf("unexpected FORGEJO_URL: %s", envMap["FORGEJO_URL"])
}
// Check envFrom references dev-secrets
if len(dev.EnvFrom) != 1 {
t.Fatalf("expected 1 envFrom, got %d", len(dev.EnvFrom))
}
if dev.EnvFrom[0].SecretRef.Name != "dev-secrets" {
t.Errorf("expected envFrom dev-secrets, got %s", dev.EnvFrom[0].SecretRef.Name)
}
// Check volume mount
if len(dev.VolumeMounts) != 1 || dev.VolumeMounts[0].MountPath != "/home/dev/workspace" {
t.Errorf("expected workspace mount at /home/dev/workspace")
}
})
t.Run("ai_proxy_container", func(t *testing.T) {
proxy := pod.Spec.Containers[1]
if proxy.Name != "ai-proxy" {
t.Errorf("expected container name ai-proxy, got %s", proxy.Name)
}
if proxy.Image != "nginx:alpine" {
t.Errorf("expected nginx:alpine, got %s", proxy.Image)
}
if len(proxy.VolumeMounts) != 2 {
t.Errorf("expected 2 volume mounts, got %d", len(proxy.VolumeMounts))
}
for _, vm := range proxy.VolumeMounts {
if !vm.ReadOnly {
t.Errorf("expected read-only mount for %s", vm.Name)
}
}
})
t.Run("ipip_sidecar_container", func(t *testing.T) {
ipip := pod.Spec.Containers[2]
if ipip.Name != "ipip-sidecar" {
t.Errorf("expected container name ipip-sidecar, got %s", ipip.Name)
}
if ipip.Image != "10.22.0.56:30500/claw-ipip-tunnel:dev" {
t.Errorf("expected ipip tunnel image, got %s", ipip.Image)
}
if ipip.SecurityContext == nil || !*ipip.SecurityContext.Privileged {
t.Error("expected privileged security context")
}
envMap := make(map[string]string)
for _, e := range ipip.Env {
if e.ValueFrom == nil {
envMap[e.Name] = e.Value
}
}
if envMap["POD_ID"] != "dev-alice-main" {
t.Errorf("unexpected POD_ID: %s", envMap["POD_ID"])
}
if envMap["VPN_GATEWAY_HOST"] != "vpn-gateway.claw-system.svc" {
t.Errorf("unexpected VPN_GATEWAY_HOST: %s", envMap["VPN_GATEWAY_HOST"])
}
// Check VPN_GATEWAY_KEY comes from secret ref
var vpnKeyEnv *corev1.EnvVar
for i := range ipip.Env {
if ipip.Env[i].Name == "VPN_GATEWAY_KEY" {
vpnKeyEnv = &ipip.Env[i]
break
}
}
if vpnKeyEnv == nil || vpnKeyEnv.ValueFrom == nil || vpnKeyEnv.ValueFrom.SecretKeyRef == nil {
t.Fatal("expected VPN_GATEWAY_KEY from secret ref")
}
if vpnKeyEnv.ValueFrom.SecretKeyRef.Name != "dev-secrets" {
t.Errorf("expected secret name dev-secrets, got %s", vpnKeyEnv.ValueFrom.SecretKeyRef.Name)
}
})
t.Run("volumes", func(t *testing.T) {
if len(pod.Spec.Volumes) != 3 {
t.Fatalf("expected 3 volumes, got %d", len(pod.Spec.Volumes))
}
volumeNames := make(map[string]bool)
for _, v := range pod.Spec.Volumes {
volumeNames[v.Name] = true
}
for _, name := range []string{"workspace", "ai-proxy-config", "ai-proxy-secrets"} {
if !volumeNames[name] {
t.Errorf("missing volume %s", name)
}
}
// Verify workspace volume references per-pod PVC
wsVol := pod.Spec.Volumes[0]
if wsVol.PersistentVolumeClaim.ClaimName != "workspace-main" {
t.Errorf("expected workspace PVC claim workspace-main, got %s", wsVol.PersistentVolumeClaim.ClaimName)
}
})
}
func TestServiceTemplate(t *testing.T) {
svc := ServiceTemplate("alice", "main")
if svc.Name != "dev-pod-main-svc" {
t.Errorf("expected name dev-pod-main-svc, got %s", svc.Name)
}
if svc.Namespace != "dev-alice" {
t.Errorf("expected namespace dev-alice, got %s", svc.Namespace)
}
if svc.Spec.Type != corev1.ServiceTypeClusterIP {
t.Errorf("expected ClusterIP, got %s", svc.Spec.Type)
}
if len(svc.Spec.Ports) != 5 {
t.Errorf("expected 5 ports, got %d", len(svc.Spec.Ports))
}
portMap := make(map[string]int32)
for _, p := range svc.Spec.Ports {
portMap[p.Name] = p.Port
}
expected := map[string]int32{
"ttyd": 7681, "ssh": 22, "forgejo": 3000,
"vscode": 8080, "ralphex-gerrit": 8090,
}
for name, port := range expected {
if portMap[name] != port {
t.Errorf("expected port %s=%d, got %d", name, port, portMap[name])
}
}
if svc.Spec.Selector["app"] != "dev-pod" || svc.Spec.Selector["podname"] != "main" {
t.Errorf("unexpected selector: %v", svc.Spec.Selector)
}
}
func TestIngressTemplate(t *testing.T) {
objs := IngressTemplate("alice", "main", "spinoff.dev")
if len(objs) != 3 {
t.Fatalf("expected 3 objects (ingress + 2 middleware), got %d", len(objs))
}
t.Run("ingress_route", func(t *testing.T) {
ir := objs[0]
if ir.GetKind() != "IngressRoute" {
t.Errorf("expected kind IngressRoute, got %s", ir.GetKind())
}
if ir.GetName() != "dev-pod-main-ingress" {
t.Errorf("expected name dev-pod-main-ingress, got %s", ir.GetName())
}
if ir.GetNamespace() != "dev-alice" {
t.Errorf("expected namespace dev-alice, got %s", ir.GetNamespace())
}
spec := ir.Object["spec"].(map[string]interface{})
routes := spec["routes"].([]interface{})
if len(routes) != 5 {
t.Errorf("expected 5 routes, got %d", len(routes))
}
// Check first route is ttyd (no explicit priority — Traefik v3 auto-calculates)
r0 := routes[0].(map[string]interface{})
if _, hasPriority := r0["priority"]; hasPriority {
t.Errorf("expected no explicit priority, got %v", r0["priority"])
}
if r0["match"] != "Host(`spinoff.dev`) && PathPrefix(`/@alice/main/`)" {
t.Errorf("unexpected match: %s", r0["match"])
}
// Check last route has strip prefix middleware
r4 := routes[4].(map[string]interface{})
middlewares := r4["middlewares"].([]interface{})
if len(middlewares) != 2 {
t.Errorf("expected 2 middlewares for ralphex route, got %d", len(middlewares))
}
})
t.Run("basic_auth_middleware", func(t *testing.T) {
mw := objs[1]
if mw.GetKind() != "Middleware" {
t.Errorf("expected kind Middleware, got %s", mw.GetKind())
}
if mw.GetName() != "spinoff-basic-auth" {
t.Errorf("expected name spinoff-basic-auth, got %s", mw.GetName())
}
})
t.Run("strip_prefix_middleware", func(t *testing.T) {
mw := objs[2]
if mw.GetName() != "strip-dev-main-ralphex-prefix" {
t.Errorf("expected name strip-dev-main-ralphex-prefix, got %s", mw.GetName())
}
spec := mw.Object["spec"].(map[string]interface{})
stripPrefix := spec["stripPrefix"].(map[string]interface{})
prefixes := stripPrefix["prefixes"].([]interface{})
if len(prefixes) != 1 || prefixes[0] != "/@alice/main/ralphex" {
t.Errorf("unexpected prefixes: %v", prefixes)
}
})
}
func TestSecretsTemplate(t *testing.T) {
devSec, aiSec := SecretsTemplate("alice", "vpn-key-123", "br_anthropic_real", "br_openai_real", "forgejo-tok-abc", "tskey-auth-abc123")
t.Run("dev_secrets", func(t *testing.T) {
if devSec.Name != "dev-secrets" {
t.Errorf("expected name dev-secrets, got %s", devSec.Name)
}
if devSec.Namespace != "dev-alice" {
t.Errorf("expected namespace dev-alice, got %s", devSec.Namespace)
}
if devSec.StringData["ANTHROPIC_API_KEY"] != "sk-devpod" {
t.Errorf("expected dummy anthropic key, got %s", devSec.StringData["ANTHROPIC_API_KEY"])
}
if devSec.StringData["ANTHROPIC_BASE_URL"] != "http://anthropic.internal" {
t.Errorf("unexpected base url: %s", devSec.StringData["ANTHROPIC_BASE_URL"])
}
if devSec.StringData["BASEROUTE_OPENAI_KEY"] != "sk-devpod" {
t.Errorf("expected dummy openai key, got %s", devSec.StringData["BASEROUTE_OPENAI_KEY"])
}
if devSec.StringData["VPN_GATEWAY_KEY"] != "vpn-key-123" {
t.Errorf("expected vpn key vpn-key-123, got %s", devSec.StringData["VPN_GATEWAY_KEY"])
}
if devSec.StringData["FORGEJO_TOKEN"] != "forgejo-tok-abc" {
t.Errorf("expected forgejo token forgejo-tok-abc, got %s", devSec.StringData["FORGEJO_TOKEN"])
}
if devSec.StringData["TAILSCALE_AUTHKEY"] != "tskey-auth-abc123" {
t.Errorf("expected tailscale key tskey-auth-abc123, got %s", devSec.StringData["TAILSCALE_AUTHKEY"])
}
})
t.Run("ai_proxy_secrets", func(t *testing.T) {
if aiSec.Name != "ai-proxy-secrets" {
t.Errorf("expected name ai-proxy-secrets, got %s", aiSec.Name)
}
if aiSec.Namespace != "dev-alice" {
t.Errorf("expected namespace dev-alice, got %s", aiSec.Namespace)
}
if aiSec.StringData["anthropic-key"] != "br_anthropic_real" {
t.Errorf("expected real anthropic key, got %s", aiSec.StringData["anthropic-key"])
}
if aiSec.StringData["openai-key"] != "br_openai_real" {
t.Errorf("expected real openai key, got %s", aiSec.StringData["openai-key"])
}
})
}
func TestSecretsTemplate_NoTailscaleKey(t *testing.T) {
devSec, _ := SecretsTemplate("alice", "vpn-key-123", "br_anthropic_real", "br_openai_real", "forgejo-tok-abc", "")
if _, exists := devSec.StringData["TAILSCALE_AUTHKEY"]; exists {
t.Error("TAILSCALE_AUTHKEY should not be present when tailscale key is empty")
}
}
func TestNetworkPolicyTemplate(t *testing.T) {
np := NetworkPolicyTemplate("alice")
if np.Name != "dev-pod-network-policy" {
t.Errorf("expected name dev-pod-network-policy, got %s", np.Name)
}
if np.Namespace != "dev-alice" {
t.Errorf("expected namespace dev-alice, got %s", np.Namespace)
}
if len(np.Spec.PolicyTypes) != 2 {
t.Errorf("expected 2 policy types, got %d", len(np.Spec.PolicyTypes))
}
t.Run("ingress_rules", func(t *testing.T) {
if len(np.Spec.Ingress) != 1 {
t.Fatalf("expected 1 ingress rule, got %d", len(np.Spec.Ingress))
}
from := np.Spec.Ingress[0].From
if len(from) != 1 {
t.Fatalf("expected 1 from peer, got %d", len(from))
}
nsLabel := from[0].NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"]
if nsLabel != "kube-system" {
t.Errorf("expected kube-system, got %s", nsLabel)
}
})
t.Run("egress_rules", func(t *testing.T) {
if len(np.Spec.Egress) != 3 {
t.Fatalf("expected 3 egress rules, got %d", len(np.Spec.Egress))
}
// VPN gateway rule
vpnRule := np.Spec.Egress[0]
if len(vpnRule.To) != 1 {
t.Fatalf("expected 1 to peer in vpn rule, got %d", len(vpnRule.To))
}
nsLabel := vpnRule.To[0].NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"]
if nsLabel != "claw-system" {
t.Errorf("expected claw-system, got %s", nsLabel)
}
podLabel := vpnRule.To[0].PodSelector.MatchLabels["app"]
if podLabel != "vpn-gateway" {
t.Errorf("expected vpn-gateway, got %s", podLabel)
}
// dev-infra rule
devInfraRule := np.Spec.Egress[1]
nsLabel = devInfraRule.To[0].NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"]
if nsLabel != "dev-infra" {
t.Errorf("expected dev-infra, got %s", nsLabel)
}
// DNS rule
dnsRule := np.Spec.Egress[2]
if len(dnsRule.Ports) != 2 {
t.Errorf("expected 2 DNS ports, got %d", len(dnsRule.Ports))
}
})
}
func TestAIProxyConfigMapTemplate(t *testing.T) {
cm := AIProxyConfigMapTemplate("alice")
if cm.Name != "ai-proxy-config" {
t.Errorf("expected name ai-proxy-config, got %s", cm.Name)
}
if cm.Namespace != "dev-alice" {
t.Errorf("expected namespace dev-alice, got %s", cm.Namespace)
}
if _, ok := cm.Data["nginx.conf.template"]; !ok {
t.Error("missing nginx.conf.template in ConfigMap data")
}
if _, ok := cm.Data["entrypoint.sh"]; !ok {
t.Error("missing entrypoint.sh in ConfigMap data")
}
}
func TestPodTemplateWithDifferentUsers(t *testing.T) {
tests := []struct {
user string
pod string
wantNS string
wantPod string
wantPath string
}{
{"bob", "dev", "dev-bob", "dev-pod-dev", "/@bob/dev"},
{"team-alpha", "staging", "dev-team-alpha", "dev-pod-staging", "/@team-alpha/staging"},
}
for _, tt := range tests {
t.Run(tt.user+"/"+tt.pod, func(t *testing.T) {
opts := PodOpts{
User: tt.user, Pod: tt.pod,
CPUReq: "1", CPULimit: "2", MemReq: "1Gi", MemLimit: "2Gi",
}
pod := PodTemplate(testCfg, opts)
if pod.Namespace != tt.wantNS {
t.Errorf("expected namespace %s, got %s", tt.wantNS, pod.Namespace)
}
if pod.Name != tt.wantPod {
t.Errorf("expected pod name %s, got %s", tt.wantPod, pod.Name)
}
// Check TTYD_BASE_PATH
dev := pod.Spec.Containers[0]
for _, e := range dev.Env {
if e.Name == "TTYD_BASE_PATH" && e.Value != tt.wantPath {
t.Errorf("expected TTYD_BASE_PATH %s, got %s", tt.wantPath, e.Value)
}
}
})
}
}

View file

@ -0,0 +1,381 @@
package k8s
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
func workflowTemplatesDir() string {
_, thisFile, _, _ := runtime.Caller(0)
return filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "..", "docker", "dev-golden", "forgejo-workflow-templates")
}
type workflowDoc struct {
Name string `yaml:"name"`
On map[string]interface{} `yaml:"on"`
Jobs map[string]interface{} `yaml:"jobs"`
}
func readWorkflowYAML(t *testing.T, name string) workflowDoc {
t.Helper()
data, err := os.ReadFile(filepath.Join(workflowTemplatesDir(), name))
if err != nil {
t.Fatalf("reading %s: %v", name, err)
}
var doc workflowDoc
if err := yaml.Unmarshal(data, &doc); err != nil {
t.Fatalf("parsing %s: %v", name, err)
}
return doc
}
func jobFromWorkflow(t *testing.T, doc workflowDoc, jobName string) map[string]interface{} {
t.Helper()
job, ok := doc.Jobs[jobName]
if !ok {
t.Fatalf("missing %s job", jobName)
}
jobMap, ok := job.(map[string]interface{})
if !ok {
t.Fatalf("job %s is not a map", jobName)
}
return jobMap
}
func stepsFromJob(t *testing.T, job map[string]interface{}) []interface{} {
t.Helper()
steps, ok := job["steps"].([]interface{})
if !ok || len(steps) == 0 {
t.Fatal("missing steps")
}
return steps
}
func stepNames(steps []interface{}) []string {
var names []string
for _, s := range steps {
step := s.(map[string]interface{})
if name, ok := step["name"].(string); ok {
names = append(names, name)
}
if uses, ok := step["uses"].(string); ok {
names = append(names, uses)
}
}
return names
}
func assertRunsOnSelfHosted(t *testing.T, job map[string]interface{}) {
t.Helper()
runsOn, ok := job["runs-on"].([]interface{})
if !ok {
t.Fatal("missing runs-on")
}
for _, r := range runsOn {
if r == "self-hosted" {
return
}
}
t.Error("expected 'self-hosted' in runs-on")
}
func assertStepsContain(t *testing.T, names []string, required []string) {
t.Helper()
for _, req := range required {
found := false
for _, name := range names {
if strings.Contains(name, req) {
found = true
break
}
}
if !found {
t.Errorf("missing required step containing %q", req)
}
}
}
func TestWorkflowTemplatesExist(t *testing.T) {
dir := workflowTemplatesDir()
expected := []string{
"claude-implement.yml",
"claude-review.yml",
"ci-build.yml",
}
for _, name := range expected {
path := filepath.Join(dir, name)
if _, err := os.Stat(path); os.IsNotExist(err) {
t.Errorf("missing workflow template: %s", path)
}
}
}
func TestClaudeImplementWorkflow(t *testing.T) {
doc := readWorkflowYAML(t, "claude-implement.yml")
t.Run("name", func(t *testing.T) {
if doc.Name != "Claude Implement" {
t.Errorf("expected name 'Claude Implement', got %v", doc.Name)
}
})
t.Run("trigger", func(t *testing.T) {
ic, ok := doc.On["issue_comment"].(map[string]interface{})
if !ok {
t.Fatal("missing issue_comment trigger")
}
types, ok := ic["types"].([]interface{})
if !ok || len(types) == 0 {
t.Fatal("missing types for issue_comment")
}
if types[0] != "created" {
t.Errorf("expected type 'created', got %v", types[0])
}
})
t.Run("job_condition", func(t *testing.T) {
impl := jobFromWorkflow(t, doc, "implement")
ifCond, ok := impl["if"].(string)
if !ok {
t.Fatal("missing 'if' condition")
}
if !strings.Contains(ifCond, "@claude implement") {
t.Error("'if' condition should check for '@claude implement'")
}
if !strings.Contains(ifCond, "pull_request == null") {
t.Error("'if' condition should exclude PR comments")
}
})
t.Run("runs_on_self_hosted", func(t *testing.T) {
impl := jobFromWorkflow(t, doc, "implement")
assertRunsOnSelfHosted(t, impl)
})
t.Run("has_required_steps", func(t *testing.T) {
impl := jobFromWorkflow(t, doc, "implement")
steps := stepsFromJob(t, impl)
assertStepsContain(t, stepNames(steps), []string{
"actions/checkout@v4",
"Configure git",
"Extract task",
"Run Claude Code",
"Create branch and push",
"Create pull request",
})
})
t.Run("permissions", func(t *testing.T) {
impl := jobFromWorkflow(t, doc, "implement")
perms, ok := impl["permissions"].(map[string]interface{})
if !ok {
t.Fatal("missing permissions")
}
if perms["contents"] != "write" {
t.Error("expected contents: write permission")
}
if perms["pull-requests"] != "write" {
t.Error("expected pull-requests: write permission")
}
})
}
func TestClaudeReviewWorkflow(t *testing.T) {
doc := readWorkflowYAML(t, "claude-review.yml")
t.Run("name", func(t *testing.T) {
if doc.Name != "Claude Review" {
t.Errorf("expected name 'Claude Review', got %v", doc.Name)
}
})
t.Run("trigger", func(t *testing.T) {
ic, ok := doc.On["issue_comment"].(map[string]interface{})
if !ok {
t.Fatal("missing issue_comment trigger")
}
types, ok := ic["types"].([]interface{})
if !ok || len(types) == 0 {
t.Fatal("missing types for issue_comment")
}
if types[0] != "created" {
t.Errorf("expected type 'created', got %v", types[0])
}
})
t.Run("job_condition", func(t *testing.T) {
review := jobFromWorkflow(t, doc, "review")
ifCond, ok := review["if"].(string)
if !ok {
t.Fatal("missing 'if' condition")
}
if !strings.Contains(ifCond, "@claude review") {
t.Error("'if' condition should check for '@claude review'")
}
if !strings.Contains(ifCond, "pull_request != null") {
t.Error("'if' condition should require PR context")
}
})
t.Run("runs_on_self_hosted", func(t *testing.T) {
review := jobFromWorkflow(t, doc, "review")
assertRunsOnSelfHosted(t, review)
})
t.Run("has_required_steps", func(t *testing.T) {
review := jobFromWorkflow(t, doc, "review")
steps := stepsFromJob(t, review)
assertStepsContain(t, stepNames(steps), []string{
"actions/checkout@v4",
"Fetch base branch",
"Run Claude Code review",
"Post review comment",
})
})
t.Run("checkout_uses_pr_ref", func(t *testing.T) {
review := jobFromWorkflow(t, doc, "review")
steps := stepsFromJob(t, review)
for _, s := range steps {
step := s.(map[string]interface{})
if uses, ok := step["uses"].(string); ok && strings.Contains(uses, "checkout") {
with, ok := step["with"].(map[string]interface{})
if !ok {
t.Fatal("checkout step missing 'with' for PR ref")
}
ref, _ := with["ref"].(string)
if ref == "" {
t.Error("checkout step should specify ref for PR branch")
}
return
}
}
t.Error("no checkout step found")
})
t.Run("permissions", func(t *testing.T) {
review := jobFromWorkflow(t, doc, "review")
perms, ok := review["permissions"].(map[string]interface{})
if !ok {
t.Fatal("missing permissions")
}
if perms["pull-requests"] != "write" {
t.Error("expected pull-requests: write permission")
}
})
}
func TestCIBuildWorkflow(t *testing.T) {
doc := readWorkflowYAML(t, "ci-build.yml")
t.Run("name", func(t *testing.T) {
if doc.Name != "CI Build" {
t.Errorf("expected name 'CI Build', got %v", doc.Name)
}
})
t.Run("trigger", func(t *testing.T) {
push, ok := doc.On["push"].(map[string]interface{})
if !ok {
t.Fatal("missing push trigger")
}
branches, ok := push["branches"].([]interface{})
if !ok || len(branches) == 0 {
t.Fatal("missing branches for push trigger")
}
if branches[0] != "main" {
t.Errorf("expected branch 'main', got %v", branches[0])
}
})
t.Run("job_exists", func(t *testing.T) {
jobFromWorkflow(t, doc, "build")
})
t.Run("runs_on_self_hosted", func(t *testing.T) {
build := jobFromWorkflow(t, doc, "build")
assertRunsOnSelfHosted(t, build)
})
t.Run("has_checkout_and_build_steps", func(t *testing.T) {
build := jobFromWorkflow(t, doc, "build")
steps := stepsFromJob(t, build)
names := stepNames(steps)
hasCheckout := false
hasBuild := false
for _, n := range names {
if strings.Contains(n, "checkout") {
hasCheckout = true
}
if strings.Contains(strings.ToLower(n), "build") {
hasBuild = true
}
}
if !hasCheckout {
t.Error("missing checkout step")
}
if !hasBuild {
t.Error("missing build step")
}
})
t.Run("build_detects_multiple_languages", func(t *testing.T) {
build := jobFromWorkflow(t, doc, "build")
steps := stepsFromJob(t, build)
for _, s := range steps {
step := s.(map[string]interface{})
name, _ := step["name"].(string)
if !strings.Contains(strings.ToLower(name), "build") {
continue
}
run, ok := step["run"].(string)
if !ok {
t.Fatal("build step missing run command")
}
for _, marker := range []string{"Cargo.toml", "go.mod", "package.json"} {
if !strings.Contains(run, marker) {
t.Errorf("build step should detect %s", marker)
}
}
return
}
t.Error("no build step found with run command")
})
}
func TestWorkflowTemplatesAreValidYAML(t *testing.T) {
dir := workflowTemplatesDir()
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatalf("reading templates dir: %v", err)
}
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yml") {
continue
}
t.Run(entry.Name(), func(t *testing.T) {
data, err := os.ReadFile(filepath.Join(dir, entry.Name()))
if err != nil {
t.Fatalf("reading %s: %v", entry.Name(), err)
}
var doc workflowDoc
if err := yaml.Unmarshal(data, &doc); err != nil {
t.Fatalf("invalid YAML in %s: %v", entry.Name(), err)
}
if doc.Name == "" {
t.Errorf("%s missing 'name' field", entry.Name())
}
if doc.Jobs == nil {
t.Errorf("%s missing 'jobs' field", entry.Name())
}
})
}
}