build source
This commit is contained in:
commit
ee1fec43ed
4171 changed files with 1351288 additions and 0 deletions
BIN
internal/k8s/._client.go
Normal file
BIN
internal/k8s/._client.go
Normal file
Binary file not shown.
BIN
internal/k8s/._cluster.go
Normal file
BIN
internal/k8s/._cluster.go
Normal file
Binary file not shown.
BIN
internal/k8s/._cluster_test.go
Normal file
BIN
internal/k8s/._cluster_test.go
Normal file
Binary file not shown.
BIN
internal/k8s/._forgejo_test.go
Normal file
BIN
internal/k8s/._forgejo_test.go
Normal file
Binary file not shown.
BIN
internal/k8s/._metrics.go
Normal file
BIN
internal/k8s/._metrics.go
Normal file
Binary file not shown.
BIN
internal/k8s/._namespace.go
Normal file
BIN
internal/k8s/._namespace.go
Normal file
Binary file not shown.
BIN
internal/k8s/._namespace_test.go
Normal file
BIN
internal/k8s/._namespace_test.go
Normal file
Binary file not shown.
BIN
internal/k8s/._pods.go
Normal file
BIN
internal/k8s/._pods.go
Normal file
Binary file not shown.
BIN
internal/k8s/._pods_test.go
Normal file
BIN
internal/k8s/._pods_test.go
Normal file
Binary file not shown.
BIN
internal/k8s/._runners.go
Normal file
BIN
internal/k8s/._runners.go
Normal file
Binary file not shown.
BIN
internal/k8s/._runners_test.go
Normal file
BIN
internal/k8s/._runners_test.go
Normal file
Binary file not shown.
BIN
internal/k8s/._sessionhost.go
Normal file
BIN
internal/k8s/._sessionhost.go
Normal file
Binary file not shown.
BIN
internal/k8s/._templates.go
Normal file
BIN
internal/k8s/._templates.go
Normal file
Binary file not shown.
BIN
internal/k8s/._templates_test.go
Normal file
BIN
internal/k8s/._templates_test.go
Normal file
Binary file not shown.
BIN
internal/k8s/._workflow_templates_test.go
Normal file
BIN
internal/k8s/._workflow_templates_test.go
Normal file
Binary file not shown.
102
internal/k8s/client.go
Normal file
102
internal/k8s/client.go
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"k8s.io/client-go/dynamic"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
)
|
||||
|
||||
// Client wraps a Kubernetes clientset with config used for template generation.
|
||||
type Client struct {
|
||||
Clientset kubernetes.Interface
|
||||
Dynamic dynamic.Interface
|
||||
Config Config
|
||||
}
|
||||
|
||||
// Config holds cluster-specific settings for pod template generation.
|
||||
type Config struct {
|
||||
Domain string // e.g. "spinoff.dev"
|
||||
Registry string // e.g. "10.22.0.56:30500"
|
||||
GoldenImage string // e.g. "dev-golden:v2"
|
||||
VPNGatewayNS string // e.g. "claw-system"
|
||||
VPNGatewaySecret string // e.g. "vpn-gateway-secrets"
|
||||
AnthropicKey string // baseroute anthropic key
|
||||
OpenAIKey string // baseroute openai key
|
||||
ForgejoURL string // e.g. "http://forgejo.dev-infra.svc:3000"
|
||||
DevPodAPIURL string // e.g. "http://dev-pod-api.dev-infra.svc:8080"
|
||||
}
|
||||
|
||||
// ConfigFromEnv reads Config from environment variables with defaults.
|
||||
func ConfigFromEnv() Config {
|
||||
return Config{
|
||||
Domain: envOrDefault("DOMAIN", "spinoff.dev"),
|
||||
Registry: envOrDefault("REGISTRY", "10.22.0.56:30500"),
|
||||
GoldenImage: envOrDefault("GOLDEN_IMAGE", "dev-golden:v2"),
|
||||
VPNGatewayNS: envOrDefault("VPN_GATEWAY_NS", "claw-system"),
|
||||
VPNGatewaySecret: envOrDefault("VPN_GATEWAY_SECRET", "vpn-gateway-secrets"),
|
||||
AnthropicKey: os.Getenv("DEFAULT_ANTHROPIC_KEY"),
|
||||
OpenAIKey: os.Getenv("DEFAULT_OPENAI_KEY"),
|
||||
ForgejoURL: envOrDefault("FORGEJO_URL", "http://forgejo.dev-infra.svc:3000"),
|
||||
DevPodAPIURL: envOrDefault("DEV_POD_API_URL", "http://dev-pod-api.dev-infra.svc:8080"),
|
||||
}
|
||||
}
|
||||
|
||||
// NewClient creates a Kubernetes client using in-cluster config,
|
||||
// falling back to kubeconfig for local development.
|
||||
func NewClient(cfg Config) (*Client, error) {
|
||||
restCfg, err := rest.InClusterConfig()
|
||||
if err != nil {
|
||||
slog.Info("in-cluster config not available, falling back to kubeconfig")
|
||||
restCfg, err = kubeconfigFromDefault()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create k8s config: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
clientset, err := kubernetes.NewForConfig(restCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create k8s clientset: %w", err)
|
||||
}
|
||||
|
||||
dyn, err := dynamic.NewForConfig(restCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create dynamic client: %w", err)
|
||||
}
|
||||
|
||||
return &Client{
|
||||
Clientset: clientset,
|
||||
Dynamic: dyn,
|
||||
Config: cfg,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewClientWithClientset creates a Client with pre-configured clients (for testing).
|
||||
func NewClientWithClientset(clientset kubernetes.Interface, dyn dynamic.Interface, cfg Config) *Client {
|
||||
return &Client{
|
||||
Clientset: clientset,
|
||||
Dynamic: dyn,
|
||||
Config: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
func kubeconfigFromDefault() (*rest.Config, error) {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get home dir: %w", err)
|
||||
}
|
||||
kubeconfig := filepath.Join(home, ".kube", "config")
|
||||
return clientcmd.BuildConfigFromFlags("", kubeconfig)
|
||||
}
|
||||
|
||||
func envOrDefault(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
166
internal/k8s/cluster.go
Normal file
166
internal/k8s/cluster.go
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
|
||||
)
|
||||
|
||||
// nodeMetricsList mirrors the metrics API response for nodes.
|
||||
type nodeMetricsList struct {
|
||||
Items []nodeMetrics `json:"items"`
|
||||
}
|
||||
|
||||
type nodeMetrics struct {
|
||||
Metadata struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"metadata"`
|
||||
Usage map[string]string `json:"usage"`
|
||||
}
|
||||
|
||||
// GetClusterStatus returns node list with capacity, allocatable, and usage.
|
||||
func (c *Client) GetClusterStatus(ctx context.Context) (*model.ClusterStatus, error) {
|
||||
nodes, err := c.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list nodes: %w", err)
|
||||
}
|
||||
|
||||
// Try to fetch node metrics (may not be available)
|
||||
metricsMap := c.fetchNodeMetrics(ctx)
|
||||
|
||||
var totalCPUCap, totalCPUAlloc, totalMemCap, totalMemAlloc int64
|
||||
status := &model.ClusterStatus{
|
||||
Nodes: make([]model.NodeStatus, 0, len(nodes.Items)),
|
||||
}
|
||||
|
||||
for _, node := range nodes.Items {
|
||||
ns := model.NodeStatus{
|
||||
Name: node.Name,
|
||||
Status: nodeConditionStatus(node),
|
||||
CPUCapacity: node.Status.Capacity.Cpu().String(),
|
||||
CPUAllocatable: node.Status.Allocatable.Cpu().String(),
|
||||
MemCapacity: node.Status.Capacity.Memory().String(),
|
||||
MemAllocatable: node.Status.Allocatable.Memory().String(),
|
||||
}
|
||||
|
||||
totalCPUCap += node.Status.Capacity.Cpu().MilliValue()
|
||||
totalCPUAlloc += node.Status.Allocatable.Cpu().MilliValue()
|
||||
totalMemCap += node.Status.Capacity.Memory().Value()
|
||||
totalMemAlloc += node.Status.Allocatable.Memory().Value()
|
||||
|
||||
if m, ok := metricsMap[node.Name]; ok {
|
||||
ns.CPUUsage = m.cpuUsage
|
||||
ns.MemUsage = m.memUsage
|
||||
}
|
||||
|
||||
status.Nodes = append(status.Nodes, ns)
|
||||
}
|
||||
|
||||
status.Total = model.ResourceSummary{
|
||||
CPUCapacity: resource.NewMilliQuantity(totalCPUCap, resource.DecimalSI).String(),
|
||||
CPUAllocatable: resource.NewMilliQuantity(totalCPUAlloc, resource.DecimalSI).String(),
|
||||
MemCapacity: resource.NewQuantity(totalMemCap, resource.BinarySI).String(),
|
||||
MemAllocatable: resource.NewQuantity(totalMemAlloc, resource.BinarySI).String(),
|
||||
}
|
||||
|
||||
return status, nil
|
||||
}
|
||||
|
||||
type nodeUsage struct {
|
||||
cpuUsage string
|
||||
memUsage string
|
||||
}
|
||||
|
||||
func (c *Client) fetchNodeMetrics(ctx context.Context) map[string]nodeUsage {
|
||||
result := make(map[string]nodeUsage)
|
||||
|
||||
restClient := c.Clientset.Discovery().RESTClient()
|
||||
if restClient == nil {
|
||||
return result
|
||||
}
|
||||
|
||||
data, err := restClient.Get().
|
||||
AbsPath("/apis/metrics.k8s.io/v1beta1/nodes").
|
||||
DoRaw(ctx)
|
||||
if err != nil {
|
||||
return result
|
||||
}
|
||||
|
||||
var metrics nodeMetricsList
|
||||
if err := json.Unmarshal(data, &metrics); err != nil {
|
||||
return result
|
||||
}
|
||||
|
||||
for _, m := range metrics.Items {
|
||||
nu := nodeUsage{}
|
||||
if cpu, ok := m.Usage["cpu"]; ok {
|
||||
nu.cpuUsage = cpu
|
||||
}
|
||||
if mem, ok := m.Usage["memory"]; ok {
|
||||
nu.memUsage = mem
|
||||
}
|
||||
result[m.Metadata.Name] = nu
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func nodeConditionStatus(node corev1.Node) string {
|
||||
for _, cond := range node.Status.Conditions {
|
||||
if cond.Type == corev1.NodeReady {
|
||||
if cond.Status == corev1.ConditionTrue {
|
||||
return "Ready"
|
||||
}
|
||||
return "NotReady"
|
||||
}
|
||||
}
|
||||
return "Unknown"
|
||||
}
|
||||
|
||||
// cacheServices maps friendly names to PVC names in the dev-infra namespace.
|
||||
var cacheServices = []struct {
|
||||
name string
|
||||
pvcName string
|
||||
}{
|
||||
{"verdaccio", "verdaccio-storage"},
|
||||
{"athens", "athens-storage"},
|
||||
{"cargo-proxy", "cargo-proxy-cache"},
|
||||
}
|
||||
|
||||
// GetCacheStats returns PVC status for cache services in dev-infra.
|
||||
func (c *Client) GetCacheStats(ctx context.Context) ([]model.CacheStat, error) {
|
||||
stats := make([]model.CacheStat, 0, len(cacheServices))
|
||||
|
||||
for _, svc := range cacheServices {
|
||||
pvc, err := c.Clientset.CoreV1().PersistentVolumeClaims("dev-infra").Get(ctx, svc.pvcName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
stats = append(stats, model.CacheStat{
|
||||
Name: svc.name,
|
||||
PVCName: svc.pvcName,
|
||||
Status: "NotFound",
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
capacity := ""
|
||||
if pvc.Status.Capacity != nil {
|
||||
if storage, ok := pvc.Status.Capacity[corev1.ResourceStorage]; ok {
|
||||
capacity = storage.String()
|
||||
}
|
||||
}
|
||||
|
||||
stats = append(stats, model.CacheStat{
|
||||
Name: svc.name,
|
||||
PVCName: svc.pvcName,
|
||||
Capacity: capacity,
|
||||
Status: string(pvc.Status.Phase),
|
||||
})
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
224
internal/k8s/cluster_test.go
Normal file
224
internal/k8s/cluster_test.go
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
func TestGetClusterStatus_Nodes(t *testing.T) {
|
||||
cs := fake.NewSimpleClientset(
|
||||
&corev1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "node-1"},
|
||||
Status: corev1.NodeStatus{
|
||||
Conditions: []corev1.NodeCondition{
|
||||
{Type: corev1.NodeReady, Status: corev1.ConditionTrue},
|
||||
},
|
||||
Capacity: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("8"),
|
||||
corev1.ResourceMemory: resource.MustParse("32Gi"),
|
||||
},
|
||||
Allocatable: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("7800m"),
|
||||
corev1.ResourceMemory: resource.MustParse("30Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
&corev1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "node-2"},
|
||||
Status: corev1.NodeStatus{
|
||||
Conditions: []corev1.NodeCondition{
|
||||
{Type: corev1.NodeReady, Status: corev1.ConditionFalse},
|
||||
},
|
||||
Capacity: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("4"),
|
||||
corev1.ResourceMemory: resource.MustParse("16Gi"),
|
||||
},
|
||||
Allocatable: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("3800m"),
|
||||
corev1.ResourceMemory: resource.MustParse("14Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
client := NewClientWithClientset(cs, nil, Config{})
|
||||
|
||||
status, err := client.GetClusterStatus(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(status.Nodes) != 2 {
|
||||
t.Fatalf("expected 2 nodes, got %d", len(status.Nodes))
|
||||
}
|
||||
|
||||
n1 := status.Nodes[0]
|
||||
if n1.Name != "node-1" {
|
||||
t.Fatalf("expected node-1, got %s", n1.Name)
|
||||
}
|
||||
if n1.Status != "Ready" {
|
||||
t.Fatalf("expected Ready, got %s", n1.Status)
|
||||
}
|
||||
if n1.CPUCapacity != "8" {
|
||||
t.Fatalf("expected CPU capacity 8, got %s", n1.CPUCapacity)
|
||||
}
|
||||
|
||||
n2 := status.Nodes[1]
|
||||
if n2.Status != "NotReady" {
|
||||
t.Fatalf("expected NotReady, got %s", n2.Status)
|
||||
}
|
||||
|
||||
// Check totals: 8 + 4 = 12 CPU, 7800m + 3800m = 11600m allocatable
|
||||
if status.Total.CPUCapacity != "12" {
|
||||
t.Fatalf("expected total CPU capacity 12, got %s", status.Total.CPUCapacity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetClusterStatus_EmptyCluster(t *testing.T) {
|
||||
cs := fake.NewSimpleClientset()
|
||||
client := NewClientWithClientset(cs, nil, Config{})
|
||||
|
||||
status, err := client.GetClusterStatus(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(status.Nodes) != 0 {
|
||||
t.Fatalf("expected 0 nodes, got %d", len(status.Nodes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetClusterStatus_NodeWithNoReadyCondition(t *testing.T) {
|
||||
cs := fake.NewSimpleClientset(
|
||||
&corev1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "node-unknown"},
|
||||
Status: corev1.NodeStatus{
|
||||
Conditions: []corev1.NodeCondition{
|
||||
{Type: corev1.NodeMemoryPressure, Status: corev1.ConditionFalse},
|
||||
},
|
||||
Capacity: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("4"),
|
||||
corev1.ResourceMemory: resource.MustParse("8Gi"),
|
||||
},
|
||||
Allocatable: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("4"),
|
||||
corev1.ResourceMemory: resource.MustParse("8Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
client := NewClientWithClientset(cs, nil, Config{})
|
||||
|
||||
status, err := client.GetClusterStatus(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if status.Nodes[0].Status != "Unknown" {
|
||||
t.Fatalf("expected Unknown, got %s", status.Nodes[0].Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetCacheStats_AllBound(t *testing.T) {
|
||||
cs := fake.NewSimpleClientset(
|
||||
&corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "verdaccio-storage", Namespace: "dev-infra"},
|
||||
Status: corev1.PersistentVolumeClaimStatus{
|
||||
Phase: corev1.ClaimBound,
|
||||
Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("10Gi")},
|
||||
},
|
||||
},
|
||||
&corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "athens-storage", Namespace: "dev-infra"},
|
||||
Status: corev1.PersistentVolumeClaimStatus{
|
||||
Phase: corev1.ClaimBound,
|
||||
Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("10Gi")},
|
||||
},
|
||||
},
|
||||
&corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "cargo-proxy-cache", Namespace: "dev-infra"},
|
||||
Status: corev1.PersistentVolumeClaimStatus{
|
||||
Phase: corev1.ClaimBound,
|
||||
Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("10Gi")},
|
||||
},
|
||||
},
|
||||
)
|
||||
client := NewClientWithClientset(cs, nil, Config{})
|
||||
|
||||
stats, err := client.GetCacheStats(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(stats) != 3 {
|
||||
t.Fatalf("expected 3 cache stats, got %d", len(stats))
|
||||
}
|
||||
|
||||
for _, s := range stats {
|
||||
if s.Status != "Bound" {
|
||||
t.Fatalf("expected Bound for %s, got %s", s.Name, s.Status)
|
||||
}
|
||||
if s.Capacity != "10Gi" {
|
||||
t.Fatalf("expected 10Gi capacity for %s, got %s", s.Name, s.Capacity)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetCacheStats_MissingPVC(t *testing.T) {
|
||||
// Only verdaccio exists
|
||||
cs := fake.NewSimpleClientset(
|
||||
&corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "verdaccio-storage", Namespace: "dev-infra"},
|
||||
Status: corev1.PersistentVolumeClaimStatus{
|
||||
Phase: corev1.ClaimBound,
|
||||
Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("10Gi")},
|
||||
},
|
||||
},
|
||||
)
|
||||
client := NewClientWithClientset(cs, nil, Config{})
|
||||
|
||||
stats, err := client.GetCacheStats(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(stats) != 3 {
|
||||
t.Fatalf("expected 3 entries, got %d", len(stats))
|
||||
}
|
||||
|
||||
for _, s := range stats {
|
||||
switch s.Name {
|
||||
case "verdaccio":
|
||||
if s.Status != "Bound" {
|
||||
t.Fatalf("expected Bound for verdaccio, got %s", s.Status)
|
||||
}
|
||||
case "athens", "cargo-proxy":
|
||||
if s.Status != "NotFound" {
|
||||
t.Fatalf("expected NotFound for %s, got %s", s.Name, s.Status)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetCacheStats_EmptyCluster(t *testing.T) {
|
||||
cs := fake.NewSimpleClientset()
|
||||
client := NewClientWithClientset(cs, nil, Config{})
|
||||
|
||||
stats, err := client.GetCacheStats(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if len(stats) != 3 {
|
||||
t.Fatalf("expected 3 entries, got %d", len(stats))
|
||||
}
|
||||
for _, s := range stats {
|
||||
if s.Status != "NotFound" {
|
||||
t.Fatalf("expected NotFound for %s, got %s", s.Name, s.Status)
|
||||
}
|
||||
}
|
||||
}
|
||||
369
internal/k8s/forgejo_test.go
Normal file
369
internal/k8s/forgejo_test.go
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/yaml"
|
||||
)
|
||||
|
||||
func toInt64(v interface{}) int64 {
|
||||
switch n := v.(type) {
|
||||
case int64:
|
||||
return n
|
||||
case float64:
|
||||
return int64(n)
|
||||
case int:
|
||||
return int64(n)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func forgejoYAMLPath() string {
|
||||
_, thisFile, _, _ := runtime.Caller(0)
|
||||
return filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "..", "k8s", "dev-infra", "forgejo.yaml")
|
||||
}
|
||||
|
||||
func readForgejoDocuments(t *testing.T) []map[string]interface{} {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(forgejoYAMLPath())
|
||||
if err != nil {
|
||||
t.Fatalf("reading forgejo.yaml: %v", err)
|
||||
}
|
||||
|
||||
var docs []map[string]interface{}
|
||||
parts := strings.Split(string(data), "\n---")
|
||||
for _, part := range parts {
|
||||
trimmed := strings.TrimSpace(part)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
// Strip leading comment lines (file-level comments before first doc)
|
||||
lines := strings.Split(trimmed, "\n")
|
||||
var contentLines []string
|
||||
for _, line := range lines {
|
||||
stripped := strings.TrimSpace(line)
|
||||
if len(contentLines) == 0 && (stripped == "" || strings.HasPrefix(stripped, "#")) {
|
||||
continue
|
||||
}
|
||||
contentLines = append(contentLines, line)
|
||||
}
|
||||
content := strings.Join(contentLines, "\n")
|
||||
if content == "" {
|
||||
continue
|
||||
}
|
||||
var doc map[string]interface{}
|
||||
if err := yaml.NewYAMLOrJSONDecoder(strings.NewReader(content), 4096).Decode(&doc); err != nil {
|
||||
t.Fatalf("decoding YAML document: %v\n---\n%s", err, content[:min(200, len(content))])
|
||||
}
|
||||
docs = append(docs, doc)
|
||||
}
|
||||
return docs
|
||||
}
|
||||
|
||||
func findDoc(docs []map[string]interface{}, kind, name string) map[string]interface{} {
|
||||
for _, doc := range docs {
|
||||
if doc["kind"] == kind {
|
||||
meta, ok := doc["metadata"].(map[string]interface{})
|
||||
if ok && meta["name"] == name {
|
||||
return doc
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestForgejoYAMLExists(t *testing.T) {
|
||||
path := forgejoYAMLPath()
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
t.Fatalf("forgejo.yaml does not exist at %s", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgejoYAMLDocumentCount(t *testing.T) {
|
||||
docs := readForgejoDocuments(t)
|
||||
// Expected: Secret (admin), ConfigMap, PVC, Deployment, Service, IngressRoute, Middleware (basic-auth), Secret (htpasswd), Middleware (trailing-slash)
|
||||
if len(docs) < 9 {
|
||||
kinds := make([]string, len(docs))
|
||||
for i, d := range docs {
|
||||
kinds[i] = d["kind"].(string)
|
||||
}
|
||||
t.Errorf("expected at least 9 YAML documents, got %d: %v", len(docs), kinds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgejoAdminSecret(t *testing.T) {
|
||||
docs := readForgejoDocuments(t)
|
||||
doc := findDoc(docs, "Secret", "forgejo-admin-secret")
|
||||
if doc == nil {
|
||||
t.Fatal("missing forgejo-admin-secret Secret")
|
||||
}
|
||||
meta := doc["metadata"].(map[string]interface{})
|
||||
if meta["namespace"] != "dev-infra" {
|
||||
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgejoConfigMap(t *testing.T) {
|
||||
docs := readForgejoDocuments(t)
|
||||
doc := findDoc(docs, "ConfigMap", "forgejo-config")
|
||||
if doc == nil {
|
||||
t.Fatal("missing forgejo-config ConfigMap")
|
||||
}
|
||||
|
||||
meta := doc["metadata"].(map[string]interface{})
|
||||
if meta["namespace"] != "dev-infra" {
|
||||
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
|
||||
}
|
||||
|
||||
data := doc["data"].(map[string]interface{})
|
||||
appIni, ok := data["app.ini"].(string)
|
||||
if !ok {
|
||||
t.Fatal("missing app.ini in ConfigMap data")
|
||||
}
|
||||
|
||||
requiredSettings := []string{
|
||||
"ROOT_URL = https://spinoff.dev/forgejo/",
|
||||
"HTTP_PORT = 3000",
|
||||
"DB_TYPE = sqlite3",
|
||||
"ENABLED = true",
|
||||
"DISABLE_REGISTRATION = true",
|
||||
"OFFLINE_MODE = true",
|
||||
"DEFAULT_BRANCH = main",
|
||||
}
|
||||
for _, s := range requiredSettings {
|
||||
if !strings.Contains(appIni, s) {
|
||||
t.Errorf("app.ini missing required setting: %s", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgejoPVC(t *testing.T) {
|
||||
docs := readForgejoDocuments(t)
|
||||
doc := findDoc(docs, "PersistentVolumeClaim", "forgejo-storage")
|
||||
if doc == nil {
|
||||
t.Fatal("missing forgejo-storage PVC")
|
||||
}
|
||||
|
||||
meta := doc["metadata"].(map[string]interface{})
|
||||
if meta["namespace"] != "dev-infra" {
|
||||
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
|
||||
}
|
||||
|
||||
spec := doc["spec"].(map[string]interface{})
|
||||
storageClass, ok := spec["storageClassName"].(string)
|
||||
if !ok || storageClass != "longhorn" {
|
||||
t.Errorf("expected storageClassName longhorn, got %v", storageClass)
|
||||
}
|
||||
|
||||
resources := spec["resources"].(map[string]interface{})
|
||||
requests := resources["requests"].(map[string]interface{})
|
||||
if requests["storage"] != "5Gi" {
|
||||
t.Errorf("expected 5Gi storage, got %v", requests["storage"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgejoDeployment(t *testing.T) {
|
||||
docs := readForgejoDocuments(t)
|
||||
doc := findDoc(docs, "Deployment", "forgejo")
|
||||
if doc == nil {
|
||||
t.Fatal("missing forgejo Deployment")
|
||||
}
|
||||
|
||||
meta := doc["metadata"].(map[string]interface{})
|
||||
if meta["namespace"] != "dev-infra" {
|
||||
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
|
||||
}
|
||||
|
||||
labels := meta["labels"].(map[string]interface{})
|
||||
if labels["app"] != "forgejo" {
|
||||
t.Errorf("expected label app=forgejo, got %v", labels["app"])
|
||||
}
|
||||
|
||||
spec := doc["spec"].(map[string]interface{})
|
||||
|
||||
strategy := spec["strategy"].(map[string]interface{})
|
||||
if strategy["type"] != "Recreate" {
|
||||
t.Errorf("expected Recreate strategy, got %v", strategy["type"])
|
||||
}
|
||||
|
||||
template := spec["template"].(map[string]interface{})
|
||||
podSpec := template["spec"].(map[string]interface{})
|
||||
|
||||
t.Run("init_containers", func(t *testing.T) {
|
||||
initContainers, ok := podSpec["initContainers"].([]interface{})
|
||||
if !ok || len(initContainers) < 2 {
|
||||
t.Fatal("expected at least 2 init containers (setup-config and create-admin)")
|
||||
}
|
||||
setupC := initContainers[0].(map[string]interface{})
|
||||
if setupC["name"] != "setup-config" {
|
||||
t.Errorf("expected first init container name setup-config, got %v", setupC["name"])
|
||||
}
|
||||
initC := initContainers[1].(map[string]interface{})
|
||||
if initC["name"] != "create-admin" {
|
||||
t.Errorf("expected second init container name create-admin, got %v", initC["name"])
|
||||
}
|
||||
image, _ := initC["image"].(string)
|
||||
if !strings.HasPrefix(image, "codeberg.org/forgejo/forgejo:") {
|
||||
t.Errorf("expected forgejo image, got %v", image)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("containers", func(t *testing.T) {
|
||||
containers := podSpec["containers"].([]interface{})
|
||||
if len(containers) != 2 {
|
||||
t.Fatalf("expected 2 containers (forgejo + ipip-sidecar), got %d", len(containers))
|
||||
}
|
||||
|
||||
forgejo := containers[0].(map[string]interface{})
|
||||
if forgejo["name"] != "forgejo" {
|
||||
t.Errorf("expected container name forgejo, got %v", forgejo["name"])
|
||||
}
|
||||
image, _ := forgejo["image"].(string)
|
||||
if !strings.HasPrefix(image, "codeberg.org/forgejo/forgejo:") {
|
||||
t.Errorf("expected forgejo image, got %v", image)
|
||||
}
|
||||
|
||||
ports := forgejo["ports"].([]interface{})
|
||||
if len(ports) != 1 {
|
||||
t.Errorf("expected 1 port, got %d", len(ports))
|
||||
}
|
||||
port := ports[0].(map[string]interface{})
|
||||
containerPort := toInt64(port["containerPort"])
|
||||
if containerPort != 3000 {
|
||||
t.Errorf("expected port 3000, got %v", port["containerPort"])
|
||||
}
|
||||
|
||||
ipip := containers[1].(map[string]interface{})
|
||||
if ipip["name"] != "ipip-sidecar" {
|
||||
t.Errorf("expected container name ipip-sidecar, got %v", ipip["name"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("volumes", func(t *testing.T) {
|
||||
volumes := podSpec["volumes"].([]interface{})
|
||||
if len(volumes) < 2 {
|
||||
t.Fatalf("expected at least 2 volumes (storage + config), got %d", len(volumes))
|
||||
}
|
||||
volumeNames := make(map[string]bool)
|
||||
for _, v := range volumes {
|
||||
vol := v.(map[string]interface{})
|
||||
volumeNames[vol["name"].(string)] = true
|
||||
}
|
||||
if !volumeNames["storage"] {
|
||||
t.Error("missing storage volume")
|
||||
}
|
||||
if !volumeNames["config"] {
|
||||
t.Error("missing config volume")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestForgejoService(t *testing.T) {
|
||||
docs := readForgejoDocuments(t)
|
||||
doc := findDoc(docs, "Service", "forgejo")
|
||||
if doc == nil {
|
||||
t.Fatal("missing forgejo Service")
|
||||
}
|
||||
|
||||
meta := doc["metadata"].(map[string]interface{})
|
||||
if meta["namespace"] != "dev-infra" {
|
||||
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
|
||||
}
|
||||
|
||||
spec := doc["spec"].(map[string]interface{})
|
||||
if spec["type"] != "ClusterIP" {
|
||||
t.Errorf("expected ClusterIP, got %v", spec["type"])
|
||||
}
|
||||
|
||||
ports := spec["ports"].([]interface{})
|
||||
if len(ports) != 1 {
|
||||
t.Fatalf("expected 1 port, got %d", len(ports))
|
||||
}
|
||||
port := ports[0].(map[string]interface{})
|
||||
portNum := toInt64(port["port"])
|
||||
if portNum != 3000 {
|
||||
t.Errorf("expected port 3000, got %v", port["port"])
|
||||
}
|
||||
|
||||
selector := spec["selector"].(map[string]interface{})
|
||||
if selector["app"] != "forgejo" {
|
||||
t.Errorf("expected selector app=forgejo, got %v", selector["app"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgejoIngressRoute(t *testing.T) {
|
||||
docs := readForgejoDocuments(t)
|
||||
doc := findDoc(docs, "IngressRoute", "forgejo-ingress")
|
||||
if doc == nil {
|
||||
t.Fatal("missing forgejo-ingress IngressRoute")
|
||||
}
|
||||
|
||||
meta := doc["metadata"].(map[string]interface{})
|
||||
if meta["namespace"] != "dev-infra" {
|
||||
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
|
||||
}
|
||||
|
||||
spec := doc["spec"].(map[string]interface{})
|
||||
entryPoints := spec["entryPoints"].([]interface{})
|
||||
if len(entryPoints) != 2 {
|
||||
t.Errorf("expected 2 entryPoints (web, websecure), got %d", len(entryPoints))
|
||||
}
|
||||
|
||||
routes := spec["routes"].([]interface{})
|
||||
if len(routes) < 2 {
|
||||
t.Fatalf("expected at least 2 routes (main + trailing-slash redirect), got %d", len(routes))
|
||||
}
|
||||
|
||||
mainRoute := routes[0].(map[string]interface{})
|
||||
matchRule, _ := mainRoute["match"].(string)
|
||||
if !strings.Contains(matchRule, "PathPrefix(`/forgejo/`)") {
|
||||
t.Errorf("expected PathPrefix /forgejo/ in main route, got %v", matchRule)
|
||||
}
|
||||
|
||||
middlewares := mainRoute["middlewares"].([]interface{})
|
||||
if len(middlewares) < 1 {
|
||||
t.Fatal("expected at least 1 middleware on main route")
|
||||
}
|
||||
mw := middlewares[0].(map[string]interface{})
|
||||
if mw["name"] != "forgejo-basic-auth" {
|
||||
t.Errorf("expected forgejo-basic-auth middleware, got %v", mw["name"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgejoBasicAuthMiddleware(t *testing.T) {
|
||||
docs := readForgejoDocuments(t)
|
||||
doc := findDoc(docs, "Middleware", "forgejo-basic-auth")
|
||||
if doc == nil {
|
||||
t.Fatal("missing forgejo-basic-auth Middleware")
|
||||
}
|
||||
|
||||
meta := doc["metadata"].(map[string]interface{})
|
||||
if meta["namespace"] != "dev-infra" {
|
||||
t.Errorf("expected namespace dev-infra, got %v", meta["namespace"])
|
||||
}
|
||||
|
||||
spec := doc["spec"].(map[string]interface{})
|
||||
basicAuth := spec["basicAuth"].(map[string]interface{})
|
||||
if basicAuth["secret"] != "forgejo-basic-auth-secret" {
|
||||
t.Errorf("expected secret forgejo-basic-auth-secret, got %v", basicAuth["secret"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgejoTrailingSlashMiddleware(t *testing.T) {
|
||||
docs := readForgejoDocuments(t)
|
||||
doc := findDoc(docs, "Middleware", "forgejo-add-trailing-slash")
|
||||
if doc == nil {
|
||||
t.Fatal("missing forgejo-add-trailing-slash Middleware")
|
||||
}
|
||||
|
||||
spec := doc["spec"].(map[string]interface{})
|
||||
redirect := spec["redirectRegex"].(map[string]interface{})
|
||||
if redirect["permanent"] != true {
|
||||
t.Error("expected permanent redirect")
|
||||
}
|
||||
}
|
||||
135
internal/k8s/metrics.go
Normal file
135
internal/k8s/metrics.go
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
// UsageRecorder records resource usage samples into the store.
|
||||
type UsageRecorder interface {
|
||||
RecordResourceSample(ctx context.Context, userID, podName string, cpuMillicores, memBytes float64) error
|
||||
}
|
||||
|
||||
// ResourceSampler periodically samples pod resource usage and records it.
|
||||
type ResourceSampler struct {
|
||||
client *Client
|
||||
recorder UsageRecorder
|
||||
logger *slog.Logger
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// NewResourceSampler creates a new resource sampler with a 60-second interval.
|
||||
func NewResourceSampler(client *Client, recorder UsageRecorder, logger *slog.Logger) *ResourceSampler {
|
||||
return &ResourceSampler{
|
||||
client: client,
|
||||
recorder: recorder,
|
||||
logger: logger,
|
||||
interval: 60 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// Start runs the sampling loop until the context is cancelled.
|
||||
func (s *ResourceSampler) Start(ctx context.Context) {
|
||||
ticker := time.NewTicker(s.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
s.logger.Info("resource sampler started", "interval", s.interval)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
s.logger.Info("resource sampler stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.sample(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// podMetricsList mirrors the k8s metrics API response.
|
||||
type podMetricsList struct {
|
||||
Items []podMetrics `json:"items"`
|
||||
}
|
||||
|
||||
type podMetrics struct {
|
||||
Metadata podMetricsMetadata `json:"metadata"`
|
||||
Containers []containerMetrics `json:"containers"`
|
||||
}
|
||||
|
||||
type podMetricsMetadata struct {
|
||||
Name string `json:"name"`
|
||||
Namespace string `json:"namespace"`
|
||||
Labels map[string]string `json:"labels"`
|
||||
}
|
||||
|
||||
type containerMetrics struct {
|
||||
Name string `json:"name"`
|
||||
Usage map[string]string `json:"usage"`
|
||||
}
|
||||
|
||||
func (s *ResourceSampler) sample(ctx context.Context) {
|
||||
// List dev-pod namespaces
|
||||
nsList, err := s.client.Clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{
|
||||
LabelSelector: "managed-by=dev-pod-api",
|
||||
})
|
||||
if err != nil {
|
||||
s.logger.Error("failed to list namespaces for sampling", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, ns := range nsList.Items {
|
||||
user := ns.Labels["user"]
|
||||
if user == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Query metrics API for this namespace
|
||||
data, err := s.client.Clientset.Discovery().RESTClient().Get().
|
||||
AbsPath(fmt.Sprintf("/apis/metrics.k8s.io/v1beta1/namespaces/%s/pods", ns.Name)).
|
||||
DoRaw(ctx)
|
||||
if err != nil {
|
||||
s.logger.Warn("failed to get pod metrics", "namespace", ns.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
var metrics podMetricsList
|
||||
if err := json.Unmarshal(data, &metrics); err != nil {
|
||||
s.logger.Warn("failed to parse pod metrics", "namespace", ns.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, pm := range metrics.Items {
|
||||
if !strings.HasPrefix(pm.Metadata.Name, "dev-pod-") {
|
||||
continue
|
||||
}
|
||||
podName := strings.TrimPrefix(pm.Metadata.Name, "dev-pod-")
|
||||
|
||||
var totalCPU, totalMem float64
|
||||
for _, cm := range pm.Containers {
|
||||
if cpuStr, ok := cm.Usage["cpu"]; ok {
|
||||
q, err := resource.ParseQuantity(cpuStr)
|
||||
if err == nil {
|
||||
totalCPU += float64(q.MilliValue())
|
||||
}
|
||||
}
|
||||
if memStr, ok := cm.Usage["memory"]; ok {
|
||||
q, err := resource.ParseQuantity(memStr)
|
||||
if err == nil {
|
||||
totalMem += float64(q.Value())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.recorder.RecordResourceSample(ctx, user, podName, totalCPU, totalMem); err != nil {
|
||||
s.logger.Warn("failed to record sample", "user", user, "pod", podName, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
49
internal/k8s/namespace.go
Normal file
49
internal/k8s/namespace.go
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
|
||||
)
|
||||
|
||||
// EnsureNamespace creates a namespace for the user if it doesn't already exist.
|
||||
// Returns nil if the namespace already exists (idempotent).
|
||||
func (c *Client) EnsureNamespace(ctx context.Context, user string) error {
|
||||
ns := &corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: model.NamespaceName(user),
|
||||
Labels: map[string]string{
|
||||
"app": "dev-pod",
|
||||
"managed-by": "dev-pod-api",
|
||||
"user": user,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := c.Clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if errors.IsAlreadyExists(err) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("create namespace %s: %w", ns.Name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteNamespace deletes the user's namespace and all resources in it.
|
||||
func (c *Client) DeleteNamespace(ctx context.Context, user string) error {
|
||||
nsName := model.NamespaceName(user)
|
||||
err := c.Clientset.CoreV1().Namespaces().Delete(ctx, nsName, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("delete namespace %s: %w", nsName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
87
internal/k8s/namespace_test.go
Normal file
87
internal/k8s/namespace_test.go
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
fakedynamic "k8s.io/client-go/dynamic/fake"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
func TestEnsureNamespace(t *testing.T) {
|
||||
t.Run("creates_new_namespace", func(t *testing.T) {
|
||||
client := NewClientWithClientset(fake.NewSimpleClientset(), fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), testCfg)
|
||||
ctx := context.Background()
|
||||
|
||||
err := client.EnsureNamespace(ctx, "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
ns, err := client.Clientset.CoreV1().Namespaces().Get(ctx, "dev-alice", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get namespace: %v", err)
|
||||
}
|
||||
if ns.Name != "dev-alice" {
|
||||
t.Errorf("expected namespace dev-alice, got %s", ns.Name)
|
||||
}
|
||||
if ns.Labels["app"] != "dev-pod" {
|
||||
t.Errorf("expected label app=dev-pod, got %s", ns.Labels["app"])
|
||||
}
|
||||
if ns.Labels["managed-by"] != "dev-pod-api" {
|
||||
t.Errorf("expected label managed-by=dev-pod-api, got %s", ns.Labels["managed-by"])
|
||||
}
|
||||
if ns.Labels["user"] != "alice" {
|
||||
t.Errorf("expected label user=alice, got %s", ns.Labels["user"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("idempotent_on_existing_namespace", func(t *testing.T) {
|
||||
client := NewClientWithClientset(fake.NewSimpleClientset(), fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), testCfg)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create twice - second call should not error
|
||||
err := client.EnsureNamespace(ctx, "bob")
|
||||
if err != nil {
|
||||
t.Fatalf("first create failed: %v", err)
|
||||
}
|
||||
err = client.EnsureNamespace(ctx, "bob")
|
||||
if err != nil {
|
||||
t.Fatalf("second create should be idempotent, got: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeleteNamespace(t *testing.T) {
|
||||
t.Run("deletes_existing_namespace", func(t *testing.T) {
|
||||
client := NewClientWithClientset(fake.NewSimpleClientset(), fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), testCfg)
|
||||
ctx := context.Background()
|
||||
|
||||
err := client.EnsureNamespace(ctx, "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("setup failed: %v", err)
|
||||
}
|
||||
|
||||
err = client.DeleteNamespace(ctx, "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("delete failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = client.Clientset.CoreV1().Namespaces().Get(ctx, "dev-alice", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
t.Error("expected namespace to be deleted")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no_error_on_nonexistent_namespace", func(t *testing.T) {
|
||||
client := NewClientWithClientset(fake.NewSimpleClientset(), fakedynamic.NewSimpleDynamicClient(runtime.NewScheme()), testCfg)
|
||||
ctx := context.Background()
|
||||
|
||||
err := client.DeleteNamespace(ctx, "nonexistent")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error for nonexistent namespace, got: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
501
internal/k8s/pods.go
Normal file
501
internal/k8s/pods.go
Normal file
|
|
@ -0,0 +1,501 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
|
||||
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrPodAlreadyExists = errors.New("pod already exists")
|
||||
ErrPodNotFound = errors.New("pod not found")
|
||||
ErrQuotaExceeded = errors.New("quota exceeded")
|
||||
)
|
||||
|
||||
var traefikIngressRouteGVR = schema.GroupVersionResource{
|
||||
Group: "traefik.io",
|
||||
Version: "v1alpha1",
|
||||
Resource: "ingressroutes",
|
||||
}
|
||||
|
||||
var traefikMiddlewareGVR = schema.GroupVersionResource{
|
||||
Group: "traefik.io",
|
||||
Version: "v1alpha1",
|
||||
Resource: "middlewares",
|
||||
}
|
||||
|
||||
// CreatePodOpts holds parameters for creating a complete pod with all resources.
|
||||
type CreatePodOpts struct {
|
||||
User string
|
||||
Pod string
|
||||
Tools string
|
||||
Task string
|
||||
CPUReq string
|
||||
CPULimit string
|
||||
MemReq string
|
||||
MemLimit string
|
||||
MaxConcurrentPods int // 0 = no limit
|
||||
MaxCPUPerPod int // 0 = no limit
|
||||
MaxRAMGBPerPod int // 0 = no limit
|
||||
ForgejoToken string
|
||||
TailscaleKey string
|
||||
}
|
||||
|
||||
// FetchVPNKey reads the VPN gateway key from the configured secret in the gateway namespace.
|
||||
func (c *Client) FetchVPNKey(ctx context.Context) (string, error) {
|
||||
secret, err := c.Clientset.CoreV1().Secrets(c.Config.VPNGatewayNS).Get(
|
||||
ctx, c.Config.VPNGatewaySecret, metav1.GetOptions{},
|
||||
)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get vpn gateway secret %s/%s: %w",
|
||||
c.Config.VPNGatewayNS, c.Config.VPNGatewaySecret, err)
|
||||
}
|
||||
|
||||
key, ok := secret.Data["VPN_GATEWAY_KEY"]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("VPN_GATEWAY_KEY field not found in secret %s/%s",
|
||||
c.Config.VPNGatewayNS, c.Config.VPNGatewaySecret)
|
||||
}
|
||||
return string(key), nil
|
||||
}
|
||||
|
||||
// ValidatePodQuota checks per-pod CPU and RAM limits against the user's quota.
|
||||
func ValidatePodQuota(cpuLimit string, maxCPUPerPod int, memLimit string, maxRAMGBPerPod int) error {
|
||||
if maxCPUPerPod > 0 && cpuLimit != "" {
|
||||
cpu, err := resource.ParseQuantity(cpuLimit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse cpu_limit %q: %w", cpuLimit, err)
|
||||
}
|
||||
maxCPU := resource.MustParse(fmt.Sprintf("%d", maxCPUPerPod))
|
||||
if cpu.Cmp(maxCPU) > 0 {
|
||||
return fmt.Errorf("%w: cpu_limit %s exceeds max %d", ErrQuotaExceeded, cpuLimit, maxCPUPerPod)
|
||||
}
|
||||
}
|
||||
if maxRAMGBPerPod > 0 && memLimit != "" {
|
||||
mem, err := resource.ParseQuantity(memLimit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse mem_limit %q: %w", memLimit, err)
|
||||
}
|
||||
maxMem := resource.MustParse(fmt.Sprintf("%dGi", maxRAMGBPerPod))
|
||||
if mem.Cmp(maxMem) > 0 {
|
||||
return fmt.Errorf("%w: mem_limit %s exceeds max %dGi", ErrQuotaExceeded, memLimit, maxRAMGBPerPod)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreatePod creates a complete dev pod with all associated k8s resources.
|
||||
// Shared resources (namespace, PVC, secrets, configmap, network policy) are created idempotently.
|
||||
// Pod-specific resources (pod, service, ingress) are cleaned up on partial failure.
|
||||
func (c *Client) CreatePod(ctx context.Context, opts CreatePodOpts) (*model.Pod, error) {
|
||||
ns := model.NamespaceName(opts.User)
|
||||
|
||||
// Check quota: count existing pods for this user
|
||||
if opts.MaxConcurrentPods > 0 {
|
||||
existingPods, err := c.Clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: "app=dev-pod",
|
||||
})
|
||||
if err != nil && !k8serrors.IsNotFound(err) {
|
||||
return nil, fmt.Errorf("list existing pods: %w", err)
|
||||
}
|
||||
if existingPods != nil && len(existingPods.Items) >= opts.MaxConcurrentPods {
|
||||
return nil, ErrQuotaExceeded
|
||||
}
|
||||
}
|
||||
|
||||
// Check per-pod resource quota
|
||||
if err := ValidatePodQuota(opts.CPULimit, opts.MaxCPUPerPod, opts.MemLimit, opts.MaxRAMGBPerPod); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check for duplicate pod
|
||||
podFullName := model.PodFullName(opts.Pod)
|
||||
_, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, podFullName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
return nil, ErrPodAlreadyExists
|
||||
}
|
||||
if !k8serrors.IsNotFound(err) {
|
||||
return nil, fmt.Errorf("check existing pod: %w", err)
|
||||
}
|
||||
|
||||
// Fetch VPN key
|
||||
vpnKey, err := c.FetchVPNKey(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch vpn key: %w", err)
|
||||
}
|
||||
|
||||
// Create shared resources (idempotent)
|
||||
if err := c.EnsureNamespace(ctx, opts.User); err != nil {
|
||||
return nil, fmt.Errorf("ensure namespace: %w", err)
|
||||
}
|
||||
|
||||
if err := c.ensureSharedResources(ctx, opts.User, vpnKey, opts.ForgejoToken, opts.TailscaleKey); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create per-pod PVC
|
||||
pvc := PVCTemplate(opts.User, opts.Pod)
|
||||
if _, err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Create(ctx, pvc, metav1.CreateOptions{}); err != nil {
|
||||
if !k8serrors.IsAlreadyExists(err) {
|
||||
return nil, fmt.Errorf("create per-pod pvc: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create pod-specific resources with cleanup on failure
|
||||
podOpts := PodOpts{
|
||||
User: opts.User,
|
||||
Pod: opts.Pod,
|
||||
Tools: opts.Tools,
|
||||
Task: opts.Task,
|
||||
CPUReq: opts.CPUReq,
|
||||
CPULimit: opts.CPULimit,
|
||||
MemReq: opts.MemReq,
|
||||
MemLimit: opts.MemLimit,
|
||||
VPNKey: vpnKey,
|
||||
AnthropicKey: c.Config.AnthropicKey,
|
||||
OpenAIKey: c.Config.OpenAIKey,
|
||||
ForgejoToken: opts.ForgejoToken,
|
||||
TailscaleKey: opts.TailscaleKey,
|
||||
}
|
||||
|
||||
var cleanups []func()
|
||||
doCleanup := func() {
|
||||
for i := len(cleanups) - 1; i >= 0; i-- {
|
||||
cleanups[i]()
|
||||
}
|
||||
}
|
||||
|
||||
// Register PVC cleanup
|
||||
cleanups = append(cleanups, func() {
|
||||
if delErr := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, pvc.Name, metav1.DeleteOptions{}); delErr != nil {
|
||||
slog.Warn("cleanup: failed to delete pvc", "pvc", pvc.Name, "error", delErr)
|
||||
}
|
||||
})
|
||||
|
||||
// Create Pod
|
||||
pod := PodTemplate(c.Config, podOpts)
|
||||
if _, err := c.Clientset.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil {
|
||||
return nil, fmt.Errorf("create pod: %w", err)
|
||||
}
|
||||
cleanups = append(cleanups, func() {
|
||||
if delErr := c.Clientset.CoreV1().Pods(ns).Delete(ctx, pod.Name, metav1.DeleteOptions{}); delErr != nil {
|
||||
slog.Warn("cleanup: failed to delete pod", "pod", pod.Name, "error", delErr)
|
||||
}
|
||||
})
|
||||
|
||||
// Create Service
|
||||
svc := ServiceTemplate(opts.User, opts.Pod)
|
||||
if _, err := c.Clientset.CoreV1().Services(ns).Create(ctx, svc, metav1.CreateOptions{}); err != nil {
|
||||
doCleanup()
|
||||
return nil, fmt.Errorf("create service: %w", err)
|
||||
}
|
||||
cleanups = append(cleanups, func() {
|
||||
if delErr := c.Clientset.CoreV1().Services(ns).Delete(ctx, svc.Name, metav1.DeleteOptions{}); delErr != nil {
|
||||
slog.Warn("cleanup: failed to delete service", "service", svc.Name, "error", delErr)
|
||||
}
|
||||
})
|
||||
|
||||
// Create IngressRoute + Middlewares
|
||||
ingressObjs := IngressTemplate(opts.User, opts.Pod, c.Config.Domain)
|
||||
for _, obj := range ingressObjs {
|
||||
gvr, err := traefikGVR(obj.GetKind())
|
||||
if err != nil {
|
||||
doCleanup()
|
||||
return nil, fmt.Errorf("resolve GVR for %s: %w", obj.GetKind(), err)
|
||||
}
|
||||
_, err = c.Dynamic.Resource(gvr).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if k8serrors.IsAlreadyExists(err) {
|
||||
continue // shared resource already exists, don't register cleanup
|
||||
}
|
||||
doCleanup()
|
||||
return nil, fmt.Errorf("create %s %s: %w", obj.GetKind(), obj.GetName(), err)
|
||||
}
|
||||
name := obj.GetName()
|
||||
kind := obj.GetKind()
|
||||
cleanups = append(cleanups, func() {
|
||||
gvr, gvrErr := traefikGVR(kind)
|
||||
if gvrErr != nil {
|
||||
slog.Warn("cleanup: unknown traefik kind", "kind", kind, "error", gvrErr)
|
||||
return
|
||||
}
|
||||
if delErr := c.Dynamic.Resource(gvr).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}); delErr != nil {
|
||||
slog.Warn("cleanup: failed to delete "+kind, "name", name, "error", delErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
return &model.Pod{
|
||||
User: opts.User,
|
||||
Name: opts.Pod,
|
||||
Tools: opts.Tools,
|
||||
CPUReq: opts.CPUReq,
|
||||
CPULimit: opts.CPULimit,
|
||||
MemReq: opts.MemReq,
|
||||
MemLimit: opts.MemLimit,
|
||||
Task: opts.Task,
|
||||
Status: "Pending",
|
||||
URL: model.PodURL(c.Config.Domain, opts.User, opts.Pod),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DeletePod removes a single pod and its associated service and ingress resources.
|
||||
// Keeps the namespace and shared resources if other pods exist.
|
||||
func (c *Client) DeletePod(ctx context.Context, user, pod string) error {
|
||||
ns := model.NamespaceName(user)
|
||||
podFullName := model.PodFullName(pod)
|
||||
|
||||
// Verify pod exists
|
||||
_, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, podFullName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if k8serrors.IsNotFound(err) {
|
||||
return ErrPodNotFound
|
||||
}
|
||||
return fmt.Errorf("get pod: %w", err)
|
||||
}
|
||||
|
||||
// Delete pod
|
||||
if err := c.Clientset.CoreV1().Pods(ns).Delete(ctx, podFullName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
|
||||
return fmt.Errorf("delete pod: %w", err)
|
||||
}
|
||||
|
||||
// Delete service
|
||||
svcName := model.ServiceName(pod)
|
||||
if err := c.Clientset.CoreV1().Services(ns).Delete(ctx, svcName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
|
||||
return fmt.Errorf("delete service: %w", err)
|
||||
}
|
||||
|
||||
// Delete ingress route and middlewares
|
||||
ingressName := fmt.Sprintf("dev-pod-%s-ingress", pod)
|
||||
if err := c.Dynamic.Resource(traefikIngressRouteGVR).Namespace(ns).Delete(ctx, ingressName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
|
||||
return fmt.Errorf("delete ingress route: %w", err)
|
||||
}
|
||||
|
||||
// Delete middlewares: basic-auth is shared, only delete pod-specific strip-prefix
|
||||
stripName := fmt.Sprintf("strip-dev-%s-ralphex-prefix", pod)
|
||||
if err := c.Dynamic.Resource(traefikMiddlewareGVR).Namespace(ns).Delete(ctx, stripName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
|
||||
return fmt.Errorf("delete strip-prefix middleware: %w", err)
|
||||
}
|
||||
|
||||
// Delete per-pod PVC
|
||||
pvcName := PVCName(pod)
|
||||
if err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) {
|
||||
return fmt.Errorf("delete per-pod pvc: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAllPods removes all pods for a user by deleting the entire namespace.
|
||||
func (c *Client) DeleteAllPods(ctx context.Context, user string) error {
|
||||
return c.DeleteNamespace(ctx, user)
|
||||
}
|
||||
|
||||
// ListPods returns all dev pods for a user with status and age.
|
||||
func (c *Client) ListPods(ctx context.Context, user string) ([]model.Pod, error) {
|
||||
ns := model.NamespaceName(user)
|
||||
|
||||
podList, err := c.Clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: "app=dev-pod",
|
||||
})
|
||||
if err != nil {
|
||||
if k8serrors.IsNotFound(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("list pods: %w", err)
|
||||
}
|
||||
|
||||
result := make([]model.Pod, 0, len(podList.Items))
|
||||
for _, p := range podList.Items {
|
||||
podName := p.Labels["podname"]
|
||||
cpuReq, cpuLimit, memReq, memLimit := containerResources(p.Spec.Containers)
|
||||
result = append(result, model.Pod{
|
||||
User: user,
|
||||
Name: podName,
|
||||
Tools: extractEnvVar(p, "dev", "DEV_TOOLS"),
|
||||
CPUReq: cpuReq,
|
||||
CPULimit: cpuLimit,
|
||||
MemReq: memReq,
|
||||
MemLimit: memLimit,
|
||||
Task: extractEnvVar(p, "dev", "TASK_DESCRIPTION"),
|
||||
Status: string(p.Status.Phase),
|
||||
Age: formatAge(p.CreationTimestamp.Time),
|
||||
URL: model.PodURL(c.Config.Domain, user, podName),
|
||||
CreatedAt: p.CreationTimestamp.Time,
|
||||
Labels: p.Labels,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetPod returns detailed info about a single dev pod.
|
||||
func (c *Client) GetPod(ctx context.Context, user, pod string) (*model.Pod, error) {
|
||||
ns := model.NamespaceName(user)
|
||||
podFullName := model.PodFullName(pod)
|
||||
|
||||
p, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, podFullName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if k8serrors.IsNotFound(err) {
|
||||
return nil, ErrPodNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("get pod: %w", err)
|
||||
}
|
||||
|
||||
cpuReq, cpuLimit, memReq, memLimit := containerResources(p.Spec.Containers)
|
||||
return &model.Pod{
|
||||
User: user,
|
||||
Name: pod,
|
||||
Tools: extractEnvVar(*p, "dev", "DEV_TOOLS"),
|
||||
CPUReq: cpuReq,
|
||||
CPULimit: cpuLimit,
|
||||
MemReq: memReq,
|
||||
MemLimit: memLimit,
|
||||
Task: extractEnvVar(*p, "dev", "TASK_DESCRIPTION"),
|
||||
Status: string(p.Status.Phase),
|
||||
Age: formatAge(p.CreationTimestamp.Time),
|
||||
URL: model.PodURL(c.Config.Domain, user, pod),
|
||||
CreatedAt: p.CreationTimestamp.Time,
|
||||
Labels: p.Labels,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ensureSharedResources creates namespace-level resources idempotently.
|
||||
func (c *Client) ensureSharedResources(ctx context.Context, user, vpnKey, forgejoToken, tailscaleKey string) error {
|
||||
ns := model.NamespaceName(user)
|
||||
|
||||
// Secrets
|
||||
devSec, aiSec := SecretsTemplate(user, vpnKey, c.Config.AnthropicKey, c.Config.OpenAIKey, forgejoToken, tailscaleKey)
|
||||
if _, err := c.Clientset.CoreV1().Secrets(ns).Create(ctx, devSec, metav1.CreateOptions{}); err != nil {
|
||||
if k8serrors.IsAlreadyExists(err) {
|
||||
existing, getErr := c.Clientset.CoreV1().Secrets(ns).Get(ctx, devSec.Name, metav1.GetOptions{})
|
||||
if getErr != nil {
|
||||
return fmt.Errorf("get dev-secrets for update: %w", getErr)
|
||||
}
|
||||
devSec.ResourceVersion = existing.ResourceVersion
|
||||
if _, err = c.Clientset.CoreV1().Secrets(ns).Update(ctx, devSec, metav1.UpdateOptions{}); err != nil {
|
||||
return fmt.Errorf("update dev-secrets: %w", err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("create dev-secrets: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err := c.Clientset.CoreV1().Secrets(ns).Create(ctx, aiSec, metav1.CreateOptions{}); err != nil {
|
||||
if k8serrors.IsAlreadyExists(err) {
|
||||
existing, getErr := c.Clientset.CoreV1().Secrets(ns).Get(ctx, aiSec.Name, metav1.GetOptions{})
|
||||
if getErr != nil {
|
||||
return fmt.Errorf("get ai-proxy-secrets for update: %w", getErr)
|
||||
}
|
||||
aiSec.ResourceVersion = existing.ResourceVersion
|
||||
if _, err = c.Clientset.CoreV1().Secrets(ns).Update(ctx, aiSec, metav1.UpdateOptions{}); err != nil {
|
||||
return fmt.Errorf("update ai-proxy-secrets: %w", err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("create ai-proxy-secrets: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Basic auth secret (for Traefik basicAuth middleware)
|
||||
basicAuthSec := BasicAuthSecretTemplate(user)
|
||||
if _, err := c.Clientset.CoreV1().Secrets(ns).Create(ctx, basicAuthSec, metav1.CreateOptions{}); err != nil && !k8serrors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create basic-auth secret: %w", err)
|
||||
}
|
||||
|
||||
// ConfigMap
|
||||
cm := AIProxyConfigMapTemplate(user)
|
||||
if _, err := c.Clientset.CoreV1().ConfigMaps(ns).Create(ctx, cm, metav1.CreateOptions{}); err != nil {
|
||||
if k8serrors.IsAlreadyExists(err) {
|
||||
existing, getErr := c.Clientset.CoreV1().ConfigMaps(ns).Get(ctx, cm.Name, metav1.GetOptions{})
|
||||
if getErr != nil {
|
||||
return fmt.Errorf("get ai-proxy-config for update: %w", getErr)
|
||||
}
|
||||
cm.ResourceVersion = existing.ResourceVersion
|
||||
if _, err = c.Clientset.CoreV1().ConfigMaps(ns).Update(ctx, cm, metav1.UpdateOptions{}); err != nil {
|
||||
return fmt.Errorf("update ai-proxy-config: %w", err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("create ai-proxy-config: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// NetworkPolicy
|
||||
np := NetworkPolicyTemplate(user)
|
||||
if _, err := c.Clientset.NetworkingV1().NetworkPolicies(ns).Create(ctx, np, metav1.CreateOptions{}); err != nil && !k8serrors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create network policy: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// traefikGVR returns the GroupVersionResource for a Traefik CRD kind.
|
||||
func traefikGVR(kind string) (schema.GroupVersionResource, error) {
|
||||
switch kind {
|
||||
case "IngressRoute":
|
||||
return traefikIngressRouteGVR, nil
|
||||
case "Middleware":
|
||||
return traefikMiddlewareGVR, nil
|
||||
default:
|
||||
return schema.GroupVersionResource{}, fmt.Errorf("unsupported traefik kind: %s", kind)
|
||||
}
|
||||
}
|
||||
|
||||
// containerResources safely extracts CPU/memory request/limit strings from the first container.
|
||||
func containerResources(containers []corev1.Container) (cpuReq, cpuLimit, memReq, memLimit string) {
|
||||
if len(containers) == 0 {
|
||||
return "0", "0", "0", "0"
|
||||
}
|
||||
c := containers[0]
|
||||
return c.Resources.Requests.Cpu().String(),
|
||||
c.Resources.Limits.Cpu().String(),
|
||||
formatMemory(c.Resources.Requests.Memory()),
|
||||
formatMemory(c.Resources.Limits.Memory())
|
||||
}
|
||||
|
||||
// extractEnvVar finds an environment variable value from a named container in a pod.
|
||||
func extractEnvVar(pod corev1.Pod, containerName, envName string) string {
|
||||
for _, c := range pod.Spec.Containers {
|
||||
if c.Name == containerName {
|
||||
for _, e := range c.Env {
|
||||
if e.Name == envName {
|
||||
return e.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// formatAge returns a human-readable duration since the given time.
|
||||
func formatAge(created time.Time) string {
|
||||
if created.IsZero() {
|
||||
return ""
|
||||
}
|
||||
d := time.Since(created)
|
||||
switch {
|
||||
case d < time.Minute:
|
||||
return fmt.Sprintf("%ds", int(d.Seconds()))
|
||||
case d < time.Hour:
|
||||
return fmt.Sprintf("%dm", int(d.Minutes()))
|
||||
case d < 24*time.Hour:
|
||||
return fmt.Sprintf("%dh", int(d.Hours()))
|
||||
default:
|
||||
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
||||
}
|
||||
}
|
||||
|
||||
// formatMemory returns a human-readable memory quantity.
|
||||
func formatMemory(q *resource.Quantity) string {
|
||||
if q == nil || q.IsZero() {
|
||||
return "0"
|
||||
}
|
||||
return q.String()
|
||||
}
|
||||
592
internal/k8s/pods_test.go
Normal file
592
internal/k8s/pods_test.go
Normal file
|
|
@ -0,0 +1,592 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
fakedynamic "k8s.io/client-go/dynamic/fake"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
|
||||
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
|
||||
)
|
||||
|
||||
// newTestClient creates a test client with fake k8s and dynamic clients,
|
||||
// and pre-seeds the VPN gateway secret.
|
||||
func newTestClient() *Client {
|
||||
fakeClient := fake.NewSimpleClientset()
|
||||
scheme := runtime.NewScheme()
|
||||
fakeDyn := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme,
|
||||
map[schema.GroupVersionResource]string{
|
||||
traefikIngressRouteGVR: "IngressRouteList",
|
||||
traefikMiddlewareGVR: "MiddlewareList",
|
||||
},
|
||||
)
|
||||
client := NewClientWithClientset(fakeClient, fakeDyn, testCfg)
|
||||
|
||||
// Seed VPN gateway secret
|
||||
ctx := context.Background()
|
||||
vpnSecret := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: testCfg.VPNGatewaySecret,
|
||||
Namespace: testCfg.VPNGatewayNS,
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
"VPN_GATEWAY_KEY": []byte("test-vpn-key-123"),
|
||||
},
|
||||
}
|
||||
_, _ = fakeClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: testCfg.VPNGatewayNS},
|
||||
}, metav1.CreateOptions{})
|
||||
_, _ = fakeClient.CoreV1().Secrets(testCfg.VPNGatewayNS).Create(ctx, vpnSecret, metav1.CreateOptions{})
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
func defaultCreateOpts() CreatePodOpts {
|
||||
return CreatePodOpts{
|
||||
User: "alice",
|
||||
Pod: "main",
|
||||
Tools: "go@1.25,rust@1.94",
|
||||
Task: "build a web server",
|
||||
CPUReq: "2",
|
||||
CPULimit: "4",
|
||||
MemReq: "4Gi",
|
||||
MemLimit: "8Gi",
|
||||
MaxConcurrentPods: 3,
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchVPNKey(t *testing.T) {
|
||||
t.Run("success", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
key, err := client.FetchVPNKey(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if key != "test-vpn-key-123" {
|
||||
t.Errorf("expected test-vpn-key-123, got %s", key)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("secret_not_found", func(t *testing.T) {
|
||||
fakeClient := fake.NewSimpleClientset()
|
||||
fakeDyn := fakedynamic.NewSimpleDynamicClient(runtime.NewScheme())
|
||||
client := NewClientWithClientset(fakeClient, fakeDyn, testCfg)
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := client.FetchVPNKey(ctx)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing secret")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("key_field_missing", func(t *testing.T) {
|
||||
fakeClient := fake.NewSimpleClientset()
|
||||
fakeDyn := fakedynamic.NewSimpleDynamicClient(runtime.NewScheme())
|
||||
client := NewClientWithClientset(fakeClient, fakeDyn, testCfg)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create secret without vpn-key field
|
||||
_, _ = fakeClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: testCfg.VPNGatewayNS},
|
||||
}, metav1.CreateOptions{})
|
||||
_, _ = fakeClient.CoreV1().Secrets(testCfg.VPNGatewayNS).Create(ctx, &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: testCfg.VPNGatewaySecret,
|
||||
Namespace: testCfg.VPNGatewayNS,
|
||||
},
|
||||
Data: map[string][]byte{"other-key": []byte("value")},
|
||||
}, metav1.CreateOptions{})
|
||||
|
||||
_, err := client.FetchVPNKey(ctx)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing vpn-key field")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCreatePod(t *testing.T) {
|
||||
t.Run("success", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
opts := defaultCreateOpts()
|
||||
|
||||
result, err := client.CreatePod(ctx, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if result.User != "alice" {
|
||||
t.Errorf("expected user alice, got %s", result.User)
|
||||
}
|
||||
if result.Name != "main" {
|
||||
t.Errorf("expected pod name main, got %s", result.Name)
|
||||
}
|
||||
if result.Status != "Pending" {
|
||||
t.Errorf("expected status Pending, got %s", result.Status)
|
||||
}
|
||||
if result.URL != "https://spinoff.dev/@alice/main/" {
|
||||
t.Errorf("expected URL https://spinoff.dev/@alice/main/, got %s", result.URL)
|
||||
}
|
||||
if result.Tools != "go@1.25,rust@1.94" {
|
||||
t.Errorf("expected tools go@1.25,rust@1.94, got %s", result.Tools)
|
||||
}
|
||||
|
||||
// Verify k8s resources were created
|
||||
ns := model.NamespaceName("alice")
|
||||
|
||||
// Namespace
|
||||
_, err = client.Clientset.CoreV1().Namespaces().Get(ctx, ns, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("namespace not created: %v", err)
|
||||
}
|
||||
|
||||
// Pod
|
||||
_, err = client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-main", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("pod not created: %v", err)
|
||||
}
|
||||
|
||||
// Service
|
||||
_, err = client.Clientset.CoreV1().Services(ns).Get(ctx, "dev-pod-main-svc", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("service not created: %v", err)
|
||||
}
|
||||
|
||||
// Per-pod PVC
|
||||
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("per-pod pvc not created: %v", err)
|
||||
}
|
||||
|
||||
// Secrets
|
||||
_, err = client.Clientset.CoreV1().Secrets(ns).Get(ctx, "dev-secrets", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("dev-secrets not created: %v", err)
|
||||
}
|
||||
_, err = client.Clientset.CoreV1().Secrets(ns).Get(ctx, "ai-proxy-secrets", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("ai-proxy-secrets not created: %v", err)
|
||||
}
|
||||
|
||||
// ConfigMap
|
||||
_, err = client.Clientset.CoreV1().ConfigMaps(ns).Get(ctx, "ai-proxy-config", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("ai-proxy-config not created: %v", err)
|
||||
}
|
||||
|
||||
// NetworkPolicy
|
||||
_, err = client.Clientset.NetworkingV1().NetworkPolicies(ns).Get(ctx, "dev-pod-network-policy", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("network policy not created: %v", err)
|
||||
}
|
||||
|
||||
// IngressRoute (via dynamic client)
|
||||
_, err = client.Dynamic.Resource(traefikIngressRouteGVR).Namespace(ns).Get(ctx, "dev-pod-main-ingress", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("ingress route not created: %v", err)
|
||||
}
|
||||
|
||||
// Middlewares
|
||||
_, err = client.Dynamic.Resource(traefikMiddlewareGVR).Namespace(ns).Get(ctx, "spinoff-basic-auth", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("basic-auth middleware not created: %v", err)
|
||||
}
|
||||
_, err = client.Dynamic.Resource(traefikMiddlewareGVR).Namespace(ns).Get(ctx, "strip-dev-main-ralphex-prefix", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("strip-prefix middleware not created: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("quota_exceeded", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create 2 pods to fill quota of 2
|
||||
opts := defaultCreateOpts()
|
||||
opts.MaxConcurrentPods = 2
|
||||
opts.Pod = "pod1"
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("first pod failed: %v", err)
|
||||
}
|
||||
opts.Pod = "pod2"
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("second pod failed: %v", err)
|
||||
}
|
||||
|
||||
// Third pod should fail
|
||||
opts.Pod = "pod3"
|
||||
_, err := client.CreatePod(ctx, opts)
|
||||
if !errors.Is(err, ErrQuotaExceeded) {
|
||||
t.Errorf("expected ErrQuotaExceeded, got: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("duplicate_pod", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
opts := defaultCreateOpts()
|
||||
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("first create failed: %v", err)
|
||||
}
|
||||
|
||||
_, err := client.CreatePod(ctx, opts)
|
||||
if !errors.Is(err, ErrPodAlreadyExists) {
|
||||
t.Errorf("expected ErrPodAlreadyExists, got: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("shared_resources_idempotent", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
opts := defaultCreateOpts()
|
||||
|
||||
// Create first pod
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("first pod failed: %v", err)
|
||||
}
|
||||
|
||||
// Create second pod for same user
|
||||
opts.Pod = "secondary"
|
||||
result, err := client.CreatePod(ctx, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("second pod failed: %v", err)
|
||||
}
|
||||
if result.Name != "secondary" {
|
||||
t.Errorf("expected pod name secondary, got %s", result.Name)
|
||||
}
|
||||
|
||||
// Both pods should exist
|
||||
ns := model.NamespaceName("alice")
|
||||
pods, err := client.Clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list pods: %v", err)
|
||||
}
|
||||
if len(pods.Items) != 2 {
|
||||
t.Errorf("expected 2 pods, got %d", len(pods.Items))
|
||||
}
|
||||
|
||||
// Each pod should have its own PVC
|
||||
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("workspace-main PVC not created: %v", err)
|
||||
}
|
||||
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-secondary", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("workspace-secondary PVC not created: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("secrets_contain_dummy_and_real_keys", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
opts := defaultCreateOpts()
|
||||
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("create failed: %v", err)
|
||||
}
|
||||
|
||||
ns := model.NamespaceName("alice")
|
||||
|
||||
// dev-secrets should have dummy keys (fake clientset preserves StringData)
|
||||
devSec, _ := client.Clientset.CoreV1().Secrets(ns).Get(ctx, "dev-secrets", metav1.GetOptions{})
|
||||
if devSec.StringData["ANTHROPIC_API_KEY"] != "sk-devpod" {
|
||||
t.Errorf("dev-secrets should have dummy anthropic key, got %s", devSec.StringData["ANTHROPIC_API_KEY"])
|
||||
}
|
||||
if devSec.StringData["VPN_GATEWAY_KEY"] != "test-vpn-key-123" {
|
||||
t.Errorf("dev-secrets should have VPN key, got %s", devSec.StringData["VPN_GATEWAY_KEY"])
|
||||
}
|
||||
|
||||
// ai-proxy-secrets should have real keys
|
||||
aiSec, _ := client.Clientset.CoreV1().Secrets(ns).Get(ctx, "ai-proxy-secrets", metav1.GetOptions{})
|
||||
if aiSec.StringData["anthropic-key"] != "br_test_anthropic" {
|
||||
t.Errorf("ai-proxy-secrets should have real anthropic key, got %s", aiSec.StringData["anthropic-key"])
|
||||
}
|
||||
if aiSec.StringData["openai-key"] != "br_test_openai" {
|
||||
t.Errorf("ai-proxy-secrets should have real openai key, got %s", aiSec.StringData["openai-key"])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeletePod(t *testing.T) {
|
||||
t.Run("deletes_existing_pod", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
opts := defaultCreateOpts()
|
||||
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("setup: create pod failed: %v", err)
|
||||
}
|
||||
|
||||
err := client.DeletePod(ctx, "alice", "main")
|
||||
if err != nil {
|
||||
t.Fatalf("delete failed: %v", err)
|
||||
}
|
||||
|
||||
ns := model.NamespaceName("alice")
|
||||
|
||||
// Pod should be gone
|
||||
_, err = client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-main", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
t.Error("pod should have been deleted")
|
||||
}
|
||||
|
||||
// Service should be gone
|
||||
_, err = client.Clientset.CoreV1().Services(ns).Get(ctx, "dev-pod-main-svc", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
t.Error("service should have been deleted")
|
||||
}
|
||||
|
||||
// IngressRoute should be gone
|
||||
_, err = client.Dynamic.Resource(traefikIngressRouteGVR).Namespace(ns).Get(ctx, "dev-pod-main-ingress", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
t.Error("ingress route should have been deleted")
|
||||
}
|
||||
|
||||
// Per-pod PVC should be gone
|
||||
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
t.Error("per-pod PVC should have been deleted")
|
||||
}
|
||||
|
||||
// Namespace should still exist
|
||||
_, err = client.Clientset.CoreV1().Namespaces().Get(ctx, ns, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("namespace should still exist: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("not_found", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create namespace so the "get pod" doesn't fail on namespace
|
||||
_ = client.EnsureNamespace(ctx, "alice")
|
||||
|
||||
err := client.DeletePod(ctx, "alice", "nonexistent")
|
||||
if !errors.Is(err, ErrPodNotFound) {
|
||||
t.Errorf("expected ErrPodNotFound, got: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("keeps_other_pods", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create two pods
|
||||
opts := defaultCreateOpts()
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("setup: create pod1 failed: %v", err)
|
||||
}
|
||||
opts.Pod = "secondary"
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("setup: create pod2 failed: %v", err)
|
||||
}
|
||||
|
||||
// Delete only the first
|
||||
if err := client.DeletePod(ctx, "alice", "main"); err != nil {
|
||||
t.Fatalf("delete failed: %v", err)
|
||||
}
|
||||
|
||||
ns := model.NamespaceName("alice")
|
||||
|
||||
// First pod gone
|
||||
_, err := client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-main", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
t.Error("first pod should be deleted")
|
||||
}
|
||||
|
||||
// First pod's PVC gone
|
||||
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-main", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
t.Error("first pod's PVC should be deleted")
|
||||
}
|
||||
|
||||
// Second pod still exists
|
||||
_, err = client.Clientset.CoreV1().Pods(ns).Get(ctx, "dev-pod-secondary", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("second pod should still exist: %v", err)
|
||||
}
|
||||
|
||||
// Second pod's PVC still exists
|
||||
_, err = client.Clientset.CoreV1().PersistentVolumeClaims(ns).Get(ctx, "workspace-secondary", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Errorf("second pod's PVC should still exist: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeleteAllPods(t *testing.T) {
|
||||
t.Run("deletes_entire_namespace", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
opts := defaultCreateOpts()
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("setup: create pod failed: %v", err)
|
||||
}
|
||||
|
||||
if err := client.DeleteAllPods(ctx, "alice"); err != nil {
|
||||
t.Fatalf("delete all failed: %v", err)
|
||||
}
|
||||
|
||||
_, err := client.Clientset.CoreV1().Namespaces().Get(ctx, "dev-alice", metav1.GetOptions{})
|
||||
if err == nil {
|
||||
t.Error("namespace should have been deleted")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("idempotent_on_nonexistent", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := client.DeleteAllPods(ctx, "nonexistent"); err != nil {
|
||||
t.Fatalf("expected no error, got: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestListPods(t *testing.T) {
|
||||
t.Run("empty_namespace", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
pods, err := client.ListPods(ctx, "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(pods) != 0 {
|
||||
t.Errorf("expected 0 pods, got %d", len(pods))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("multiple_pods", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
opts := defaultCreateOpts()
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("create pod1 failed: %v", err)
|
||||
}
|
||||
opts.Pod = "secondary"
|
||||
opts.Tools = "python@3.12"
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("create pod2 failed: %v", err)
|
||||
}
|
||||
|
||||
pods, err := client.ListPods(ctx, "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("list failed: %v", err)
|
||||
}
|
||||
if len(pods) != 2 {
|
||||
t.Fatalf("expected 2 pods, got %d", len(pods))
|
||||
}
|
||||
|
||||
// Check pod details are populated
|
||||
podNames := make(map[string]bool)
|
||||
for _, p := range pods {
|
||||
podNames[p.Name] = true
|
||||
if p.User != "alice" {
|
||||
t.Errorf("expected user alice, got %s", p.User)
|
||||
}
|
||||
if p.URL == "" {
|
||||
t.Error("expected non-empty URL")
|
||||
}
|
||||
}
|
||||
if !podNames["main"] || !podNames["secondary"] {
|
||||
t.Errorf("expected pods main and secondary, got %v", podNames)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("multiple_users_isolated", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
// Alice's pod
|
||||
opts := defaultCreateOpts()
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("create alice pod failed: %v", err)
|
||||
}
|
||||
|
||||
// Bob's pod
|
||||
opts.User = "bob"
|
||||
opts.Pod = "dev"
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("create bob pod failed: %v", err)
|
||||
}
|
||||
|
||||
// Alice should see only her pod
|
||||
alicePods, err := client.ListPods(ctx, "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("list alice pods failed: %v", err)
|
||||
}
|
||||
if len(alicePods) != 1 {
|
||||
t.Errorf("expected 1 pod for alice, got %d", len(alicePods))
|
||||
}
|
||||
|
||||
// Bob should see only his pod
|
||||
bobPods, err := client.ListPods(ctx, "bob")
|
||||
if err != nil {
|
||||
t.Fatalf("list bob pods failed: %v", err)
|
||||
}
|
||||
if len(bobPods) != 1 {
|
||||
t.Errorf("expected 1 pod for bob, got %d", len(bobPods))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetPod(t *testing.T) {
|
||||
t.Run("success", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
opts := defaultCreateOpts()
|
||||
|
||||
if _, err := client.CreatePod(ctx, opts); err != nil {
|
||||
t.Fatalf("setup: create pod failed: %v", err)
|
||||
}
|
||||
|
||||
pod, err := client.GetPod(ctx, "alice", "main")
|
||||
if err != nil {
|
||||
t.Fatalf("get pod failed: %v", err)
|
||||
}
|
||||
|
||||
if pod.User != "alice" {
|
||||
t.Errorf("expected user alice, got %s", pod.User)
|
||||
}
|
||||
if pod.Name != "main" {
|
||||
t.Errorf("expected name main, got %s", pod.Name)
|
||||
}
|
||||
if pod.Tools != "go@1.25,rust@1.94" {
|
||||
t.Errorf("expected tools go@1.25,rust@1.94, got %s", pod.Tools)
|
||||
}
|
||||
if pod.Task != "build a web server" {
|
||||
t.Errorf("expected task, got %s", pod.Task)
|
||||
}
|
||||
if pod.URL != "https://spinoff.dev/@alice/main/" {
|
||||
t.Errorf("expected URL, got %s", pod.URL)
|
||||
}
|
||||
if pod.CPUReq != "2" {
|
||||
t.Errorf("expected cpu req 2, got %s", pod.CPUReq)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("not_found", func(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
_ = client.EnsureNamespace(ctx, "alice")
|
||||
|
||||
_, err := client.GetPod(ctx, "alice", "nonexistent")
|
||||
if !errors.Is(err, ErrPodNotFound) {
|
||||
t.Errorf("expected ErrPodNotFound, got: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
248
internal/k8s/runners.go
Normal file
248
internal/k8s/runners.go
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
|
||||
)
|
||||
|
||||
// CreateRunnerPodOpts holds parameters for creating a runner pod.
|
||||
type CreateRunnerPodOpts struct {
|
||||
User string
|
||||
RunnerID string
|
||||
Tools string
|
||||
Task string
|
||||
RepoURL string
|
||||
Branch string
|
||||
CPUReq string
|
||||
MemReq string
|
||||
ForgejoRunnerToken string
|
||||
ForgejoURL string
|
||||
}
|
||||
|
||||
// CreateRunnerPod creates an ephemeral runner pod in the user's namespace.
|
||||
func (c *Client) CreateRunnerPod(ctx context.Context, opts CreateRunnerPodOpts) (string, error) {
|
||||
ns := model.NamespaceName(opts.User)
|
||||
podName := model.RunnerPodName(opts.RunnerID)
|
||||
|
||||
cpuReq := opts.CPUReq
|
||||
if cpuReq == "" {
|
||||
cpuReq = "2"
|
||||
}
|
||||
memReq := opts.MemReq
|
||||
if memReq == "" {
|
||||
memReq = "4Gi"
|
||||
}
|
||||
|
||||
forgejoURL := opts.ForgejoURL
|
||||
if forgejoURL == "" {
|
||||
forgejoURL = c.Config.ForgejoURL
|
||||
}
|
||||
|
||||
devPodAPIURL := c.Config.DevPodAPIURL
|
||||
if devPodAPIURL == "" {
|
||||
devPodAPIURL = "http://dev-pod-api.dev-infra.svc:8080"
|
||||
}
|
||||
|
||||
privileged := true
|
||||
pod := &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: podName,
|
||||
Namespace: ns,
|
||||
Labels: map[string]string{
|
||||
"app": "dev-pod-runner",
|
||||
"runner-id": opts.RunnerID,
|
||||
"user": opts.User,
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: corev1.RestartPolicyNever,
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "runner",
|
||||
Image: fmt.Sprintf("%s/%s", c.Config.Registry, c.Config.GoldenImage),
|
||||
Env: []corev1.EnvVar{
|
||||
{Name: "RUNNER_MODE", Value: "true"},
|
||||
{Name: "RUNNER_ID", Value: opts.RunnerID},
|
||||
{Name: "DEV_POD_API_URL", Value: devPodAPIURL},
|
||||
{Name: "FORGEJO_URL", Value: forgejoURL},
|
||||
{Name: "FORGEJO_RUNNER_TOKEN", Value: opts.ForgejoRunnerToken},
|
||||
{Name: "RUNNER_REPO", Value: opts.RepoURL},
|
||||
{Name: "RUNNER_BRANCH", Value: opts.Branch},
|
||||
{Name: "RUNNER_TASK", Value: opts.Task},
|
||||
{Name: "DEV_TOOLS", Value: opts.Tools},
|
||||
},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse(cpuReq),
|
||||
corev1.ResourceMemory: resource.MustParse(memReq),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse(cpuReq),
|
||||
corev1.ResourceMemory: resource.MustParse(memReq),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "ipip-sidecar",
|
||||
Image: fmt.Sprintf("%s/claw-ipip-tunnel:dev", c.Config.Registry),
|
||||
ImagePullPolicy: corev1.PullAlways,
|
||||
SecurityContext: &corev1.SecurityContext{
|
||||
Privileged: &privileged,
|
||||
Capabilities: &corev1.Capabilities{
|
||||
Add: []corev1.Capability{"NET_ADMIN"},
|
||||
},
|
||||
},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("10m"),
|
||||
corev1.ResourceMemory: resource.MustParse("8Mi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("32Mi"),
|
||||
},
|
||||
},
|
||||
Env: []corev1.EnvVar{
|
||||
{Name: "POD_ID", Value: podName},
|
||||
{Name: "VPN_GATEWAY_HOST", Value: fmt.Sprintf("vpn-gateway.%s.svc", c.Config.VPNGatewayNS)},
|
||||
{
|
||||
Name: "VPN_GATEWAY_KEY",
|
||||
ValueFrom: &corev1.EnvVarSource{
|
||||
SecretKeyRef: &corev1.SecretKeySelector{
|
||||
LocalObjectReference: corev1.LocalObjectReference{Name: "dev-secrets"},
|
||||
Key: "VPN_GATEWAY_KEY",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if _, err := c.Clientset.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil {
|
||||
return "", fmt.Errorf("create runner pod: %w", err)
|
||||
}
|
||||
|
||||
return podName, nil
|
||||
}
|
||||
|
||||
// DeleteRunnerPod deletes a runner pod and its scratch PVC from the user's namespace.
|
||||
func (c *Client) DeleteRunnerPod(ctx context.Context, user, podName string) error {
|
||||
ns := model.NamespaceName(user)
|
||||
|
||||
if err := c.Clientset.CoreV1().Pods(ns).Delete(ctx, podName, metav1.DeleteOptions{}); err != nil {
|
||||
slog.Warn("delete runner pod", "pod", podName, "ns", ns, "error", err)
|
||||
}
|
||||
|
||||
pvcName := "workspace-" + podName
|
||||
if err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Delete(ctx, pvcName, metav1.DeleteOptions{}); err != nil {
|
||||
slog.Warn("delete runner pvc", "pvc", pvcName, "ns", ns, "error", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunnerCleanupStore is the subset of store operations needed by the cleanup goroutine.
|
||||
type RunnerCleanupStore interface {
|
||||
ListRunners(ctx context.Context, userFilter string, statusFilter string) ([]model.Runner, error)
|
||||
UpdateRunnerStatus(ctx context.Context, id string, newStatus model.RunnerStatus, forgejoRunnerID string) error
|
||||
DeleteRunner(ctx context.Context, id string) error
|
||||
GetStaleRunners(ctx context.Context, ttl time.Duration) ([]model.Runner, error)
|
||||
}
|
||||
|
||||
// RunnerCleaner polls for completed/stale runners and cleans up their k8s resources.
|
||||
type RunnerCleaner struct {
|
||||
k8s *Client
|
||||
store RunnerCleanupStore
|
||||
logger *slog.Logger
|
||||
ttl time.Duration
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
// NewRunnerCleaner creates a runner cleanup goroutine.
|
||||
func NewRunnerCleaner(k8s *Client, st RunnerCleanupStore, logger *slog.Logger) *RunnerCleaner {
|
||||
return &RunnerCleaner{
|
||||
k8s: k8s,
|
||||
store: st,
|
||||
logger: logger,
|
||||
ttl: 2 * time.Hour,
|
||||
interval: 30 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// Start runs the cleanup loop until the context is cancelled.
|
||||
func (rc *RunnerCleaner) Start(ctx context.Context) {
|
||||
rc.logger.Info("runner cleaner started", "ttl", rc.ttl, "interval", rc.interval)
|
||||
ticker := time.NewTicker(rc.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
rc.logger.Info("runner cleaner stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
rc.cleanupCompleted(ctx)
|
||||
rc.cleanupStale(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunnerCleaner) cleanupCompleted(ctx context.Context) {
|
||||
for _, status := range []string{"completed", "failed"} {
|
||||
runners, err := rc.store.ListRunners(ctx, "", status)
|
||||
if err != nil {
|
||||
rc.logger.Error("list runners for cleanup", "status", status, "error", err)
|
||||
continue
|
||||
}
|
||||
for _, r := range runners {
|
||||
rc.logger.Info("cleaning up runner", "id", r.ID, "status", r.Status, "user", r.User)
|
||||
|
||||
if err := rc.store.UpdateRunnerStatus(ctx, r.ID, model.RunnerStatusCleanupPending, ""); err != nil {
|
||||
rc.logger.Error("mark runner cleanup_pending", "id", r.ID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if r.PodName != "" {
|
||||
if err := rc.k8s.DeleteRunnerPod(ctx, r.User, r.PodName); err != nil {
|
||||
rc.logger.Error("delete runner pod", "id", r.ID, "pod", r.PodName, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := rc.store.DeleteRunner(ctx, r.ID); err != nil {
|
||||
rc.logger.Error("delete runner record", "id", r.ID, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *RunnerCleaner) cleanupStale(ctx context.Context) {
|
||||
stale, err := rc.store.GetStaleRunners(ctx, rc.ttl)
|
||||
if err != nil {
|
||||
rc.logger.Error("get stale runners", "error", err)
|
||||
return
|
||||
}
|
||||
for _, r := range stale {
|
||||
rc.logger.Warn("force-destroying stale runner", "id", r.ID, "status", r.Status,
|
||||
"age", time.Since(r.CreatedAt).Round(time.Second))
|
||||
|
||||
if r.PodName != "" {
|
||||
if err := rc.k8s.DeleteRunnerPod(ctx, r.User, r.PodName); err != nil {
|
||||
rc.logger.Error("delete stale runner pod", "id", r.ID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := rc.store.DeleteRunner(ctx, r.ID); err != nil {
|
||||
rc.logger.Error("delete stale runner record", "id", r.ID, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
194
internal/k8s/runners_test.go
Normal file
194
internal/k8s/runners_test.go
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
|
||||
)
|
||||
|
||||
func defaultRunnerOpts() CreateRunnerPodOpts {
|
||||
return CreateRunnerPodOpts{
|
||||
User: "alice",
|
||||
RunnerID: "runner-abc12345",
|
||||
Tools: "go,rust",
|
||||
Task: "implement feature",
|
||||
RepoURL: "alice/myrepo",
|
||||
Branch: "main",
|
||||
CPUReq: "2",
|
||||
MemReq: "4Gi",
|
||||
ForgejoRunnerToken: "test-runner-token",
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateRunnerPod(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := client.EnsureNamespace(ctx, "alice"); err != nil {
|
||||
t.Fatalf("ensure namespace: %v", err)
|
||||
}
|
||||
|
||||
opts := defaultRunnerOpts()
|
||||
podName, err := client.CreateRunnerPod(ctx, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("create runner pod: %v", err)
|
||||
}
|
||||
|
||||
expectedPodName := model.RunnerPodName("runner-abc12345")
|
||||
if podName != expectedPodName {
|
||||
t.Errorf("expected pod name %q, got %q", expectedPodName, podName)
|
||||
}
|
||||
|
||||
ns := model.NamespaceName("alice")
|
||||
pod, err := client.Clientset.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("get runner pod: %v", err)
|
||||
}
|
||||
|
||||
if pod.Labels["app"] != "dev-pod-runner" {
|
||||
t.Errorf("expected app label dev-pod-runner, got %q", pod.Labels["app"])
|
||||
}
|
||||
if pod.Labels["runner-id"] != "runner-abc12345" {
|
||||
t.Errorf("expected runner-id label runner-abc12345, got %q", pod.Labels["runner-id"])
|
||||
}
|
||||
if pod.Labels["user"] != "alice" {
|
||||
t.Errorf("expected user label alice, got %q", pod.Labels["user"])
|
||||
}
|
||||
|
||||
if len(pod.Spec.Containers) != 2 {
|
||||
t.Fatalf("expected 2 containers (runner + ipip-sidecar), got %d", len(pod.Spec.Containers))
|
||||
}
|
||||
|
||||
container := pod.Spec.Containers[0]
|
||||
if container.Name != "runner" {
|
||||
t.Errorf("expected container name runner, got %q", container.Name)
|
||||
}
|
||||
|
||||
sidecar := pod.Spec.Containers[1]
|
||||
if sidecar.Name != "ipip-sidecar" {
|
||||
t.Errorf("expected sidecar name ipip-sidecar, got %q", sidecar.Name)
|
||||
}
|
||||
if sidecar.SecurityContext == nil || !*sidecar.SecurityContext.Privileged {
|
||||
t.Error("expected ipip-sidecar to be privileged")
|
||||
}
|
||||
|
||||
envMap := make(map[string]string)
|
||||
for _, e := range container.Env {
|
||||
envMap[e.Name] = e.Value
|
||||
}
|
||||
if envMap["RUNNER_MODE"] != "true" {
|
||||
t.Error("expected RUNNER_MODE=true")
|
||||
}
|
||||
if envMap["RUNNER_ID"] != "runner-abc12345" {
|
||||
t.Errorf("expected RUNNER_ID=runner-abc12345, got %q", envMap["RUNNER_ID"])
|
||||
}
|
||||
if envMap["DEV_POD_API_URL"] == "" {
|
||||
t.Error("expected DEV_POD_API_URL to be set")
|
||||
}
|
||||
if envMap["FORGEJO_RUNNER_TOKEN"] != "test-runner-token" {
|
||||
t.Errorf("expected FORGEJO_RUNNER_TOKEN=test-runner-token, got %q", envMap["FORGEJO_RUNNER_TOKEN"])
|
||||
}
|
||||
if envMap["RUNNER_REPO"] != "alice/myrepo" {
|
||||
t.Errorf("expected RUNNER_REPO=alice/myrepo, got %q", envMap["RUNNER_REPO"])
|
||||
}
|
||||
if envMap["RUNNER_BRANCH"] != "main" {
|
||||
t.Errorf("expected RUNNER_BRANCH=main, got %q", envMap["RUNNER_BRANCH"])
|
||||
}
|
||||
if envMap["DEV_TOOLS"] != "go,rust" {
|
||||
t.Errorf("expected DEV_TOOLS=go,rust, got %q", envMap["DEV_TOOLS"])
|
||||
}
|
||||
|
||||
cpuReq := container.Resources.Requests.Cpu()
|
||||
if cpuReq.String() != "2" {
|
||||
t.Errorf("expected cpu request 2, got %s", cpuReq.String())
|
||||
}
|
||||
memReq := container.Resources.Requests.Memory()
|
||||
if memReq.String() != "4Gi" {
|
||||
t.Errorf("expected memory request 4Gi, got %s", memReq.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateRunnerPod_DefaultResources(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := client.EnsureNamespace(ctx, "alice"); err != nil {
|
||||
t.Fatalf("ensure namespace: %v", err)
|
||||
}
|
||||
|
||||
opts := CreateRunnerPodOpts{
|
||||
User: "alice",
|
||||
RunnerID: "runner-defaults",
|
||||
RepoURL: "alice/repo",
|
||||
}
|
||||
podName, err := client.CreateRunnerPod(ctx, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("create runner pod: %v", err)
|
||||
}
|
||||
|
||||
ns := model.NamespaceName("alice")
|
||||
pod, err := client.Clientset.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("get runner pod: %v", err)
|
||||
}
|
||||
|
||||
container := pod.Spec.Containers[0]
|
||||
cpuReq := container.Resources.Requests.Cpu()
|
||||
if cpuReq.String() != "2" {
|
||||
t.Errorf("expected default cpu request 2, got %s", cpuReq.String())
|
||||
}
|
||||
memReq := container.Resources.Requests.Memory()
|
||||
if memReq.String() != "4Gi" {
|
||||
t.Errorf("expected default memory request 4Gi, got %s", memReq.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRunnerPod(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
if err := client.EnsureNamespace(ctx, "alice"); err != nil {
|
||||
t.Fatalf("ensure namespace: %v", err)
|
||||
}
|
||||
|
||||
opts := defaultRunnerOpts()
|
||||
podName, err := client.CreateRunnerPod(ctx, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("create runner pod: %v", err)
|
||||
}
|
||||
|
||||
if err := client.DeleteRunnerPod(ctx, "alice", podName); err != nil {
|
||||
t.Fatalf("delete runner pod: %v", err)
|
||||
}
|
||||
|
||||
ns := model.NamespaceName("alice")
|
||||
pods, err := client.Clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{
|
||||
LabelSelector: "app=dev-pod-runner",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list pods: %v", err)
|
||||
}
|
||||
if len(pods.Items) != 0 {
|
||||
t.Errorf("expected 0 runner pods after delete, got %d", len(pods.Items))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRunnerPod_NonExistent(t *testing.T) {
|
||||
client := newTestClient()
|
||||
ctx := context.Background()
|
||||
|
||||
err := client.DeleteRunnerPod(ctx, "alice", "nonexistent-pod")
|
||||
if err != nil {
|
||||
t.Fatalf("delete nonexistent pod should not fail, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPodName(t *testing.T) {
|
||||
name := model.RunnerPodName("runner-abc123")
|
||||
if name != "dev-pod-runner-abc123" {
|
||||
t.Errorf("expected dev-pod-runner-abc123, got %s", name)
|
||||
}
|
||||
}
|
||||
433
internal/k8s/sessionhost.go
Normal file
433
internal/k8s/sessionhost.go
Normal file
|
|
@ -0,0 +1,433 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
netv1 "k8s.io/api/networking/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
|
||||
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
|
||||
)
|
||||
|
||||
// SessionHostSpec captures the defaults + per-install knobs. The values
|
||||
// are chosen to match web-tui's k8s/session-host manifests.
|
||||
type SessionHostSpec struct {
|
||||
// Image is the dev-container image (golden image). Defaults to the
|
||||
// value baked into the ConfigFromEnv loader.
|
||||
Image string
|
||||
// PortWatchImage is the sidecar image built from web-tui/cmd/port-watch.
|
||||
PortWatchImage string
|
||||
// ApexDomain is the root domain used to form per-user hostnames
|
||||
// (e.g. "spinoff.dev" → "*.alice.spinoff.dev").
|
||||
ApexDomain string
|
||||
// ClusterIssuer is the cert-manager ClusterIssuer used to obtain the
|
||||
// per-user wildcard cert via DNS-01 (e.g. "letsencrypt-cloudflare-dns01").
|
||||
ClusterIssuer string
|
||||
// GatewayService is the in-cluster Service name that the per-user
|
||||
// ingress forwards <port>.<user>.<apex>/* traffic to.
|
||||
GatewayService string
|
||||
// GatewayServiceNamespace is where GatewayService lives (e.g. "web-tui").
|
||||
GatewayServiceNamespace string
|
||||
}
|
||||
|
||||
// EnsureSessionHost is idempotent: guarantees the user has a
|
||||
// long-lived session-host pod + supporting PVCs, Service, Ingress, and
|
||||
// per-user wildcard cert. Re-running on an already-provisioned user is
|
||||
// a no-op that reports current status.
|
||||
func (c *Client) EnsureSessionHost(ctx context.Context, user string, spec SessionHostSpec) (SessionHostStatus, error) {
|
||||
ns := model.NamespaceName(user)
|
||||
|
||||
if err := c.EnsureNamespace(ctx, user); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
if err := c.ensureSessionHostPVCs(ctx, ns); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
if err := c.ensureSessionHostServiceAccount(ctx, ns); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
pod, err := c.ensureSessionHostPod(ctx, ns, user, spec)
|
||||
if err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
if err := c.ensureSessionHostService(ctx, ns); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
if err := c.ensureSessionHostCertificate(ctx, ns, user, spec); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
if err := c.ensureSessionHostIngress(ctx, ns, user, spec); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
return c.readSessionHostStatus(ctx, ns, user, pod)
|
||||
}
|
||||
|
||||
// SessionHostStatusForUser returns the current status of the user's
|
||||
// session-host pod without creating anything new.
|
||||
func (c *Client) SessionHostStatusForUser(ctx context.Context, user string) (SessionHostStatus, error) {
|
||||
ns := model.NamespaceName(user)
|
||||
pod, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, "session-host", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return SessionHostStatus{Ready: false}, nil
|
||||
}
|
||||
return SessionHostStatus{}, fmt.Errorf("get session-host pod: %w", err)
|
||||
}
|
||||
return c.readSessionHostStatus(ctx, ns, user, pod)
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
const (
|
||||
sessionHostPodName = "session-host"
|
||||
sessionHostServiceName = "session-host"
|
||||
sessionHostSA = "session-host"
|
||||
workspacePVC = "workspace"
|
||||
dotfilesPVC = "dotfiles"
|
||||
)
|
||||
|
||||
func (c *Client) ensureSessionHostPVCs(ctx context.Context, ns string) error {
|
||||
pvcs := []struct {
|
||||
name string
|
||||
size string
|
||||
}{
|
||||
{workspacePVC, "20Gi"},
|
||||
{dotfilesPVC, "1Gi"},
|
||||
}
|
||||
for _, p := range pvcs {
|
||||
want := &corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: p.name},
|
||||
Spec: corev1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
|
||||
Resources: corev1.VolumeResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceStorage: resource.MustParse(p.size),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Create(ctx, want, metav1.CreateOptions{})
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create pvc %s/%s: %w", ns, p.name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) ensureSessionHostServiceAccount(ctx context.Context, ns string) error {
|
||||
sa := &corev1.ServiceAccount{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: sessionHostSA},
|
||||
}
|
||||
_, err := c.Clientset.CoreV1().ServiceAccounts(ns).Create(ctx, sa, metav1.CreateOptions{})
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create sa %s/%s: %w", ns, sessionHostSA, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) ensureSessionHostPod(ctx context.Context, ns, user string, spec SessionHostSpec) (*corev1.Pod, error) {
|
||||
existing, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, sessionHostPodName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
return existing, nil
|
||||
}
|
||||
if !errors.IsNotFound(err) {
|
||||
return nil, fmt.Errorf("get pod %s/%s: %w", ns, sessionHostPodName, err)
|
||||
}
|
||||
pod := sessionHostPodManifest(user, spec)
|
||||
pod, err = c.Clientset.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if errors.IsAlreadyExists(err) {
|
||||
return c.Clientset.CoreV1().Pods(ns).Get(ctx, sessionHostPodName, metav1.GetOptions{})
|
||||
}
|
||||
return nil, fmt.Errorf("create pod %s/%s: %w", ns, sessionHostPodName, err)
|
||||
}
|
||||
return pod, nil
|
||||
}
|
||||
|
||||
func sessionHostPodManifest(user string, spec SessionHostSpec) *corev1.Pod {
|
||||
one := int64(1000)
|
||||
return &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: sessionHostPodName,
|
||||
Labels: map[string]string{
|
||||
"app.kubernetes.io/name": "session-host",
|
||||
"app.kubernetes.io/component": "dev-pod",
|
||||
"web-tui.spinoff.dev/user": user,
|
||||
"managed-by": "dev-pod-api",
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: corev1.RestartPolicyAlways,
|
||||
ServiceAccountName: sessionHostSA,
|
||||
SecurityContext: &corev1.PodSecurityContext{FSGroup: &one},
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "dev",
|
||||
Image: spec.Image,
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
Command: []string{"/usr/local/bin/dev-entrypoint"},
|
||||
TTY: true,
|
||||
Stdin: true,
|
||||
Env: []corev1.EnvVar{
|
||||
{Name: "HOME", Value: "/home/dev"},
|
||||
{Name: "USER", Value: "dev"},
|
||||
{Name: "WEBTUI_USER", Value: user},
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{Name: "workspace", MountPath: "/home/dev/workspace"},
|
||||
{Name: "dotfiles", MountPath: "/home/dev/.config"},
|
||||
{Name: "tmp", MountPath: "/tmp"},
|
||||
},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("250m"),
|
||||
corev1.ResourceMemory: resource.MustParse("256Mi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "port-watch",
|
||||
Image: spec.PortWatchImage,
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
Args: []string{"-listen=:9100", "-uploads=/home/dev/uploads"},
|
||||
Ports: []corev1.ContainerPort{
|
||||
{ContainerPort: 9100, Name: "portwatch"},
|
||||
},
|
||||
ReadinessProbe: &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromInt32(9100)},
|
||||
},
|
||||
PeriodSeconds: 5,
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{Name: "workspace", MountPath: "/home/dev/workspace", ReadOnly: true},
|
||||
},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("20m"),
|
||||
corev1.ResourceMemory: resource.MustParse("32Mi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("100m"),
|
||||
corev1.ResourceMemory: resource.MustParse("64Mi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Volumes: []corev1.Volume{
|
||||
{Name: "workspace", VolumeSource: corev1.VolumeSource{
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: workspacePVC}}},
|
||||
{Name: "dotfiles", VolumeSource: corev1.VolumeSource{
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: dotfilesPVC}}},
|
||||
{Name: "tmp", VolumeSource: corev1.VolumeSource{
|
||||
EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: ptrQuantity("1Gi")}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) ensureSessionHostService(ctx context.Context, ns string) error {
|
||||
svc := &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: sessionHostServiceName},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Selector: map[string]string{"app.kubernetes.io/name": "session-host"},
|
||||
Ports: []corev1.ServicePort{
|
||||
{Name: "portwatch", Port: 9100, TargetPort: intstr.FromInt32(9100)},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := c.Clientset.CoreV1().Services(ns).Create(ctx, svc, metav1.CreateOptions{})
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create svc: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) ensureSessionHostCertificate(ctx context.Context, ns, user string, spec SessionHostSpec) error {
|
||||
name := "wildcard-" + user
|
||||
gvr := schema.GroupVersionResource{Group: "cert-manager.io", Version: "v1", Resource: "certificates"}
|
||||
wildcardHost := "*." + user + "." + spec.ApexDomain
|
||||
obj := &unstructured.Unstructured{Object: map[string]any{
|
||||
"apiVersion": "cert-manager.io/v1",
|
||||
"kind": "Certificate",
|
||||
"metadata": map[string]any{
|
||||
"name": name,
|
||||
},
|
||||
"spec": map[string]any{
|
||||
"secretName": name + "-tls",
|
||||
"dnsNames": []string{wildcardHost},
|
||||
"duration": "2160h",
|
||||
"renewBefore": "720h",
|
||||
"privateKey": map[string]any{"algorithm": "ECDSA", "size": int64(256)},
|
||||
"issuerRef": map[string]any{
|
||||
"name": spec.ClusterIssuer,
|
||||
"kind": "ClusterIssuer",
|
||||
"group": "cert-manager.io",
|
||||
},
|
||||
},
|
||||
}}
|
||||
_, err := c.Dynamic.Resource(gvr).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{})
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create certificate: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) ensureSessionHostIngress(ctx context.Context, ns, user string, spec SessionHostSpec) error {
|
||||
host := "*." + user + "." + spec.ApexDomain
|
||||
pathType := netv1.PathTypePrefix
|
||||
className := "traefik"
|
||||
ing := &netv1.Ingress{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "preview-" + user,
|
||||
Annotations: map[string]string{
|
||||
"cert-manager.io/cluster-issuer": spec.ClusterIssuer,
|
||||
"traefik.ingress.kubernetes.io/router.tls": "true",
|
||||
},
|
||||
},
|
||||
Spec: netv1.IngressSpec{
|
||||
IngressClassName: &className,
|
||||
TLS: []netv1.IngressTLS{
|
||||
{Hosts: []string{host}, SecretName: "wildcard-" + user + "-tls"},
|
||||
},
|
||||
Rules: []netv1.IngressRule{
|
||||
{
|
||||
Host: host,
|
||||
IngressRuleValue: netv1.IngressRuleValue{
|
||||
HTTP: &netv1.HTTPIngressRuleValue{
|
||||
Paths: []netv1.HTTPIngressPath{
|
||||
{
|
||||
Path: "/",
|
||||
PathType: &pathType,
|
||||
Backend: netv1.IngressBackend{
|
||||
Service: &netv1.IngressServiceBackend{
|
||||
Name: spec.GatewayService,
|
||||
Port: netv1.ServiceBackendPort{Name: "http"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := c.Clientset.NetworkingV1().Ingresses(ns).Create(ctx, ing, metav1.CreateOptions{})
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create ingress: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) readSessionHostStatus(ctx context.Context, ns, user string, pod *corev1.Pod) (SessionHostStatus, error) {
|
||||
ready := pod.Status.Phase == corev1.PodRunning && allContainersReady(pod)
|
||||
certReady, certReason := c.readCertStatus(ctx, ns, "wildcard-"+user)
|
||||
return SessionHostStatus{
|
||||
Ready: ready,
|
||||
PodName: pod.Name,
|
||||
Namespace: ns,
|
||||
CertReady: certReady,
|
||||
CertPendingReason: certReason,
|
||||
CreatedAt: pod.CreationTimestamp.Time,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) readCertStatus(ctx context.Context, ns, name string) (bool, string) {
|
||||
gvr := schema.GroupVersionResource{Group: "cert-manager.io", Version: "v1", Resource: "certificates"}
|
||||
obj, err := c.Dynamic.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return false, "not-found"
|
||||
}
|
||||
conds, ok, _ := unstructuredNested(obj.Object, "status", "conditions")
|
||||
if !ok {
|
||||
return false, "no-status"
|
||||
}
|
||||
list, ok := conds.([]any)
|
||||
if !ok {
|
||||
return false, "bad-status"
|
||||
}
|
||||
for _, c := range list {
|
||||
m, ok := c.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if m["type"] == "Ready" {
|
||||
if m["status"] == "True" {
|
||||
return true, ""
|
||||
}
|
||||
if reason, ok := m["reason"].(string); ok {
|
||||
return false, reason
|
||||
}
|
||||
return false, "not-ready"
|
||||
}
|
||||
}
|
||||
return false, "pending"
|
||||
}
|
||||
|
||||
// SessionHostStatus is duplicated here so the k8s package doesn't need
|
||||
// to import the api package (avoids an import cycle). The api package
|
||||
// mirrors the fields.
|
||||
type SessionHostStatus struct {
|
||||
Ready bool
|
||||
PodName string
|
||||
Namespace string
|
||||
CertReady bool
|
||||
CertPendingReason string
|
||||
AtchSessionCount int
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// ---- tiny helpers ----
|
||||
|
||||
func allContainersReady(pod *corev1.Pod) bool {
|
||||
if len(pod.Status.ContainerStatuses) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, cs := range pod.Status.ContainerStatuses {
|
||||
if !cs.Ready {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func unstructuredNested(obj map[string]any, keys ...string) (any, bool, error) {
|
||||
cur := any(obj)
|
||||
for _, k := range keys {
|
||||
m, ok := cur.(map[string]any)
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
v, ok := m[k]
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur = v
|
||||
}
|
||||
return cur, true, nil
|
||||
}
|
||||
|
||||
func ptrQuantity(s string) *resource.Quantity {
|
||||
q := resource.MustParse(s)
|
||||
return &q
|
||||
}
|
||||
599
internal/k8s/templates.go
Normal file
599
internal/k8s/templates.go
Normal file
|
|
@ -0,0 +1,599 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
|
||||
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
|
||||
)
|
||||
|
||||
// PodOpts holds all parameters needed to create a dev pod and its resources.
|
||||
type PodOpts struct {
|
||||
User string
|
||||
Pod string
|
||||
Tools string
|
||||
Task string
|
||||
CPUReq string
|
||||
CPULimit string
|
||||
MemReq string
|
||||
MemLimit string
|
||||
VPNKey string
|
||||
AnthropicKey string
|
||||
OpenAIKey string
|
||||
ForgejoToken string
|
||||
TailscaleKey string
|
||||
}
|
||||
|
||||
// PVCName returns the per-pod PVC name for a given pod.
|
||||
func PVCName(pod string) string {
|
||||
return fmt.Sprintf("workspace-%s", pod)
|
||||
}
|
||||
|
||||
// PVCTemplate returns a PersistentVolumeClaim for a specific pod's workspace.
|
||||
func PVCTemplate(user, pod string) *corev1.PersistentVolumeClaim {
|
||||
storageClass := "longhorn"
|
||||
return &corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: PVCName(pod),
|
||||
Namespace: model.NamespaceName(user),
|
||||
Labels: map[string]string{
|
||||
"app": "dev-pod",
|
||||
"podname": pod,
|
||||
},
|
||||
},
|
||||
Spec: corev1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
|
||||
StorageClassName: &storageClass,
|
||||
Resources: corev1.VolumeResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceStorage: resource.MustParse("20Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// PodTemplate returns a Pod spec matching the YAML pod-template.yaml.
|
||||
func PodTemplate(cfg Config, opts PodOpts) *corev1.Pod {
|
||||
ns := model.NamespaceName(opts.User)
|
||||
podName := model.PodFullName(opts.Pod)
|
||||
return &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: podName,
|
||||
Namespace: ns,
|
||||
Labels: map[string]string{
|
||||
"app": "dev-pod",
|
||||
"podname": opts.Pod,
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
HostAliases: []corev1.HostAlias{
|
||||
{
|
||||
IP: "127.0.0.1",
|
||||
Hostnames: []string{"anthropic.internal", "openai.internal"},
|
||||
},
|
||||
},
|
||||
Containers: []corev1.Container{
|
||||
devContainer(cfg, opts),
|
||||
aiProxyContainer(),
|
||||
ipipSidecarContainer(cfg, opts),
|
||||
},
|
||||
Volumes: []corev1.Volume{
|
||||
{
|
||||
Name: "workspace",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: PVCName(opts.Pod),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "ai-proxy-config",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
ConfigMap: &corev1.ConfigMapVolumeSource{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: "ai-proxy-config",
|
||||
},
|
||||
DefaultMode: int32Ptr(0755),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "ai-proxy-secrets",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: "ai-proxy-secrets",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func devContainer(cfg Config, opts PodOpts) corev1.Container {
|
||||
return corev1.Container{
|
||||
Name: "dev",
|
||||
Image: fmt.Sprintf("%s/%s", cfg.Registry, cfg.GoldenImage),
|
||||
ImagePullPolicy: corev1.PullAlways,
|
||||
Ports: []corev1.ContainerPort{
|
||||
{ContainerPort: 7681, Name: "ttyd"},
|
||||
{ContainerPort: 22, Name: "ssh"},
|
||||
{ContainerPort: 3000, Name: "forgejo"},
|
||||
{ContainerPort: 8080, Name: "vscode"},
|
||||
{ContainerPort: 8090, Name: "ralphex-gerrit"},
|
||||
},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse(opts.CPUReq),
|
||||
corev1.ResourceMemory: resource.MustParse(opts.MemReq),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse(opts.CPULimit),
|
||||
corev1.ResourceMemory: resource.MustParse(opts.MemLimit),
|
||||
},
|
||||
},
|
||||
EnvFrom: []corev1.EnvFromSource{
|
||||
{
|
||||
SecretRef: &corev1.SecretEnvSource{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: "dev-secrets",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Env: []corev1.EnvVar{
|
||||
{Name: "TASK_DESCRIPTION", Value: opts.Task},
|
||||
{Name: "DEV_TOOLS", Value: opts.Tools},
|
||||
{Name: "DEV_BASE_PATH", Value: fmt.Sprintf("/@%s/%s", opts.User, opts.Pod)},
|
||||
{Name: "DEV_EXTERNAL_HOST", Value: cfg.Domain},
|
||||
{Name: "TTYD_BASE_PATH", Value: fmt.Sprintf("/@%s/%s", opts.User, opts.Pod)},
|
||||
{Name: "FORGEJO_URL", Value: cfg.ForgejoURL},
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{Name: "workspace", MountPath: "/home/dev/workspace"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func aiProxyContainer() corev1.Container {
|
||||
return corev1.Container{
|
||||
Name: "ai-proxy",
|
||||
Image: "nginx:alpine",
|
||||
Command: []string{"/bin/sh", "/etc/nginx/templates/entrypoint.sh"},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("10m"),
|
||||
corev1.ResourceMemory: resource.MustParse("16Mi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("32Mi"),
|
||||
},
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{Name: "ai-proxy-config", MountPath: "/etc/nginx/templates", ReadOnly: true},
|
||||
{Name: "ai-proxy-secrets", MountPath: "/secrets", ReadOnly: true},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func ipipSidecarContainer(cfg Config, opts PodOpts) corev1.Container {
|
||||
return corev1.Container{
|
||||
Name: "ipip-sidecar",
|
||||
Image: fmt.Sprintf("%s/claw-ipip-tunnel:dev", cfg.Registry),
|
||||
ImagePullPolicy: corev1.PullAlways,
|
||||
SecurityContext: &corev1.SecurityContext{
|
||||
Privileged: boolPtr(true),
|
||||
Capabilities: &corev1.Capabilities{
|
||||
Add: []corev1.Capability{"NET_ADMIN"},
|
||||
},
|
||||
},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("10m"),
|
||||
corev1.ResourceMemory: resource.MustParse("8Mi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("32Mi"),
|
||||
},
|
||||
},
|
||||
Env: []corev1.EnvVar{
|
||||
{Name: "POD_ID", Value: fmt.Sprintf("dev-%s-%s", opts.User, opts.Pod)},
|
||||
{Name: "VPN_GATEWAY_HOST", Value: fmt.Sprintf("vpn-gateway.%s.svc", cfg.VPNGatewayNS)},
|
||||
{
|
||||
Name: "VPN_GATEWAY_KEY",
|
||||
ValueFrom: &corev1.EnvVarSource{
|
||||
SecretKeyRef: &corev1.SecretKeySelector{
|
||||
LocalObjectReference: corev1.LocalObjectReference{Name: "dev-secrets"},
|
||||
Key: "VPN_GATEWAY_KEY",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ServiceTemplate returns a Service for the dev pod matching service.yaml.
|
||||
func ServiceTemplate(user, pod string) *corev1.Service {
|
||||
ns := model.NamespaceName(user)
|
||||
return &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: model.ServiceName(pod),
|
||||
Namespace: ns,
|
||||
Labels: map[string]string{
|
||||
"app": "dev-pod",
|
||||
"podname": pod,
|
||||
},
|
||||
},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Type: corev1.ServiceTypeClusterIP,
|
||||
Ports: []corev1.ServicePort{
|
||||
{Name: "ttyd", Port: 7681, TargetPort: intstr.FromInt32(7681), Protocol: corev1.ProtocolTCP},
|
||||
{Name: "ssh", Port: 22, TargetPort: intstr.FromInt32(22), Protocol: corev1.ProtocolTCP},
|
||||
{Name: "forgejo", Port: 3000, TargetPort: intstr.FromInt32(3000), Protocol: corev1.ProtocolTCP},
|
||||
{Name: "vscode", Port: 8080, TargetPort: intstr.FromInt32(8080), Protocol: corev1.ProtocolTCP},
|
||||
{Name: "ralphex-gerrit", Port: 8090, TargetPort: intstr.FromInt32(8090), Protocol: corev1.ProtocolTCP},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
"app": "dev-pod",
|
||||
"podname": pod,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// IngressTemplate returns Traefik IngressRoute as an unstructured object matching ingress.yaml.
|
||||
// Returns the IngressRoute and two Middleware objects.
|
||||
func IngressTemplate(user, pod, domain string) []*unstructured.Unstructured {
|
||||
ns := model.NamespaceName(user)
|
||||
svcName := model.ServiceName(pod)
|
||||
|
||||
ingressRoute := &unstructured.Unstructured{
|
||||
Object: map[string]interface{}{
|
||||
"apiVersion": "traefik.io/v1alpha1",
|
||||
"kind": "IngressRoute",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": fmt.Sprintf("dev-pod-%s-ingress", pod),
|
||||
"namespace": ns,
|
||||
},
|
||||
"spec": map[string]interface{}{
|
||||
"entryPoints": []interface{}{"web", "websecure"},
|
||||
// No TLS section: Caddy terminates TLS and forwards HTTP to Traefik.
|
||||
// With TLS here, Traefik v3 only routes on websecure, breaking Caddy -> port 80 flow.
|
||||
// No explicit priority: Traefik v3 auto-calculates from rule length,
|
||||
// ensuring these beat the OpenClaw catch-all (Host:* / PathPrefix:/).
|
||||
"routes": []interface{}{
|
||||
// ttyd — base route
|
||||
map[string]interface{}{
|
||||
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/`)", domain, user, pod),
|
||||
"kind": "Rule",
|
||||
"middlewares": []interface{}{
|
||||
map[string]interface{}{"name": "spinoff-basic-auth"},
|
||||
},
|
||||
"services": []interface{}{
|
||||
map[string]interface{}{"name": svcName, "port": int64(7681)},
|
||||
},
|
||||
},
|
||||
// Forgejo
|
||||
map[string]interface{}{
|
||||
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/forgejo/`)", domain, user, pod),
|
||||
"kind": "Rule",
|
||||
"middlewares": []interface{}{
|
||||
map[string]interface{}{"name": "spinoff-basic-auth"},
|
||||
},
|
||||
"services": []interface{}{
|
||||
map[string]interface{}{"name": svcName, "port": int64(3000)},
|
||||
},
|
||||
},
|
||||
// VS Code
|
||||
map[string]interface{}{
|
||||
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/vscode/`)", domain, user, pod),
|
||||
"kind": "Rule",
|
||||
"middlewares": []interface{}{
|
||||
map[string]interface{}{"name": "spinoff-basic-auth"},
|
||||
},
|
||||
"services": []interface{}{
|
||||
map[string]interface{}{"name": svcName, "port": int64(8080)},
|
||||
},
|
||||
},
|
||||
// Gerrit code review (handles own base path, shares port 8090 with ralphex)
|
||||
map[string]interface{}{
|
||||
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/gerrit/`)", domain, user, pod),
|
||||
"kind": "Rule",
|
||||
"middlewares": []interface{}{
|
||||
map[string]interface{}{"name": "spinoff-basic-auth"},
|
||||
},
|
||||
"services": []interface{}{
|
||||
map[string]interface{}{"name": svcName, "port": int64(8090)},
|
||||
},
|
||||
},
|
||||
// Ralphex dashboard (with prefix stripping)
|
||||
map[string]interface{}{
|
||||
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/ralphex/`)", domain, user, pod),
|
||||
"kind": "Rule",
|
||||
"middlewares": []interface{}{
|
||||
map[string]interface{}{"name": "spinoff-basic-auth"},
|
||||
map[string]interface{}{"name": fmt.Sprintf("strip-dev-%s-ralphex-prefix", pod)},
|
||||
},
|
||||
"services": []interface{}{
|
||||
map[string]interface{}{"name": svcName, "port": int64(8090)},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
basicAuthMiddleware := &unstructured.Unstructured{
|
||||
Object: map[string]interface{}{
|
||||
"apiVersion": "traefik.io/v1alpha1",
|
||||
"kind": "Middleware",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": "spinoff-basic-auth",
|
||||
"namespace": ns,
|
||||
},
|
||||
"spec": map[string]interface{}{
|
||||
"basicAuth": map[string]interface{}{
|
||||
"secret": "spinoff-basic-auth",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
stripPrefixMiddleware := &unstructured.Unstructured{
|
||||
Object: map[string]interface{}{
|
||||
"apiVersion": "traefik.io/v1alpha1",
|
||||
"kind": "Middleware",
|
||||
"metadata": map[string]interface{}{
|
||||
"name": fmt.Sprintf("strip-dev-%s-ralphex-prefix", pod),
|
||||
"namespace": ns,
|
||||
},
|
||||
"spec": map[string]interface{}{
|
||||
"stripPrefix": map[string]interface{}{
|
||||
"prefixes": []interface{}{
|
||||
fmt.Sprintf("/@%s/%s/ralphex", user, pod),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return []*unstructured.Unstructured{ingressRoute, basicAuthMiddleware, stripPrefixMiddleware}
|
||||
}
|
||||
|
||||
// SecretsTemplate returns dev-secrets and ai-proxy-secrets matching secrets-template.yaml.
|
||||
func SecretsTemplate(user, vpnKey, anthropicKey, openaiKey, forgejoToken, tailscaleKey string) (*corev1.Secret, *corev1.Secret) {
|
||||
ns := model.NamespaceName(user)
|
||||
|
||||
stringData := map[string]string{
|
||||
"ANTHROPIC_API_KEY": "sk-devpod",
|
||||
"ANTHROPIC_BASE_URL": "http://anthropic.internal",
|
||||
"BASEROUTE_OPENAI_KEY": "sk-devpod",
|
||||
"VPN_GATEWAY_KEY": vpnKey,
|
||||
"FORGEJO_TOKEN": forgejoToken,
|
||||
}
|
||||
if tailscaleKey != "" {
|
||||
stringData["TAILSCALE_AUTHKEY"] = tailscaleKey
|
||||
}
|
||||
|
||||
devSecrets := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dev-secrets",
|
||||
Namespace: ns,
|
||||
},
|
||||
Type: corev1.SecretTypeOpaque,
|
||||
StringData: stringData,
|
||||
}
|
||||
|
||||
aiProxySecrets := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "ai-proxy-secrets",
|
||||
Namespace: ns,
|
||||
},
|
||||
Type: corev1.SecretTypeOpaque,
|
||||
StringData: map[string]string{
|
||||
"anthropic-key": anthropicKey,
|
||||
"openai-key": openaiKey,
|
||||
},
|
||||
}
|
||||
|
||||
return devSecrets, aiProxySecrets
|
||||
}
|
||||
|
||||
// BasicAuthSecretTemplate returns the spinoff-basic-auth Secret containing htpasswd data.
|
||||
// Matches the secret created by dev-pod-create.sh for Traefik basicAuth middleware.
|
||||
func BasicAuthSecretTemplate(user string) *corev1.Secret {
|
||||
ns := model.NamespaceName(user)
|
||||
|
||||
return &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "spinoff-basic-auth",
|
||||
Namespace: ns,
|
||||
},
|
||||
Type: corev1.SecretTypeOpaque,
|
||||
StringData: map[string]string{
|
||||
"users": "admin:$2y$05$i2Wxz4GpO8.jnMXWk1len.xcP0.wPSL.ozfd/OgX8BCo9chO8F2WO\n",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// NetworkPolicyTemplate returns a NetworkPolicy matching network-policy.yaml.
|
||||
func NetworkPolicyTemplate(user string) *networkingv1.NetworkPolicy {
|
||||
ns := model.NamespaceName(user)
|
||||
dnsPort := intstr.FromInt32(53)
|
||||
|
||||
return &networkingv1.NetworkPolicy{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "dev-pod-network-policy",
|
||||
Namespace: ns,
|
||||
},
|
||||
Spec: networkingv1.NetworkPolicySpec{
|
||||
PodSelector: metav1.LabelSelector{},
|
||||
PolicyTypes: []networkingv1.PolicyType{
|
||||
networkingv1.PolicyTypeIngress,
|
||||
networkingv1.PolicyTypeEgress,
|
||||
},
|
||||
Ingress: []networkingv1.NetworkPolicyIngressRule{
|
||||
{
|
||||
From: []networkingv1.NetworkPolicyPeer{
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"kubernetes.io/metadata.name": "kube-system",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Egress: []networkingv1.NetworkPolicyEgressRule{
|
||||
// VPN gateway in claw-system
|
||||
{
|
||||
To: []networkingv1.NetworkPolicyPeer{
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"kubernetes.io/metadata.name": "claw-system",
|
||||
},
|
||||
},
|
||||
PodSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"app": "vpn-gateway",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
// dev-infra namespace (cache services)
|
||||
{
|
||||
To: []networkingv1.NetworkPolicyPeer{
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"kubernetes.io/metadata.name": "dev-infra",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
// DNS resolution via kube-system
|
||||
{
|
||||
To: []networkingv1.NetworkPolicyPeer{
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"kubernetes.io/metadata.name": "kube-system",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Ports: []networkingv1.NetworkPolicyPort{
|
||||
{Protocol: protocolPtr(corev1.ProtocolUDP), Port: &dnsPort},
|
||||
{Protocol: protocolPtr(corev1.ProtocolTCP), Port: &dnsPort},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// AIProxyConfigMapTemplate returns the ai-proxy-config ConfigMap matching ai-proxy-config.yaml.
|
||||
func AIProxyConfigMapTemplate(user string) *corev1.ConfigMap {
|
||||
ns := model.NamespaceName(user)
|
||||
|
||||
nginxConfTemplate := `events {
|
||||
worker_connections 64;
|
||||
}
|
||||
|
||||
http {
|
||||
resolver %%RESOLVER%% valid=30s;
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name anthropic.internal;
|
||||
|
||||
client_max_body_size 50m;
|
||||
|
||||
location / {
|
||||
set $upstream https://anthropic.baseroute.tech;
|
||||
proxy_pass $upstream;
|
||||
proxy_ssl_server_name on;
|
||||
proxy_set_header Host anthropic.baseroute.tech;
|
||||
proxy_set_header x-api-key "%%ANTHROPIC_KEY%%";
|
||||
proxy_set_header Authorization "";
|
||||
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_connect_timeout 10s;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name openai.internal;
|
||||
|
||||
client_max_body_size 50m;
|
||||
|
||||
location / {
|
||||
set $upstream https://openai.baseroute.tech;
|
||||
proxy_pass $upstream;
|
||||
proxy_ssl_server_name on;
|
||||
proxy_set_header Host openai.baseroute.tech;
|
||||
proxy_set_header Authorization "Bearer %%OPENAI_KEY%%";
|
||||
proxy_set_header x-api-key "";
|
||||
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_connect_timeout 10s;
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
entrypointSh := `#!/bin/sh
|
||||
set -e
|
||||
# Escape sed special chars (& \ |) in secret values to prevent replacement corruption
|
||||
escape_sed() { printf '%s\n' "$1" | sed 's/[&\|/]/\\&/g'; }
|
||||
ANTHROPIC_KEY=$(escape_sed "$(cat /secrets/anthropic-key)")
|
||||
OPENAI_KEY=$(escape_sed "$(cat /secrets/openai-key)")
|
||||
RESOLVER=$(awk '/^nameserver/{print $2; exit}' /etc/resolv.conf)
|
||||
sed \
|
||||
-e "s|%%ANTHROPIC_KEY%%|${ANTHROPIC_KEY}|g" \
|
||||
-e "s|%%OPENAI_KEY%%|${OPENAI_KEY}|g" \
|
||||
-e "s|%%RESOLVER%%|${RESOLVER}|g" \
|
||||
/etc/nginx/templates/nginx.conf.template > /etc/nginx/nginx.conf
|
||||
exec nginx -g 'daemon off;'
|
||||
`
|
||||
|
||||
return &corev1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "ai-proxy-config",
|
||||
Namespace: ns,
|
||||
},
|
||||
Data: map[string]string{
|
||||
"nginx.conf.template": nginxConfTemplate,
|
||||
"entrypoint.sh": entrypointSh,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func boolPtr(b bool) *bool {
|
||||
return &b
|
||||
}
|
||||
|
||||
func int32Ptr(i int32) *int32 {
|
||||
return &i
|
||||
}
|
||||
|
||||
func protocolPtr(p corev1.Protocol) *corev1.Protocol {
|
||||
return &p
|
||||
}
|
||||
540
internal/k8s/templates_test.go
Normal file
540
internal/k8s/templates_test.go
Normal file
|
|
@ -0,0 +1,540 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
)
|
||||
|
||||
var testCfg = Config{
|
||||
Domain: "spinoff.dev",
|
||||
Registry: "10.22.0.56:30500",
|
||||
GoldenImage: "dev-golden:v2",
|
||||
VPNGatewayNS: "claw-system",
|
||||
VPNGatewaySecret: "vpn-gateway-secrets",
|
||||
AnthropicKey: "br_test_anthropic",
|
||||
OpenAIKey: "br_test_openai",
|
||||
ForgejoURL: "http://forgejo.dev-infra.svc:3000",
|
||||
}
|
||||
|
||||
var testOpts = PodOpts{
|
||||
User: "alice",
|
||||
Pod: "main",
|
||||
Tools: "go@1.25,rust@1.94",
|
||||
Task: "build a web server",
|
||||
CPUReq: "2",
|
||||
CPULimit: "4",
|
||||
MemReq: "4Gi",
|
||||
MemLimit: "8Gi",
|
||||
VPNKey: "test-vpn-key",
|
||||
AnthropicKey: "br_test_anthropic",
|
||||
OpenAIKey: "br_test_openai",
|
||||
}
|
||||
|
||||
func TestPVCTemplate(t *testing.T) {
|
||||
pvc := PVCTemplate("alice", "main")
|
||||
|
||||
if pvc.Name != "workspace-main" {
|
||||
t.Errorf("expected name workspace-main, got %s", pvc.Name)
|
||||
}
|
||||
if pvc.Namespace != "dev-alice" {
|
||||
t.Errorf("expected namespace dev-alice, got %s", pvc.Namespace)
|
||||
}
|
||||
if len(pvc.Spec.AccessModes) != 1 || pvc.Spec.AccessModes[0] != corev1.ReadWriteOnce {
|
||||
t.Errorf("expected ReadWriteOnce access mode")
|
||||
}
|
||||
if *pvc.Spec.StorageClassName != "longhorn" {
|
||||
t.Errorf("expected longhorn storage class, got %s", *pvc.Spec.StorageClassName)
|
||||
}
|
||||
storage := pvc.Spec.Resources.Requests[corev1.ResourceStorage]
|
||||
if storage.Cmp(resource.MustParse("20Gi")) != 0 {
|
||||
t.Errorf("expected 20Gi storage, got %s", storage.String())
|
||||
}
|
||||
if pvc.Labels["app"] != "dev-pod" {
|
||||
t.Errorf("expected label app=dev-pod, got %s", pvc.Labels["app"])
|
||||
}
|
||||
if pvc.Labels["podname"] != "main" {
|
||||
t.Errorf("expected label podname=main, got %s", pvc.Labels["podname"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPVCTemplatePerPod(t *testing.T) {
|
||||
tests := []struct {
|
||||
user string
|
||||
pod string
|
||||
wantPVC string
|
||||
}{
|
||||
{"alice", "main", "workspace-main"},
|
||||
{"bob", "build1", "workspace-build1"},
|
||||
{"alice", "runner-42", "workspace-runner-42"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.user+"/"+tt.pod, func(t *testing.T) {
|
||||
pvc := PVCTemplate(tt.user, tt.pod)
|
||||
if pvc.Name != tt.wantPVC {
|
||||
t.Errorf("expected PVC name %s, got %s", tt.wantPVC, pvc.Name)
|
||||
}
|
||||
if pvc.Labels["podname"] != tt.pod {
|
||||
t.Errorf("expected podname label %s, got %s", tt.pod, pvc.Labels["podname"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPVCName(t *testing.T) {
|
||||
if got := PVCName("main"); got != "workspace-main" {
|
||||
t.Errorf("expected workspace-main, got %s", got)
|
||||
}
|
||||
if got := PVCName("build1"); got != "workspace-build1" {
|
||||
t.Errorf("expected workspace-build1, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodTemplate(t *testing.T) {
|
||||
pod := PodTemplate(testCfg, testOpts)
|
||||
|
||||
t.Run("metadata", func(t *testing.T) {
|
||||
if pod.Name != "dev-pod-main" {
|
||||
t.Errorf("expected name dev-pod-main, got %s", pod.Name)
|
||||
}
|
||||
if pod.Namespace != "dev-alice" {
|
||||
t.Errorf("expected namespace dev-alice, got %s", pod.Namespace)
|
||||
}
|
||||
if pod.Labels["app"] != "dev-pod" {
|
||||
t.Errorf("expected label app=dev-pod, got %s", pod.Labels["app"])
|
||||
}
|
||||
if pod.Labels["podname"] != "main" {
|
||||
t.Errorf("expected label podname=main, got %s", pod.Labels["podname"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("host_aliases", func(t *testing.T) {
|
||||
if len(pod.Spec.HostAliases) != 1 {
|
||||
t.Fatalf("expected 1 host alias, got %d", len(pod.Spec.HostAliases))
|
||||
}
|
||||
ha := pod.Spec.HostAliases[0]
|
||||
if ha.IP != "127.0.0.1" {
|
||||
t.Errorf("expected IP 127.0.0.1, got %s", ha.IP)
|
||||
}
|
||||
if len(ha.Hostnames) != 2 {
|
||||
t.Fatalf("expected 2 hostnames, got %d", len(ha.Hostnames))
|
||||
}
|
||||
if ha.Hostnames[0] != "anthropic.internal" || ha.Hostnames[1] != "openai.internal" {
|
||||
t.Errorf("unexpected hostnames: %v", ha.Hostnames)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("containers_count", func(t *testing.T) {
|
||||
if len(pod.Spec.Containers) != 3 {
|
||||
t.Fatalf("expected 3 containers, got %d", len(pod.Spec.Containers))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dev_container", func(t *testing.T) {
|
||||
dev := pod.Spec.Containers[0]
|
||||
if dev.Name != "dev" {
|
||||
t.Errorf("expected container name dev, got %s", dev.Name)
|
||||
}
|
||||
if dev.Image != "10.22.0.56:30500/dev-golden:v2" {
|
||||
t.Errorf("expected golden image, got %s", dev.Image)
|
||||
}
|
||||
if dev.ImagePullPolicy != corev1.PullAlways {
|
||||
t.Errorf("expected PullAlways, got %s", dev.ImagePullPolicy)
|
||||
}
|
||||
if len(dev.Ports) != 5 {
|
||||
t.Errorf("expected 5 ports, got %d", len(dev.Ports))
|
||||
}
|
||||
|
||||
// Check resource requests
|
||||
cpuReq := dev.Resources.Requests[corev1.ResourceCPU]
|
||||
if cpuReq.Cmp(resource.MustParse("2")) != 0 {
|
||||
t.Errorf("expected cpu req 2, got %s", cpuReq.String())
|
||||
}
|
||||
memLim := dev.Resources.Limits[corev1.ResourceMemory]
|
||||
if memLim.Cmp(resource.MustParse("8Gi")) != 0 {
|
||||
t.Errorf("expected mem limit 8Gi, got %s", memLim.String())
|
||||
}
|
||||
|
||||
// Check env vars
|
||||
envMap := make(map[string]string)
|
||||
for _, e := range dev.Env {
|
||||
envMap[e.Name] = e.Value
|
||||
}
|
||||
if envMap["TASK_DESCRIPTION"] != "build a web server" {
|
||||
t.Errorf("unexpected TASK_DESCRIPTION: %s", envMap["TASK_DESCRIPTION"])
|
||||
}
|
||||
if envMap["DEV_TOOLS"] != "go@1.25,rust@1.94" {
|
||||
t.Errorf("unexpected DEV_TOOLS: %s", envMap["DEV_TOOLS"])
|
||||
}
|
||||
if envMap["TTYD_BASE_PATH"] != "/@alice/main" {
|
||||
t.Errorf("unexpected TTYD_BASE_PATH: %s", envMap["TTYD_BASE_PATH"])
|
||||
}
|
||||
if envMap["DEV_EXTERNAL_HOST"] != "spinoff.dev" {
|
||||
t.Errorf("unexpected DEV_EXTERNAL_HOST: %s", envMap["DEV_EXTERNAL_HOST"])
|
||||
}
|
||||
if envMap["FORGEJO_URL"] != "http://forgejo.dev-infra.svc:3000" {
|
||||
t.Errorf("unexpected FORGEJO_URL: %s", envMap["FORGEJO_URL"])
|
||||
}
|
||||
|
||||
// Check envFrom references dev-secrets
|
||||
if len(dev.EnvFrom) != 1 {
|
||||
t.Fatalf("expected 1 envFrom, got %d", len(dev.EnvFrom))
|
||||
}
|
||||
if dev.EnvFrom[0].SecretRef.Name != "dev-secrets" {
|
||||
t.Errorf("expected envFrom dev-secrets, got %s", dev.EnvFrom[0].SecretRef.Name)
|
||||
}
|
||||
|
||||
// Check volume mount
|
||||
if len(dev.VolumeMounts) != 1 || dev.VolumeMounts[0].MountPath != "/home/dev/workspace" {
|
||||
t.Errorf("expected workspace mount at /home/dev/workspace")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ai_proxy_container", func(t *testing.T) {
|
||||
proxy := pod.Spec.Containers[1]
|
||||
if proxy.Name != "ai-proxy" {
|
||||
t.Errorf("expected container name ai-proxy, got %s", proxy.Name)
|
||||
}
|
||||
if proxy.Image != "nginx:alpine" {
|
||||
t.Errorf("expected nginx:alpine, got %s", proxy.Image)
|
||||
}
|
||||
if len(proxy.VolumeMounts) != 2 {
|
||||
t.Errorf("expected 2 volume mounts, got %d", len(proxy.VolumeMounts))
|
||||
}
|
||||
for _, vm := range proxy.VolumeMounts {
|
||||
if !vm.ReadOnly {
|
||||
t.Errorf("expected read-only mount for %s", vm.Name)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ipip_sidecar_container", func(t *testing.T) {
|
||||
ipip := pod.Spec.Containers[2]
|
||||
if ipip.Name != "ipip-sidecar" {
|
||||
t.Errorf("expected container name ipip-sidecar, got %s", ipip.Name)
|
||||
}
|
||||
if ipip.Image != "10.22.0.56:30500/claw-ipip-tunnel:dev" {
|
||||
t.Errorf("expected ipip tunnel image, got %s", ipip.Image)
|
||||
}
|
||||
if ipip.SecurityContext == nil || !*ipip.SecurityContext.Privileged {
|
||||
t.Error("expected privileged security context")
|
||||
}
|
||||
|
||||
envMap := make(map[string]string)
|
||||
for _, e := range ipip.Env {
|
||||
if e.ValueFrom == nil {
|
||||
envMap[e.Name] = e.Value
|
||||
}
|
||||
}
|
||||
if envMap["POD_ID"] != "dev-alice-main" {
|
||||
t.Errorf("unexpected POD_ID: %s", envMap["POD_ID"])
|
||||
}
|
||||
if envMap["VPN_GATEWAY_HOST"] != "vpn-gateway.claw-system.svc" {
|
||||
t.Errorf("unexpected VPN_GATEWAY_HOST: %s", envMap["VPN_GATEWAY_HOST"])
|
||||
}
|
||||
|
||||
// Check VPN_GATEWAY_KEY comes from secret ref
|
||||
var vpnKeyEnv *corev1.EnvVar
|
||||
for i := range ipip.Env {
|
||||
if ipip.Env[i].Name == "VPN_GATEWAY_KEY" {
|
||||
vpnKeyEnv = &ipip.Env[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if vpnKeyEnv == nil || vpnKeyEnv.ValueFrom == nil || vpnKeyEnv.ValueFrom.SecretKeyRef == nil {
|
||||
t.Fatal("expected VPN_GATEWAY_KEY from secret ref")
|
||||
}
|
||||
if vpnKeyEnv.ValueFrom.SecretKeyRef.Name != "dev-secrets" {
|
||||
t.Errorf("expected secret name dev-secrets, got %s", vpnKeyEnv.ValueFrom.SecretKeyRef.Name)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("volumes", func(t *testing.T) {
|
||||
if len(pod.Spec.Volumes) != 3 {
|
||||
t.Fatalf("expected 3 volumes, got %d", len(pod.Spec.Volumes))
|
||||
}
|
||||
volumeNames := make(map[string]bool)
|
||||
for _, v := range pod.Spec.Volumes {
|
||||
volumeNames[v.Name] = true
|
||||
}
|
||||
for _, name := range []string{"workspace", "ai-proxy-config", "ai-proxy-secrets"} {
|
||||
if !volumeNames[name] {
|
||||
t.Errorf("missing volume %s", name)
|
||||
}
|
||||
}
|
||||
// Verify workspace volume references per-pod PVC
|
||||
wsVol := pod.Spec.Volumes[0]
|
||||
if wsVol.PersistentVolumeClaim.ClaimName != "workspace-main" {
|
||||
t.Errorf("expected workspace PVC claim workspace-main, got %s", wsVol.PersistentVolumeClaim.ClaimName)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestServiceTemplate(t *testing.T) {
|
||||
svc := ServiceTemplate("alice", "main")
|
||||
|
||||
if svc.Name != "dev-pod-main-svc" {
|
||||
t.Errorf("expected name dev-pod-main-svc, got %s", svc.Name)
|
||||
}
|
||||
if svc.Namespace != "dev-alice" {
|
||||
t.Errorf("expected namespace dev-alice, got %s", svc.Namespace)
|
||||
}
|
||||
if svc.Spec.Type != corev1.ServiceTypeClusterIP {
|
||||
t.Errorf("expected ClusterIP, got %s", svc.Spec.Type)
|
||||
}
|
||||
if len(svc.Spec.Ports) != 5 {
|
||||
t.Errorf("expected 5 ports, got %d", len(svc.Spec.Ports))
|
||||
}
|
||||
|
||||
portMap := make(map[string]int32)
|
||||
for _, p := range svc.Spec.Ports {
|
||||
portMap[p.Name] = p.Port
|
||||
}
|
||||
expected := map[string]int32{
|
||||
"ttyd": 7681, "ssh": 22, "forgejo": 3000,
|
||||
"vscode": 8080, "ralphex-gerrit": 8090,
|
||||
}
|
||||
for name, port := range expected {
|
||||
if portMap[name] != port {
|
||||
t.Errorf("expected port %s=%d, got %d", name, port, portMap[name])
|
||||
}
|
||||
}
|
||||
|
||||
if svc.Spec.Selector["app"] != "dev-pod" || svc.Spec.Selector["podname"] != "main" {
|
||||
t.Errorf("unexpected selector: %v", svc.Spec.Selector)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIngressTemplate(t *testing.T) {
|
||||
objs := IngressTemplate("alice", "main", "spinoff.dev")
|
||||
|
||||
if len(objs) != 3 {
|
||||
t.Fatalf("expected 3 objects (ingress + 2 middleware), got %d", len(objs))
|
||||
}
|
||||
|
||||
t.Run("ingress_route", func(t *testing.T) {
|
||||
ir := objs[0]
|
||||
if ir.GetKind() != "IngressRoute" {
|
||||
t.Errorf("expected kind IngressRoute, got %s", ir.GetKind())
|
||||
}
|
||||
if ir.GetName() != "dev-pod-main-ingress" {
|
||||
t.Errorf("expected name dev-pod-main-ingress, got %s", ir.GetName())
|
||||
}
|
||||
if ir.GetNamespace() != "dev-alice" {
|
||||
t.Errorf("expected namespace dev-alice, got %s", ir.GetNamespace())
|
||||
}
|
||||
|
||||
spec := ir.Object["spec"].(map[string]interface{})
|
||||
routes := spec["routes"].([]interface{})
|
||||
if len(routes) != 5 {
|
||||
t.Errorf("expected 5 routes, got %d", len(routes))
|
||||
}
|
||||
|
||||
// Check first route is ttyd (no explicit priority — Traefik v3 auto-calculates)
|
||||
r0 := routes[0].(map[string]interface{})
|
||||
if _, hasPriority := r0["priority"]; hasPriority {
|
||||
t.Errorf("expected no explicit priority, got %v", r0["priority"])
|
||||
}
|
||||
if r0["match"] != "Host(`spinoff.dev`) && PathPrefix(`/@alice/main/`)" {
|
||||
t.Errorf("unexpected match: %s", r0["match"])
|
||||
}
|
||||
|
||||
// Check last route has strip prefix middleware
|
||||
r4 := routes[4].(map[string]interface{})
|
||||
middlewares := r4["middlewares"].([]interface{})
|
||||
if len(middlewares) != 2 {
|
||||
t.Errorf("expected 2 middlewares for ralphex route, got %d", len(middlewares))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("basic_auth_middleware", func(t *testing.T) {
|
||||
mw := objs[1]
|
||||
if mw.GetKind() != "Middleware" {
|
||||
t.Errorf("expected kind Middleware, got %s", mw.GetKind())
|
||||
}
|
||||
if mw.GetName() != "spinoff-basic-auth" {
|
||||
t.Errorf("expected name spinoff-basic-auth, got %s", mw.GetName())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("strip_prefix_middleware", func(t *testing.T) {
|
||||
mw := objs[2]
|
||||
if mw.GetName() != "strip-dev-main-ralphex-prefix" {
|
||||
t.Errorf("expected name strip-dev-main-ralphex-prefix, got %s", mw.GetName())
|
||||
}
|
||||
spec := mw.Object["spec"].(map[string]interface{})
|
||||
stripPrefix := spec["stripPrefix"].(map[string]interface{})
|
||||
prefixes := stripPrefix["prefixes"].([]interface{})
|
||||
if len(prefixes) != 1 || prefixes[0] != "/@alice/main/ralphex" {
|
||||
t.Errorf("unexpected prefixes: %v", prefixes)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSecretsTemplate(t *testing.T) {
|
||||
devSec, aiSec := SecretsTemplate("alice", "vpn-key-123", "br_anthropic_real", "br_openai_real", "forgejo-tok-abc", "tskey-auth-abc123")
|
||||
|
||||
t.Run("dev_secrets", func(t *testing.T) {
|
||||
if devSec.Name != "dev-secrets" {
|
||||
t.Errorf("expected name dev-secrets, got %s", devSec.Name)
|
||||
}
|
||||
if devSec.Namespace != "dev-alice" {
|
||||
t.Errorf("expected namespace dev-alice, got %s", devSec.Namespace)
|
||||
}
|
||||
if devSec.StringData["ANTHROPIC_API_KEY"] != "sk-devpod" {
|
||||
t.Errorf("expected dummy anthropic key, got %s", devSec.StringData["ANTHROPIC_API_KEY"])
|
||||
}
|
||||
if devSec.StringData["ANTHROPIC_BASE_URL"] != "http://anthropic.internal" {
|
||||
t.Errorf("unexpected base url: %s", devSec.StringData["ANTHROPIC_BASE_URL"])
|
||||
}
|
||||
if devSec.StringData["BASEROUTE_OPENAI_KEY"] != "sk-devpod" {
|
||||
t.Errorf("expected dummy openai key, got %s", devSec.StringData["BASEROUTE_OPENAI_KEY"])
|
||||
}
|
||||
if devSec.StringData["VPN_GATEWAY_KEY"] != "vpn-key-123" {
|
||||
t.Errorf("expected vpn key vpn-key-123, got %s", devSec.StringData["VPN_GATEWAY_KEY"])
|
||||
}
|
||||
if devSec.StringData["FORGEJO_TOKEN"] != "forgejo-tok-abc" {
|
||||
t.Errorf("expected forgejo token forgejo-tok-abc, got %s", devSec.StringData["FORGEJO_TOKEN"])
|
||||
}
|
||||
if devSec.StringData["TAILSCALE_AUTHKEY"] != "tskey-auth-abc123" {
|
||||
t.Errorf("expected tailscale key tskey-auth-abc123, got %s", devSec.StringData["TAILSCALE_AUTHKEY"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ai_proxy_secrets", func(t *testing.T) {
|
||||
if aiSec.Name != "ai-proxy-secrets" {
|
||||
t.Errorf("expected name ai-proxy-secrets, got %s", aiSec.Name)
|
||||
}
|
||||
if aiSec.Namespace != "dev-alice" {
|
||||
t.Errorf("expected namespace dev-alice, got %s", aiSec.Namespace)
|
||||
}
|
||||
if aiSec.StringData["anthropic-key"] != "br_anthropic_real" {
|
||||
t.Errorf("expected real anthropic key, got %s", aiSec.StringData["anthropic-key"])
|
||||
}
|
||||
if aiSec.StringData["openai-key"] != "br_openai_real" {
|
||||
t.Errorf("expected real openai key, got %s", aiSec.StringData["openai-key"])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSecretsTemplate_NoTailscaleKey(t *testing.T) {
|
||||
devSec, _ := SecretsTemplate("alice", "vpn-key-123", "br_anthropic_real", "br_openai_real", "forgejo-tok-abc", "")
|
||||
if _, exists := devSec.StringData["TAILSCALE_AUTHKEY"]; exists {
|
||||
t.Error("TAILSCALE_AUTHKEY should not be present when tailscale key is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetworkPolicyTemplate(t *testing.T) {
|
||||
np := NetworkPolicyTemplate("alice")
|
||||
|
||||
if np.Name != "dev-pod-network-policy" {
|
||||
t.Errorf("expected name dev-pod-network-policy, got %s", np.Name)
|
||||
}
|
||||
if np.Namespace != "dev-alice" {
|
||||
t.Errorf("expected namespace dev-alice, got %s", np.Namespace)
|
||||
}
|
||||
if len(np.Spec.PolicyTypes) != 2 {
|
||||
t.Errorf("expected 2 policy types, got %d", len(np.Spec.PolicyTypes))
|
||||
}
|
||||
|
||||
t.Run("ingress_rules", func(t *testing.T) {
|
||||
if len(np.Spec.Ingress) != 1 {
|
||||
t.Fatalf("expected 1 ingress rule, got %d", len(np.Spec.Ingress))
|
||||
}
|
||||
from := np.Spec.Ingress[0].From
|
||||
if len(from) != 1 {
|
||||
t.Fatalf("expected 1 from peer, got %d", len(from))
|
||||
}
|
||||
nsLabel := from[0].NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"]
|
||||
if nsLabel != "kube-system" {
|
||||
t.Errorf("expected kube-system, got %s", nsLabel)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("egress_rules", func(t *testing.T) {
|
||||
if len(np.Spec.Egress) != 3 {
|
||||
t.Fatalf("expected 3 egress rules, got %d", len(np.Spec.Egress))
|
||||
}
|
||||
|
||||
// VPN gateway rule
|
||||
vpnRule := np.Spec.Egress[0]
|
||||
if len(vpnRule.To) != 1 {
|
||||
t.Fatalf("expected 1 to peer in vpn rule, got %d", len(vpnRule.To))
|
||||
}
|
||||
nsLabel := vpnRule.To[0].NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"]
|
||||
if nsLabel != "claw-system" {
|
||||
t.Errorf("expected claw-system, got %s", nsLabel)
|
||||
}
|
||||
podLabel := vpnRule.To[0].PodSelector.MatchLabels["app"]
|
||||
if podLabel != "vpn-gateway" {
|
||||
t.Errorf("expected vpn-gateway, got %s", podLabel)
|
||||
}
|
||||
|
||||
// dev-infra rule
|
||||
devInfraRule := np.Spec.Egress[1]
|
||||
nsLabel = devInfraRule.To[0].NamespaceSelector.MatchLabels["kubernetes.io/metadata.name"]
|
||||
if nsLabel != "dev-infra" {
|
||||
t.Errorf("expected dev-infra, got %s", nsLabel)
|
||||
}
|
||||
|
||||
// DNS rule
|
||||
dnsRule := np.Spec.Egress[2]
|
||||
if len(dnsRule.Ports) != 2 {
|
||||
t.Errorf("expected 2 DNS ports, got %d", len(dnsRule.Ports))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAIProxyConfigMapTemplate(t *testing.T) {
|
||||
cm := AIProxyConfigMapTemplate("alice")
|
||||
|
||||
if cm.Name != "ai-proxy-config" {
|
||||
t.Errorf("expected name ai-proxy-config, got %s", cm.Name)
|
||||
}
|
||||
if cm.Namespace != "dev-alice" {
|
||||
t.Errorf("expected namespace dev-alice, got %s", cm.Namespace)
|
||||
}
|
||||
if _, ok := cm.Data["nginx.conf.template"]; !ok {
|
||||
t.Error("missing nginx.conf.template in ConfigMap data")
|
||||
}
|
||||
if _, ok := cm.Data["entrypoint.sh"]; !ok {
|
||||
t.Error("missing entrypoint.sh in ConfigMap data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodTemplateWithDifferentUsers(t *testing.T) {
|
||||
tests := []struct {
|
||||
user string
|
||||
pod string
|
||||
wantNS string
|
||||
wantPod string
|
||||
wantPath string
|
||||
}{
|
||||
{"bob", "dev", "dev-bob", "dev-pod-dev", "/@bob/dev"},
|
||||
{"team-alpha", "staging", "dev-team-alpha", "dev-pod-staging", "/@team-alpha/staging"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.user+"/"+tt.pod, func(t *testing.T) {
|
||||
opts := PodOpts{
|
||||
User: tt.user, Pod: tt.pod,
|
||||
CPUReq: "1", CPULimit: "2", MemReq: "1Gi", MemLimit: "2Gi",
|
||||
}
|
||||
pod := PodTemplate(testCfg, opts)
|
||||
if pod.Namespace != tt.wantNS {
|
||||
t.Errorf("expected namespace %s, got %s", tt.wantNS, pod.Namespace)
|
||||
}
|
||||
if pod.Name != tt.wantPod {
|
||||
t.Errorf("expected pod name %s, got %s", tt.wantPod, pod.Name)
|
||||
}
|
||||
// Check TTYD_BASE_PATH
|
||||
dev := pod.Spec.Containers[0]
|
||||
for _, e := range dev.Env {
|
||||
if e.Name == "TTYD_BASE_PATH" && e.Value != tt.wantPath {
|
||||
t.Errorf("expected TTYD_BASE_PATH %s, got %s", tt.wantPath, e.Value)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
381
internal/k8s/workflow_templates_test.go
Normal file
381
internal/k8s/workflow_templates_test.go
Normal file
|
|
@ -0,0 +1,381 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func workflowTemplatesDir() string {
|
||||
_, thisFile, _, _ := runtime.Caller(0)
|
||||
return filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "..", "docker", "dev-golden", "forgejo-workflow-templates")
|
||||
}
|
||||
|
||||
type workflowDoc struct {
|
||||
Name string `yaml:"name"`
|
||||
On map[string]interface{} `yaml:"on"`
|
||||
Jobs map[string]interface{} `yaml:"jobs"`
|
||||
}
|
||||
|
||||
func readWorkflowYAML(t *testing.T, name string) workflowDoc {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(workflowTemplatesDir(), name))
|
||||
if err != nil {
|
||||
t.Fatalf("reading %s: %v", name, err)
|
||||
}
|
||||
var doc workflowDoc
|
||||
if err := yaml.Unmarshal(data, &doc); err != nil {
|
||||
t.Fatalf("parsing %s: %v", name, err)
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
func jobFromWorkflow(t *testing.T, doc workflowDoc, jobName string) map[string]interface{} {
|
||||
t.Helper()
|
||||
job, ok := doc.Jobs[jobName]
|
||||
if !ok {
|
||||
t.Fatalf("missing %s job", jobName)
|
||||
}
|
||||
jobMap, ok := job.(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("job %s is not a map", jobName)
|
||||
}
|
||||
return jobMap
|
||||
}
|
||||
|
||||
func stepsFromJob(t *testing.T, job map[string]interface{}) []interface{} {
|
||||
t.Helper()
|
||||
steps, ok := job["steps"].([]interface{})
|
||||
if !ok || len(steps) == 0 {
|
||||
t.Fatal("missing steps")
|
||||
}
|
||||
return steps
|
||||
}
|
||||
|
||||
func stepNames(steps []interface{}) []string {
|
||||
var names []string
|
||||
for _, s := range steps {
|
||||
step := s.(map[string]interface{})
|
||||
if name, ok := step["name"].(string); ok {
|
||||
names = append(names, name)
|
||||
}
|
||||
if uses, ok := step["uses"].(string); ok {
|
||||
names = append(names, uses)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func assertRunsOnSelfHosted(t *testing.T, job map[string]interface{}) {
|
||||
t.Helper()
|
||||
runsOn, ok := job["runs-on"].([]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing runs-on")
|
||||
}
|
||||
for _, r := range runsOn {
|
||||
if r == "self-hosted" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Error("expected 'self-hosted' in runs-on")
|
||||
}
|
||||
|
||||
func assertStepsContain(t *testing.T, names []string, required []string) {
|
||||
t.Helper()
|
||||
for _, req := range required {
|
||||
found := false
|
||||
for _, name := range names {
|
||||
if strings.Contains(name, req) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("missing required step containing %q", req)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkflowTemplatesExist(t *testing.T) {
|
||||
dir := workflowTemplatesDir()
|
||||
expected := []string{
|
||||
"claude-implement.yml",
|
||||
"claude-review.yml",
|
||||
"ci-build.yml",
|
||||
}
|
||||
for _, name := range expected {
|
||||
path := filepath.Join(dir, name)
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
t.Errorf("missing workflow template: %s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaudeImplementWorkflow(t *testing.T) {
|
||||
doc := readWorkflowYAML(t, "claude-implement.yml")
|
||||
|
||||
t.Run("name", func(t *testing.T) {
|
||||
if doc.Name != "Claude Implement" {
|
||||
t.Errorf("expected name 'Claude Implement', got %v", doc.Name)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("trigger", func(t *testing.T) {
|
||||
ic, ok := doc.On["issue_comment"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing issue_comment trigger")
|
||||
}
|
||||
types, ok := ic["types"].([]interface{})
|
||||
if !ok || len(types) == 0 {
|
||||
t.Fatal("missing types for issue_comment")
|
||||
}
|
||||
if types[0] != "created" {
|
||||
t.Errorf("expected type 'created', got %v", types[0])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("job_condition", func(t *testing.T) {
|
||||
impl := jobFromWorkflow(t, doc, "implement")
|
||||
ifCond, ok := impl["if"].(string)
|
||||
if !ok {
|
||||
t.Fatal("missing 'if' condition")
|
||||
}
|
||||
if !strings.Contains(ifCond, "@claude implement") {
|
||||
t.Error("'if' condition should check for '@claude implement'")
|
||||
}
|
||||
if !strings.Contains(ifCond, "pull_request == null") {
|
||||
t.Error("'if' condition should exclude PR comments")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("runs_on_self_hosted", func(t *testing.T) {
|
||||
impl := jobFromWorkflow(t, doc, "implement")
|
||||
assertRunsOnSelfHosted(t, impl)
|
||||
})
|
||||
|
||||
t.Run("has_required_steps", func(t *testing.T) {
|
||||
impl := jobFromWorkflow(t, doc, "implement")
|
||||
steps := stepsFromJob(t, impl)
|
||||
assertStepsContain(t, stepNames(steps), []string{
|
||||
"actions/checkout@v4",
|
||||
"Configure git",
|
||||
"Extract task",
|
||||
"Run Claude Code",
|
||||
"Create branch and push",
|
||||
"Create pull request",
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("permissions", func(t *testing.T) {
|
||||
impl := jobFromWorkflow(t, doc, "implement")
|
||||
perms, ok := impl["permissions"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing permissions")
|
||||
}
|
||||
if perms["contents"] != "write" {
|
||||
t.Error("expected contents: write permission")
|
||||
}
|
||||
if perms["pull-requests"] != "write" {
|
||||
t.Error("expected pull-requests: write permission")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestClaudeReviewWorkflow(t *testing.T) {
|
||||
doc := readWorkflowYAML(t, "claude-review.yml")
|
||||
|
||||
t.Run("name", func(t *testing.T) {
|
||||
if doc.Name != "Claude Review" {
|
||||
t.Errorf("expected name 'Claude Review', got %v", doc.Name)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("trigger", func(t *testing.T) {
|
||||
ic, ok := doc.On["issue_comment"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing issue_comment trigger")
|
||||
}
|
||||
types, ok := ic["types"].([]interface{})
|
||||
if !ok || len(types) == 0 {
|
||||
t.Fatal("missing types for issue_comment")
|
||||
}
|
||||
if types[0] != "created" {
|
||||
t.Errorf("expected type 'created', got %v", types[0])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("job_condition", func(t *testing.T) {
|
||||
review := jobFromWorkflow(t, doc, "review")
|
||||
ifCond, ok := review["if"].(string)
|
||||
if !ok {
|
||||
t.Fatal("missing 'if' condition")
|
||||
}
|
||||
if !strings.Contains(ifCond, "@claude review") {
|
||||
t.Error("'if' condition should check for '@claude review'")
|
||||
}
|
||||
if !strings.Contains(ifCond, "pull_request != null") {
|
||||
t.Error("'if' condition should require PR context")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("runs_on_self_hosted", func(t *testing.T) {
|
||||
review := jobFromWorkflow(t, doc, "review")
|
||||
assertRunsOnSelfHosted(t, review)
|
||||
})
|
||||
|
||||
t.Run("has_required_steps", func(t *testing.T) {
|
||||
review := jobFromWorkflow(t, doc, "review")
|
||||
steps := stepsFromJob(t, review)
|
||||
assertStepsContain(t, stepNames(steps), []string{
|
||||
"actions/checkout@v4",
|
||||
"Fetch base branch",
|
||||
"Run Claude Code review",
|
||||
"Post review comment",
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("checkout_uses_pr_ref", func(t *testing.T) {
|
||||
review := jobFromWorkflow(t, doc, "review")
|
||||
steps := stepsFromJob(t, review)
|
||||
for _, s := range steps {
|
||||
step := s.(map[string]interface{})
|
||||
if uses, ok := step["uses"].(string); ok && strings.Contains(uses, "checkout") {
|
||||
with, ok := step["with"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("checkout step missing 'with' for PR ref")
|
||||
}
|
||||
ref, _ := with["ref"].(string)
|
||||
if ref == "" {
|
||||
t.Error("checkout step should specify ref for PR branch")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Error("no checkout step found")
|
||||
})
|
||||
|
||||
t.Run("permissions", func(t *testing.T) {
|
||||
review := jobFromWorkflow(t, doc, "review")
|
||||
perms, ok := review["permissions"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing permissions")
|
||||
}
|
||||
if perms["pull-requests"] != "write" {
|
||||
t.Error("expected pull-requests: write permission")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCIBuildWorkflow(t *testing.T) {
|
||||
doc := readWorkflowYAML(t, "ci-build.yml")
|
||||
|
||||
t.Run("name", func(t *testing.T) {
|
||||
if doc.Name != "CI Build" {
|
||||
t.Errorf("expected name 'CI Build', got %v", doc.Name)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("trigger", func(t *testing.T) {
|
||||
push, ok := doc.On["push"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatal("missing push trigger")
|
||||
}
|
||||
branches, ok := push["branches"].([]interface{})
|
||||
if !ok || len(branches) == 0 {
|
||||
t.Fatal("missing branches for push trigger")
|
||||
}
|
||||
if branches[0] != "main" {
|
||||
t.Errorf("expected branch 'main', got %v", branches[0])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("job_exists", func(t *testing.T) {
|
||||
jobFromWorkflow(t, doc, "build")
|
||||
})
|
||||
|
||||
t.Run("runs_on_self_hosted", func(t *testing.T) {
|
||||
build := jobFromWorkflow(t, doc, "build")
|
||||
assertRunsOnSelfHosted(t, build)
|
||||
})
|
||||
|
||||
t.Run("has_checkout_and_build_steps", func(t *testing.T) {
|
||||
build := jobFromWorkflow(t, doc, "build")
|
||||
steps := stepsFromJob(t, build)
|
||||
names := stepNames(steps)
|
||||
|
||||
hasCheckout := false
|
||||
hasBuild := false
|
||||
for _, n := range names {
|
||||
if strings.Contains(n, "checkout") {
|
||||
hasCheckout = true
|
||||
}
|
||||
if strings.Contains(strings.ToLower(n), "build") {
|
||||
hasBuild = true
|
||||
}
|
||||
}
|
||||
if !hasCheckout {
|
||||
t.Error("missing checkout step")
|
||||
}
|
||||
if !hasBuild {
|
||||
t.Error("missing build step")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("build_detects_multiple_languages", func(t *testing.T) {
|
||||
build := jobFromWorkflow(t, doc, "build")
|
||||
steps := stepsFromJob(t, build)
|
||||
|
||||
for _, s := range steps {
|
||||
step := s.(map[string]interface{})
|
||||
name, _ := step["name"].(string)
|
||||
if !strings.Contains(strings.ToLower(name), "build") {
|
||||
continue
|
||||
}
|
||||
run, ok := step["run"].(string)
|
||||
if !ok {
|
||||
t.Fatal("build step missing run command")
|
||||
}
|
||||
for _, marker := range []string{"Cargo.toml", "go.mod", "package.json"} {
|
||||
if !strings.Contains(run, marker) {
|
||||
t.Errorf("build step should detect %s", marker)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Error("no build step found with run command")
|
||||
})
|
||||
}
|
||||
|
||||
func TestWorkflowTemplatesAreValidYAML(t *testing.T) {
|
||||
dir := workflowTemplatesDir()
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("reading templates dir: %v", err)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yml") {
|
||||
continue
|
||||
}
|
||||
t.Run(entry.Name(), func(t *testing.T) {
|
||||
data, err := os.ReadFile(filepath.Join(dir, entry.Name()))
|
||||
if err != nil {
|
||||
t.Fatalf("reading %s: %v", entry.Name(), err)
|
||||
}
|
||||
var doc workflowDoc
|
||||
if err := yaml.Unmarshal(data, &doc); err != nil {
|
||||
t.Fatalf("invalid YAML in %s: %v", entry.Name(), err)
|
||||
}
|
||||
if doc.Name == "" {
|
||||
t.Errorf("%s missing 'name' field", entry.Name())
|
||||
}
|
||||
if doc.Jobs == nil {
|
||||
t.Errorf("%s missing 'jobs' field", entry.Name())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue