package k8s import ( "fmt" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/intstr" "github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model" ) // PodOpts holds all parameters needed to create a dev pod and its resources. type PodOpts struct { User string Pod string Tools string Task string CPUReq string CPULimit string MemReq string MemLimit string VPNKey string AnthropicKey string OpenAIKey string ForgejoToken string TailscaleKey string } // PVCName returns the per-pod PVC name for a given pod. func PVCName(pod string) string { return fmt.Sprintf("workspace-%s", pod) } // PVCTemplate returns a PersistentVolumeClaim for a specific pod's workspace. func PVCTemplate(user, pod string) *corev1.PersistentVolumeClaim { storageClass := "longhorn" return &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: PVCName(pod), Namespace: model.NamespaceName(user), Labels: map[string]string{ "app": "dev-pod", "podname": pod, }, }, Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, StorageClassName: &storageClass, Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceStorage: resource.MustParse("20Gi"), }, }, }, } } // PodTemplate returns a Pod spec matching the YAML pod-template.yaml. func PodTemplate(cfg Config, opts PodOpts) *corev1.Pod { ns := model.NamespaceName(opts.User) podName := model.PodFullName(opts.Pod) return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Namespace: ns, Labels: map[string]string{ "app": "dev-pod", "podname": opts.Pod, }, }, Spec: corev1.PodSpec{ HostAliases: []corev1.HostAlias{ { IP: "127.0.0.1", Hostnames: []string{"anthropic.internal", "openai.internal"}, }, }, Containers: []corev1.Container{ devContainer(cfg, opts), aiProxyContainer(), ipipSidecarContainer(cfg, opts), }, Volumes: []corev1.Volume{ { Name: "workspace", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: PVCName(opts.Pod), }, }, }, { Name: "ai-proxy-config", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ Name: "ai-proxy-config", }, DefaultMode: int32Ptr(0755), }, }, }, { Name: "ai-proxy-secrets", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "ai-proxy-secrets", }, }, }, }, }, } } func devContainer(cfg Config, opts PodOpts) corev1.Container { return corev1.Container{ Name: "dev", Image: fmt.Sprintf("%s/%s", cfg.Registry, cfg.GoldenImage), ImagePullPolicy: corev1.PullAlways, Ports: []corev1.ContainerPort{ {ContainerPort: 7681, Name: "ttyd"}, {ContainerPort: 22, Name: "ssh"}, {ContainerPort: 3000, Name: "forgejo"}, {ContainerPort: 8080, Name: "vscode"}, {ContainerPort: 8090, Name: "ralphex-gerrit"}, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(opts.CPUReq), corev1.ResourceMemory: resource.MustParse(opts.MemReq), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(opts.CPULimit), corev1.ResourceMemory: resource.MustParse(opts.MemLimit), }, }, EnvFrom: []corev1.EnvFromSource{ { SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{ Name: "dev-secrets", }, }, }, }, Env: []corev1.EnvVar{ {Name: "TASK_DESCRIPTION", Value: opts.Task}, {Name: "DEV_TOOLS", Value: opts.Tools}, {Name: "DEV_BASE_PATH", Value: fmt.Sprintf("/@%s/%s", opts.User, opts.Pod)}, {Name: "DEV_EXTERNAL_HOST", Value: cfg.Domain}, {Name: "TTYD_BASE_PATH", Value: fmt.Sprintf("/@%s/%s", opts.User, opts.Pod)}, {Name: "FORGEJO_URL", Value: cfg.ForgejoURL}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "workspace", MountPath: "/home/dev/workspace"}, }, } } func aiProxyContainer() corev1.Container { return corev1.Container{ Name: "ai-proxy", Image: "nginx:alpine", Command: []string{"/bin/sh", "/etc/nginx/templates/entrypoint.sh"}, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("10m"), corev1.ResourceMemory: resource.MustParse("16Mi"), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("50m"), corev1.ResourceMemory: resource.MustParse("32Mi"), }, }, VolumeMounts: []corev1.VolumeMount{ {Name: "ai-proxy-config", MountPath: "/etc/nginx/templates", ReadOnly: true}, {Name: "ai-proxy-secrets", MountPath: "/secrets", ReadOnly: true}, }, } } func ipipSidecarContainer(cfg Config, opts PodOpts) corev1.Container { return corev1.Container{ Name: "ipip-sidecar", Image: fmt.Sprintf("%s/claw-ipip-tunnel:dev", cfg.Registry), ImagePullPolicy: corev1.PullAlways, SecurityContext: &corev1.SecurityContext{ Privileged: boolPtr(true), Capabilities: &corev1.Capabilities{ Add: []corev1.Capability{"NET_ADMIN"}, }, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("10m"), corev1.ResourceMemory: resource.MustParse("8Mi"), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("50m"), corev1.ResourceMemory: resource.MustParse("32Mi"), }, }, Env: []corev1.EnvVar{ {Name: "POD_ID", Value: fmt.Sprintf("dev-%s-%s", opts.User, opts.Pod)}, {Name: "VPN_GATEWAY_HOST", Value: fmt.Sprintf("vpn-gateway.%s.svc", cfg.VPNGatewayNS)}, { Name: "VPN_GATEWAY_KEY", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: "dev-secrets"}, Key: "VPN_GATEWAY_KEY", }, }, }, }, } } // ServiceTemplate returns a Service for the dev pod matching service.yaml. func ServiceTemplate(user, pod string) *corev1.Service { ns := model.NamespaceName(user) return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: model.ServiceName(pod), Namespace: ns, Labels: map[string]string{ "app": "dev-pod", "podname": pod, }, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, Ports: []corev1.ServicePort{ {Name: "ttyd", Port: 7681, TargetPort: intstr.FromInt32(7681), Protocol: corev1.ProtocolTCP}, {Name: "ssh", Port: 22, TargetPort: intstr.FromInt32(22), Protocol: corev1.ProtocolTCP}, {Name: "forgejo", Port: 3000, TargetPort: intstr.FromInt32(3000), Protocol: corev1.ProtocolTCP}, {Name: "vscode", Port: 8080, TargetPort: intstr.FromInt32(8080), Protocol: corev1.ProtocolTCP}, {Name: "ralphex-gerrit", Port: 8090, TargetPort: intstr.FromInt32(8090), Protocol: corev1.ProtocolTCP}, }, Selector: map[string]string{ "app": "dev-pod", "podname": pod, }, }, } } // IngressTemplate returns Traefik IngressRoute as an unstructured object matching ingress.yaml. // Returns the IngressRoute and two Middleware objects. func IngressTemplate(user, pod, domain string) []*unstructured.Unstructured { ns := model.NamespaceName(user) svcName := model.ServiceName(pod) ingressRoute := &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "traefik.io/v1alpha1", "kind": "IngressRoute", "metadata": map[string]interface{}{ "name": fmt.Sprintf("dev-pod-%s-ingress", pod), "namespace": ns, }, "spec": map[string]interface{}{ "entryPoints": []interface{}{"web", "websecure"}, // No TLS section: Caddy terminates TLS and forwards HTTP to Traefik. // With TLS here, Traefik v3 only routes on websecure, breaking Caddy -> port 80 flow. // No explicit priority: Traefik v3 auto-calculates from rule length, // ensuring these beat the OpenClaw catch-all (Host:* / PathPrefix:/). "routes": []interface{}{ // ttyd — base route map[string]interface{}{ "match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/`)", domain, user, pod), "kind": "Rule", "middlewares": []interface{}{ map[string]interface{}{"name": "spinoff-basic-auth"}, }, "services": []interface{}{ map[string]interface{}{"name": svcName, "port": int64(7681)}, }, }, // Forgejo map[string]interface{}{ "match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/forgejo/`)", domain, user, pod), "kind": "Rule", "middlewares": []interface{}{ map[string]interface{}{"name": "spinoff-basic-auth"}, }, "services": []interface{}{ map[string]interface{}{"name": svcName, "port": int64(3000)}, }, }, // VS Code map[string]interface{}{ "match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/vscode/`)", domain, user, pod), "kind": "Rule", "middlewares": []interface{}{ map[string]interface{}{"name": "spinoff-basic-auth"}, }, "services": []interface{}{ map[string]interface{}{"name": svcName, "port": int64(8080)}, }, }, // Gerrit code review (handles own base path, shares port 8090 with ralphex) map[string]interface{}{ "match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/gerrit/`)", domain, user, pod), "kind": "Rule", "middlewares": []interface{}{ map[string]interface{}{"name": "spinoff-basic-auth"}, }, "services": []interface{}{ map[string]interface{}{"name": svcName, "port": int64(8090)}, }, }, // Ralphex dashboard (with prefix stripping) map[string]interface{}{ "match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/ralphex/`)", domain, user, pod), "kind": "Rule", "middlewares": []interface{}{ map[string]interface{}{"name": "spinoff-basic-auth"}, map[string]interface{}{"name": fmt.Sprintf("strip-dev-%s-ralphex-prefix", pod)}, }, "services": []interface{}{ map[string]interface{}{"name": svcName, "port": int64(8090)}, }, }, }, }, }, } basicAuthMiddleware := &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "traefik.io/v1alpha1", "kind": "Middleware", "metadata": map[string]interface{}{ "name": "spinoff-basic-auth", "namespace": ns, }, "spec": map[string]interface{}{ "basicAuth": map[string]interface{}{ "secret": "spinoff-basic-auth", }, }, }, } stripPrefixMiddleware := &unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "traefik.io/v1alpha1", "kind": "Middleware", "metadata": map[string]interface{}{ "name": fmt.Sprintf("strip-dev-%s-ralphex-prefix", pod), "namespace": ns, }, "spec": map[string]interface{}{ "stripPrefix": map[string]interface{}{ "prefixes": []interface{}{ fmt.Sprintf("/@%s/%s/ralphex", user, pod), }, }, }, }, } return []*unstructured.Unstructured{ingressRoute, basicAuthMiddleware, stripPrefixMiddleware} } // SecretsTemplate returns dev-secrets and ai-proxy-secrets matching secrets-template.yaml. func SecretsTemplate(user, vpnKey, anthropicKey, openaiKey, forgejoToken, tailscaleKey string) (*corev1.Secret, *corev1.Secret) { ns := model.NamespaceName(user) stringData := map[string]string{ "ANTHROPIC_API_KEY": "sk-devpod", "ANTHROPIC_BASE_URL": "http://anthropic.internal", "BASEROUTE_OPENAI_KEY": "sk-devpod", "VPN_GATEWAY_KEY": vpnKey, "FORGEJO_TOKEN": forgejoToken, } if tailscaleKey != "" { stringData["TAILSCALE_AUTHKEY"] = tailscaleKey } devSecrets := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "dev-secrets", Namespace: ns, }, Type: corev1.SecretTypeOpaque, StringData: stringData, } aiProxySecrets := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "ai-proxy-secrets", Namespace: ns, }, Type: corev1.SecretTypeOpaque, StringData: map[string]string{ "anthropic-key": anthropicKey, "openai-key": openaiKey, }, } return devSecrets, aiProxySecrets } // BasicAuthSecretTemplate returns the spinoff-basic-auth Secret containing htpasswd data. // Matches the secret created by dev-pod-create.sh for Traefik basicAuth middleware. func BasicAuthSecretTemplate(user string) *corev1.Secret { ns := model.NamespaceName(user) return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "spinoff-basic-auth", Namespace: ns, }, Type: corev1.SecretTypeOpaque, StringData: map[string]string{ "users": "admin:$2y$05$i2Wxz4GpO8.jnMXWk1len.xcP0.wPSL.ozfd/OgX8BCo9chO8F2WO\n", }, } } // NetworkPolicyTemplate returns a NetworkPolicy matching network-policy.yaml. func NetworkPolicyTemplate(user string) *networkingv1.NetworkPolicy { ns := model.NamespaceName(user) dnsPort := intstr.FromInt32(53) return &networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: "dev-pod-network-policy", Namespace: ns, }, Spec: networkingv1.NetworkPolicySpec{ PodSelector: metav1.LabelSelector{}, PolicyTypes: []networkingv1.PolicyType{ networkingv1.PolicyTypeIngress, networkingv1.PolicyTypeEgress, }, Ingress: []networkingv1.NetworkPolicyIngressRule{ { From: []networkingv1.NetworkPolicyPeer{ { NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "kubernetes.io/metadata.name": "kube-system", }, }, }, }, }, }, Egress: []networkingv1.NetworkPolicyEgressRule{ // VPN gateway in claw-system { To: []networkingv1.NetworkPolicyPeer{ { NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "kubernetes.io/metadata.name": "claw-system", }, }, PodSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": "vpn-gateway", }, }, }, }, }, // dev-infra namespace (cache services) { To: []networkingv1.NetworkPolicyPeer{ { NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "kubernetes.io/metadata.name": "dev-infra", }, }, }, }, }, // DNS resolution via kube-system { To: []networkingv1.NetworkPolicyPeer{ { NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "kubernetes.io/metadata.name": "kube-system", }, }, }, }, Ports: []networkingv1.NetworkPolicyPort{ {Protocol: protocolPtr(corev1.ProtocolUDP), Port: &dnsPort}, {Protocol: protocolPtr(corev1.ProtocolTCP), Port: &dnsPort}, }, }, }, }, } } // AIProxyConfigMapTemplate returns the ai-proxy-config ConfigMap matching ai-proxy-config.yaml. func AIProxyConfigMapTemplate(user string) *corev1.ConfigMap { ns := model.NamespaceName(user) nginxConfTemplate := `events { worker_connections 64; } http { resolver %%RESOLVER%% valid=30s; server { listen 80; server_name anthropic.internal; client_max_body_size 50m; location / { set $upstream https://anthropic.baseroute.tech; proxy_pass $upstream; proxy_ssl_server_name on; proxy_set_header Host anthropic.baseroute.tech; proxy_set_header x-api-key "%%ANTHROPIC_KEY%%"; proxy_set_header Authorization ""; proxy_buffering off; proxy_read_timeout 300s; proxy_connect_timeout 10s; } } server { listen 80; server_name openai.internal; client_max_body_size 50m; location / { set $upstream https://openai.baseroute.tech; proxy_pass $upstream; proxy_ssl_server_name on; proxy_set_header Host openai.baseroute.tech; proxy_set_header Authorization "Bearer %%OPENAI_KEY%%"; proxy_set_header x-api-key ""; proxy_buffering off; proxy_read_timeout 300s; proxy_connect_timeout 10s; } } } ` entrypointSh := `#!/bin/sh set -e # Escape sed special chars (& \ |) in secret values to prevent replacement corruption escape_sed() { printf '%s\n' "$1" | sed 's/[&\|/]/\\&/g'; } ANTHROPIC_KEY=$(escape_sed "$(cat /secrets/anthropic-key)") OPENAI_KEY=$(escape_sed "$(cat /secrets/openai-key)") RESOLVER=$(awk '/^nameserver/{print $2; exit}' /etc/resolv.conf) sed \ -e "s|%%ANTHROPIC_KEY%%|${ANTHROPIC_KEY}|g" \ -e "s|%%OPENAI_KEY%%|${OPENAI_KEY}|g" \ -e "s|%%RESOLVER%%|${RESOLVER}|g" \ /etc/nginx/templates/nginx.conf.template > /etc/nginx/nginx.conf exec nginx -g 'daemon off;' ` return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "ai-proxy-config", Namespace: ns, }, Data: map[string]string{ "nginx.conf.template": nginxConfTemplate, "entrypoint.sh": entrypointSh, }, } } func boolPtr(b bool) *bool { return &b } func int32Ptr(i int32) *int32 { return &i } func protocolPtr(p corev1.Protocol) *corev1.Protocol { return &p }