dev-pod-api-build/internal/k8s/templates.go
2026-04-16 04:16:36 +00:00

599 lines
17 KiB
Go

package k8s
import (
"fmt"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/intstr"
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
)
// PodOpts holds all parameters needed to create a dev pod and its resources.
type PodOpts struct {
User string
Pod string
Tools string
Task string
CPUReq string
CPULimit string
MemReq string
MemLimit string
VPNKey string
AnthropicKey string
OpenAIKey string
ForgejoToken string
TailscaleKey string
}
// PVCName returns the per-pod PVC name for a given pod.
func PVCName(pod string) string {
return fmt.Sprintf("workspace-%s", pod)
}
// PVCTemplate returns a PersistentVolumeClaim for a specific pod's workspace.
func PVCTemplate(user, pod string) *corev1.PersistentVolumeClaim {
storageClass := "longhorn"
return &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: PVCName(pod),
Namespace: model.NamespaceName(user),
Labels: map[string]string{
"app": "dev-pod",
"podname": pod,
},
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
StorageClassName: &storageClass,
Resources: corev1.VolumeResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: resource.MustParse("20Gi"),
},
},
},
}
}
// PodTemplate returns a Pod spec matching the YAML pod-template.yaml.
func PodTemplate(cfg Config, opts PodOpts) *corev1.Pod {
ns := model.NamespaceName(opts.User)
podName := model.PodFullName(opts.Pod)
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: ns,
Labels: map[string]string{
"app": "dev-pod",
"podname": opts.Pod,
},
},
Spec: corev1.PodSpec{
HostAliases: []corev1.HostAlias{
{
IP: "127.0.0.1",
Hostnames: []string{"anthropic.internal", "openai.internal"},
},
},
Containers: []corev1.Container{
devContainer(cfg, opts),
aiProxyContainer(),
ipipSidecarContainer(cfg, opts),
},
Volumes: []corev1.Volume{
{
Name: "workspace",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: PVCName(opts.Pod),
},
},
},
{
Name: "ai-proxy-config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "ai-proxy-config",
},
DefaultMode: int32Ptr(0755),
},
},
},
{
Name: "ai-proxy-secrets",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "ai-proxy-secrets",
},
},
},
},
},
}
}
func devContainer(cfg Config, opts PodOpts) corev1.Container {
return corev1.Container{
Name: "dev",
Image: fmt.Sprintf("%s/%s", cfg.Registry, cfg.GoldenImage),
ImagePullPolicy: corev1.PullAlways,
Ports: []corev1.ContainerPort{
{ContainerPort: 7681, Name: "ttyd"},
{ContainerPort: 22, Name: "ssh"},
{ContainerPort: 3000, Name: "forgejo"},
{ContainerPort: 8080, Name: "vscode"},
{ContainerPort: 8090, Name: "ralphex-gerrit"},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(opts.CPUReq),
corev1.ResourceMemory: resource.MustParse(opts.MemReq),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(opts.CPULimit),
corev1.ResourceMemory: resource.MustParse(opts.MemLimit),
},
},
EnvFrom: []corev1.EnvFromSource{
{
SecretRef: &corev1.SecretEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: "dev-secrets",
},
},
},
},
Env: []corev1.EnvVar{
{Name: "TASK_DESCRIPTION", Value: opts.Task},
{Name: "DEV_TOOLS", Value: opts.Tools},
{Name: "DEV_BASE_PATH", Value: fmt.Sprintf("/@%s/%s", opts.User, opts.Pod)},
{Name: "DEV_EXTERNAL_HOST", Value: cfg.Domain},
{Name: "TTYD_BASE_PATH", Value: fmt.Sprintf("/@%s/%s", opts.User, opts.Pod)},
{Name: "FORGEJO_URL", Value: cfg.ForgejoURL},
},
VolumeMounts: []corev1.VolumeMount{
{Name: "workspace", MountPath: "/home/dev/workspace"},
},
}
}
func aiProxyContainer() corev1.Container {
return corev1.Container{
Name: "ai-proxy",
Image: "nginx:alpine",
Command: []string{"/bin/sh", "/etc/nginx/templates/entrypoint.sh"},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("16Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("32Mi"),
},
},
VolumeMounts: []corev1.VolumeMount{
{Name: "ai-proxy-config", MountPath: "/etc/nginx/templates", ReadOnly: true},
{Name: "ai-proxy-secrets", MountPath: "/secrets", ReadOnly: true},
},
}
}
func ipipSidecarContainer(cfg Config, opts PodOpts) corev1.Container {
return corev1.Container{
Name: "ipip-sidecar",
Image: fmt.Sprintf("%s/claw-ipip-tunnel:dev", cfg.Registry),
ImagePullPolicy: corev1.PullAlways,
SecurityContext: &corev1.SecurityContext{
Privileged: boolPtr(true),
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"NET_ADMIN"},
},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
corev1.ResourceMemory: resource.MustParse("8Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("32Mi"),
},
},
Env: []corev1.EnvVar{
{Name: "POD_ID", Value: fmt.Sprintf("dev-%s-%s", opts.User, opts.Pod)},
{Name: "VPN_GATEWAY_HOST", Value: fmt.Sprintf("vpn-gateway.%s.svc", cfg.VPNGatewayNS)},
{
Name: "VPN_GATEWAY_KEY",
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "dev-secrets"},
Key: "VPN_GATEWAY_KEY",
},
},
},
},
}
}
// ServiceTemplate returns a Service for the dev pod matching service.yaml.
func ServiceTemplate(user, pod string) *corev1.Service {
ns := model.NamespaceName(user)
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: model.ServiceName(pod),
Namespace: ns,
Labels: map[string]string{
"app": "dev-pod",
"podname": pod,
},
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Ports: []corev1.ServicePort{
{Name: "ttyd", Port: 7681, TargetPort: intstr.FromInt32(7681), Protocol: corev1.ProtocolTCP},
{Name: "ssh", Port: 22, TargetPort: intstr.FromInt32(22), Protocol: corev1.ProtocolTCP},
{Name: "forgejo", Port: 3000, TargetPort: intstr.FromInt32(3000), Protocol: corev1.ProtocolTCP},
{Name: "vscode", Port: 8080, TargetPort: intstr.FromInt32(8080), Protocol: corev1.ProtocolTCP},
{Name: "ralphex-gerrit", Port: 8090, TargetPort: intstr.FromInt32(8090), Protocol: corev1.ProtocolTCP},
},
Selector: map[string]string{
"app": "dev-pod",
"podname": pod,
},
},
}
}
// IngressTemplate returns Traefik IngressRoute as an unstructured object matching ingress.yaml.
// Returns the IngressRoute and two Middleware objects.
func IngressTemplate(user, pod, domain string) []*unstructured.Unstructured {
ns := model.NamespaceName(user)
svcName := model.ServiceName(pod)
ingressRoute := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "traefik.io/v1alpha1",
"kind": "IngressRoute",
"metadata": map[string]interface{}{
"name": fmt.Sprintf("dev-pod-%s-ingress", pod),
"namespace": ns,
},
"spec": map[string]interface{}{
"entryPoints": []interface{}{"web", "websecure"},
// No TLS section: Caddy terminates TLS and forwards HTTP to Traefik.
// With TLS here, Traefik v3 only routes on websecure, breaking Caddy -> port 80 flow.
// No explicit priority: Traefik v3 auto-calculates from rule length,
// ensuring these beat the OpenClaw catch-all (Host:* / PathPrefix:/).
"routes": []interface{}{
// ttyd — base route
map[string]interface{}{
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/`)", domain, user, pod),
"kind": "Rule",
"middlewares": []interface{}{
map[string]interface{}{"name": "spinoff-basic-auth"},
},
"services": []interface{}{
map[string]interface{}{"name": svcName, "port": int64(7681)},
},
},
// Forgejo
map[string]interface{}{
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/forgejo/`)", domain, user, pod),
"kind": "Rule",
"middlewares": []interface{}{
map[string]interface{}{"name": "spinoff-basic-auth"},
},
"services": []interface{}{
map[string]interface{}{"name": svcName, "port": int64(3000)},
},
},
// VS Code
map[string]interface{}{
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/vscode/`)", domain, user, pod),
"kind": "Rule",
"middlewares": []interface{}{
map[string]interface{}{"name": "spinoff-basic-auth"},
},
"services": []interface{}{
map[string]interface{}{"name": svcName, "port": int64(8080)},
},
},
// Gerrit code review (handles own base path, shares port 8090 with ralphex)
map[string]interface{}{
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/gerrit/`)", domain, user, pod),
"kind": "Rule",
"middlewares": []interface{}{
map[string]interface{}{"name": "spinoff-basic-auth"},
},
"services": []interface{}{
map[string]interface{}{"name": svcName, "port": int64(8090)},
},
},
// Ralphex dashboard (with prefix stripping)
map[string]interface{}{
"match": fmt.Sprintf("Host(`%s`) && PathPrefix(`/@%s/%s/ralphex/`)", domain, user, pod),
"kind": "Rule",
"middlewares": []interface{}{
map[string]interface{}{"name": "spinoff-basic-auth"},
map[string]interface{}{"name": fmt.Sprintf("strip-dev-%s-ralphex-prefix", pod)},
},
"services": []interface{}{
map[string]interface{}{"name": svcName, "port": int64(8090)},
},
},
},
},
},
}
basicAuthMiddleware := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "traefik.io/v1alpha1",
"kind": "Middleware",
"metadata": map[string]interface{}{
"name": "spinoff-basic-auth",
"namespace": ns,
},
"spec": map[string]interface{}{
"basicAuth": map[string]interface{}{
"secret": "spinoff-basic-auth",
},
},
},
}
stripPrefixMiddleware := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "traefik.io/v1alpha1",
"kind": "Middleware",
"metadata": map[string]interface{}{
"name": fmt.Sprintf("strip-dev-%s-ralphex-prefix", pod),
"namespace": ns,
},
"spec": map[string]interface{}{
"stripPrefix": map[string]interface{}{
"prefixes": []interface{}{
fmt.Sprintf("/@%s/%s/ralphex", user, pod),
},
},
},
},
}
return []*unstructured.Unstructured{ingressRoute, basicAuthMiddleware, stripPrefixMiddleware}
}
// SecretsTemplate returns dev-secrets and ai-proxy-secrets matching secrets-template.yaml.
func SecretsTemplate(user, vpnKey, anthropicKey, openaiKey, forgejoToken, tailscaleKey string) (*corev1.Secret, *corev1.Secret) {
ns := model.NamespaceName(user)
stringData := map[string]string{
"ANTHROPIC_API_KEY": "sk-devpod",
"ANTHROPIC_BASE_URL": "http://anthropic.internal",
"BASEROUTE_OPENAI_KEY": "sk-devpod",
"VPN_GATEWAY_KEY": vpnKey,
"FORGEJO_TOKEN": forgejoToken,
}
if tailscaleKey != "" {
stringData["TAILSCALE_AUTHKEY"] = tailscaleKey
}
devSecrets := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "dev-secrets",
Namespace: ns,
},
Type: corev1.SecretTypeOpaque,
StringData: stringData,
}
aiProxySecrets := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "ai-proxy-secrets",
Namespace: ns,
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{
"anthropic-key": anthropicKey,
"openai-key": openaiKey,
},
}
return devSecrets, aiProxySecrets
}
// BasicAuthSecretTemplate returns the spinoff-basic-auth Secret containing htpasswd data.
// Matches the secret created by dev-pod-create.sh for Traefik basicAuth middleware.
func BasicAuthSecretTemplate(user string) *corev1.Secret {
ns := model.NamespaceName(user)
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "spinoff-basic-auth",
Namespace: ns,
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{
"users": "admin:$2y$05$i2Wxz4GpO8.jnMXWk1len.xcP0.wPSL.ozfd/OgX8BCo9chO8F2WO\n",
},
}
}
// NetworkPolicyTemplate returns a NetworkPolicy matching network-policy.yaml.
func NetworkPolicyTemplate(user string) *networkingv1.NetworkPolicy {
ns := model.NamespaceName(user)
dnsPort := intstr.FromInt32(53)
return &networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: "dev-pod-network-policy",
Namespace: ns,
},
Spec: networkingv1.NetworkPolicySpec{
PodSelector: metav1.LabelSelector{},
PolicyTypes: []networkingv1.PolicyType{
networkingv1.PolicyTypeIngress,
networkingv1.PolicyTypeEgress,
},
Ingress: []networkingv1.NetworkPolicyIngressRule{
{
From: []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"kubernetes.io/metadata.name": "kube-system",
},
},
},
},
},
},
Egress: []networkingv1.NetworkPolicyEgressRule{
// VPN gateway in claw-system
{
To: []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"kubernetes.io/metadata.name": "claw-system",
},
},
PodSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "vpn-gateway",
},
},
},
},
},
// dev-infra namespace (cache services)
{
To: []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"kubernetes.io/metadata.name": "dev-infra",
},
},
},
},
},
// DNS resolution via kube-system
{
To: []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"kubernetes.io/metadata.name": "kube-system",
},
},
},
},
Ports: []networkingv1.NetworkPolicyPort{
{Protocol: protocolPtr(corev1.ProtocolUDP), Port: &dnsPort},
{Protocol: protocolPtr(corev1.ProtocolTCP), Port: &dnsPort},
},
},
},
},
}
}
// AIProxyConfigMapTemplate returns the ai-proxy-config ConfigMap matching ai-proxy-config.yaml.
func AIProxyConfigMapTemplate(user string) *corev1.ConfigMap {
ns := model.NamespaceName(user)
nginxConfTemplate := `events {
worker_connections 64;
}
http {
resolver %%RESOLVER%% valid=30s;
server {
listen 80;
server_name anthropic.internal;
client_max_body_size 50m;
location / {
set $upstream https://anthropic.baseroute.tech;
proxy_pass $upstream;
proxy_ssl_server_name on;
proxy_set_header Host anthropic.baseroute.tech;
proxy_set_header x-api-key "%%ANTHROPIC_KEY%%";
proxy_set_header Authorization "";
proxy_buffering off;
proxy_read_timeout 300s;
proxy_connect_timeout 10s;
}
}
server {
listen 80;
server_name openai.internal;
client_max_body_size 50m;
location / {
set $upstream https://openai.baseroute.tech;
proxy_pass $upstream;
proxy_ssl_server_name on;
proxy_set_header Host openai.baseroute.tech;
proxy_set_header Authorization "Bearer %%OPENAI_KEY%%";
proxy_set_header x-api-key "";
proxy_buffering off;
proxy_read_timeout 300s;
proxy_connect_timeout 10s;
}
}
}
`
entrypointSh := `#!/bin/sh
set -e
# Escape sed special chars (& \ |) in secret values to prevent replacement corruption
escape_sed() { printf '%s\n' "$1" | sed 's/[&\|/]/\\&/g'; }
ANTHROPIC_KEY=$(escape_sed "$(cat /secrets/anthropic-key)")
OPENAI_KEY=$(escape_sed "$(cat /secrets/openai-key)")
RESOLVER=$(awk '/^nameserver/{print $2; exit}' /etc/resolv.conf)
sed \
-e "s|%%ANTHROPIC_KEY%%|${ANTHROPIC_KEY}|g" \
-e "s|%%OPENAI_KEY%%|${OPENAI_KEY}|g" \
-e "s|%%RESOLVER%%|${RESOLVER}|g" \
/etc/nginx/templates/nginx.conf.template > /etc/nginx/nginx.conf
exec nginx -g 'daemon off;'
`
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "ai-proxy-config",
Namespace: ns,
},
Data: map[string]string{
"nginx.conf.template": nginxConfTemplate,
"entrypoint.sh": entrypointSh,
},
}
}
func boolPtr(b bool) *bool {
return &b
}
func int32Ptr(i int32) *int32 {
return &i
}
func protocolPtr(p corev1.Protocol) *corev1.Protocol {
return &p
}