package k8s import ( "context" "fmt" "time" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" "github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model" ) // SessionHostSpec captures the defaults + per-install knobs. The values // are chosen to match web-tui's k8s/session-host manifests. type SessionHostSpec struct { // Image is the dev-container image (golden image). Defaults to the // value baked into the ConfigFromEnv loader. Image string // PortWatchImage is the sidecar image built from web-tui/cmd/port-watch. PortWatchImage string // ApexDomain is the root domain used to form per-user hostnames // (e.g. "spinoff.dev" → "*.alice.spinoff.dev"). ApexDomain string // ClusterIssuer is the cert-manager ClusterIssuer used to obtain the // per-user wildcard cert via DNS-01 (e.g. "letsencrypt-cloudflare-dns01"). ClusterIssuer string // GatewayService is the in-cluster Service name that the per-user // ingress forwards ../* traffic to. GatewayService string // GatewayServiceNamespace is where GatewayService lives (e.g. "web-tui"). GatewayServiceNamespace string } // EnsureSessionHost is idempotent: guarantees the user has a // long-lived session-host pod + supporting PVCs, Service, Ingress, and // per-user wildcard cert. Re-running on an already-provisioned user is // a no-op that reports current status. func (c *Client) EnsureSessionHost(ctx context.Context, user string, spec SessionHostSpec) (SessionHostStatus, error) { ns := model.NamespaceName(user) if err := c.EnsureNamespace(ctx, user); err != nil { return SessionHostStatus{}, err } if err := c.ensureSessionHostPVCs(ctx, ns); err != nil { return SessionHostStatus{}, err } if err := c.ensureSessionHostServiceAccount(ctx, ns); err != nil { return SessionHostStatus{}, err } pod, err := c.ensureSessionHostPod(ctx, ns, user, spec) if err != nil { return SessionHostStatus{}, err } if err := c.ensureSessionHostService(ctx, ns); err != nil { return SessionHostStatus{}, err } if err := c.ensureSessionHostCertificate(ctx, ns, user, spec); err != nil { return SessionHostStatus{}, err } if err := c.ensureSessionHostIngress(ctx, ns, user, spec); err != nil { return SessionHostStatus{}, err } return c.readSessionHostStatus(ctx, ns, user, pod) } // SessionHostStatusForUser returns the current status of the user's // session-host pod without creating anything new. func (c *Client) SessionHostStatusForUser(ctx context.Context, user string) (SessionHostStatus, error) { ns := model.NamespaceName(user) pod, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, "session-host", metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { return SessionHostStatus{Ready: false}, nil } return SessionHostStatus{}, fmt.Errorf("get session-host pod: %w", err) } return c.readSessionHostStatus(ctx, ns, user, pod) } // ---- helpers ---- const ( sessionHostPodName = "session-host" sessionHostServiceName = "session-host" sessionHostSA = "session-host" workspacePVC = "workspace" dotfilesPVC = "dotfiles" ) func (c *Client) ensureSessionHostPVCs(ctx context.Context, ns string) error { pvcs := []struct { name string size string }{ {workspacePVC, "20Gi"}, {dotfilesPVC, "1Gi"}, } for _, p := range pvcs { want := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: p.name}, Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceStorage: resource.MustParse(p.size), }, }, }, } _, err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Create(ctx, want, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("create pvc %s/%s: %w", ns, p.name, err) } } return nil } func (c *Client) ensureSessionHostServiceAccount(ctx context.Context, ns string) error { sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{Name: sessionHostSA}, } _, err := c.Clientset.CoreV1().ServiceAccounts(ns).Create(ctx, sa, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("create sa %s/%s: %w", ns, sessionHostSA, err) } return nil } func (c *Client) ensureSessionHostPod(ctx context.Context, ns, user string, spec SessionHostSpec) (*corev1.Pod, error) { existing, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, sessionHostPodName, metav1.GetOptions{}) if err == nil { return existing, nil } if !errors.IsNotFound(err) { return nil, fmt.Errorf("get pod %s/%s: %w", ns, sessionHostPodName, err) } pod := sessionHostPodManifest(user, spec) pod, err = c.Clientset.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}) if err != nil { if errors.IsAlreadyExists(err) { return c.Clientset.CoreV1().Pods(ns).Get(ctx, sessionHostPodName, metav1.GetOptions{}) } return nil, fmt.Errorf("create pod %s/%s: %w", ns, sessionHostPodName, err) } return pod, nil } func sessionHostPodManifest(user string, spec SessionHostSpec) *corev1.Pod { one := int64(1000) return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: sessionHostPodName, Labels: map[string]string{ "app.kubernetes.io/name": "session-host", "app.kubernetes.io/component": "dev-pod", "web-tui.spinoff.dev/user": user, "managed-by": "dev-pod-api", }, }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyAlways, ServiceAccountName: sessionHostSA, SecurityContext: &corev1.PodSecurityContext{FSGroup: &one}, Containers: []corev1.Container{ { Name: "dev", Image: spec.Image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/usr/local/bin/dev-entrypoint"}, TTY: true, Stdin: true, Env: []corev1.EnvVar{ {Name: "HOME", Value: "/home/dev"}, {Name: "USER", Value: "dev"}, {Name: "WEBTUI_USER", Value: user}, }, VolumeMounts: []corev1.VolumeMount{ {Name: "workspace", MountPath: "/home/dev/workspace"}, {Name: "dotfiles", MountPath: "/home/dev/.config"}, {Name: "tmp", MountPath: "/tmp"}, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("250m"), corev1.ResourceMemory: resource.MustParse("256Mi"), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), }, }, }, { Name: "port-watch", Image: spec.PortWatchImage, ImagePullPolicy: corev1.PullIfNotPresent, Args: []string{"-listen=:9100", "-uploads=/home/dev/uploads"}, Ports: []corev1.ContainerPort{ {ContainerPort: 9100, Name: "portwatch"}, }, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromInt32(9100)}, }, PeriodSeconds: 5, }, VolumeMounts: []corev1.VolumeMount{ {Name: "workspace", MountPath: "/home/dev/workspace", ReadOnly: true}, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("20m"), corev1.ResourceMemory: resource.MustParse("32Mi"), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("64Mi"), }, }, }, }, Volumes: []corev1.Volume{ {Name: "workspace", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: workspacePVC}}}, {Name: "dotfiles", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: dotfilesPVC}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: ptrQuantity("1Gi")}}}, }, }, } } func (c *Client) ensureSessionHostService(ctx context.Context, ns string) error { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Name: sessionHostServiceName}, Spec: corev1.ServiceSpec{ Selector: map[string]string{"app.kubernetes.io/name": "session-host"}, Ports: []corev1.ServicePort{ {Name: "portwatch", Port: 9100, TargetPort: intstr.FromInt32(9100)}, }, }, } _, err := c.Clientset.CoreV1().Services(ns).Create(ctx, svc, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("create svc: %w", err) } return nil } func (c *Client) ensureSessionHostCertificate(ctx context.Context, ns, user string, spec SessionHostSpec) error { name := "wildcard-" + user gvr := schema.GroupVersionResource{Group: "cert-manager.io", Version: "v1", Resource: "certificates"} wildcardHost := "*." + user + "." + spec.ApexDomain obj := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "cert-manager.io/v1", "kind": "Certificate", "metadata": map[string]any{ "name": name, }, "spec": map[string]any{ "secretName": name + "-tls", "dnsNames": []string{wildcardHost}, "duration": "2160h", "renewBefore": "720h", "privateKey": map[string]any{"algorithm": "ECDSA", "size": int64(256)}, "issuerRef": map[string]any{ "name": spec.ClusterIssuer, "kind": "ClusterIssuer", "group": "cert-manager.io", }, }, }} _, err := c.Dynamic.Resource(gvr).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("create certificate: %w", err) } return nil } func (c *Client) ensureSessionHostIngress(ctx context.Context, ns, user string, spec SessionHostSpec) error { host := "*." + user + "." + spec.ApexDomain pathType := netv1.PathTypePrefix className := "traefik" ing := &netv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ Name: "preview-" + user, Annotations: map[string]string{ "cert-manager.io/cluster-issuer": spec.ClusterIssuer, "traefik.ingress.kubernetes.io/router.tls": "true", }, }, Spec: netv1.IngressSpec{ IngressClassName: &className, TLS: []netv1.IngressTLS{ {Hosts: []string{host}, SecretName: "wildcard-" + user + "-tls"}, }, Rules: []netv1.IngressRule{ { Host: host, IngressRuleValue: netv1.IngressRuleValue{ HTTP: &netv1.HTTPIngressRuleValue{ Paths: []netv1.HTTPIngressPath{ { Path: "/", PathType: &pathType, Backend: netv1.IngressBackend{ Service: &netv1.IngressServiceBackend{ Name: spec.GatewayService, Port: netv1.ServiceBackendPort{Name: "http"}, }, }, }, }, }, }, }, }, }, } _, err := c.Clientset.NetworkingV1().Ingresses(ns).Create(ctx, ing, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("create ingress: %w", err) } return nil } func (c *Client) readSessionHostStatus(ctx context.Context, ns, user string, pod *corev1.Pod) (SessionHostStatus, error) { ready := pod.Status.Phase == corev1.PodRunning && allContainersReady(pod) certReady, certReason := c.readCertStatus(ctx, ns, "wildcard-"+user) return SessionHostStatus{ Ready: ready, PodName: pod.Name, Namespace: ns, CertReady: certReady, CertPendingReason: certReason, CreatedAt: pod.CreationTimestamp.Time, }, nil } func (c *Client) readCertStatus(ctx context.Context, ns, name string) (bool, string) { gvr := schema.GroupVersionResource{Group: "cert-manager.io", Version: "v1", Resource: "certificates"} obj, err := c.Dynamic.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) if err != nil { return false, "not-found" } conds, ok, _ := unstructuredNested(obj.Object, "status", "conditions") if !ok { return false, "no-status" } list, ok := conds.([]any) if !ok { return false, "bad-status" } for _, c := range list { m, ok := c.(map[string]any) if !ok { continue } if m["type"] == "Ready" { if m["status"] == "True" { return true, "" } if reason, ok := m["reason"].(string); ok { return false, reason } return false, "not-ready" } } return false, "pending" } // SessionHostStatus is duplicated here so the k8s package doesn't need // to import the api package (avoids an import cycle). The api package // mirrors the fields. type SessionHostStatus struct { Ready bool PodName string Namespace string CertReady bool CertPendingReason string AtchSessionCount int CreatedAt time.Time } // ---- tiny helpers ---- func allContainersReady(pod *corev1.Pod) bool { if len(pod.Status.ContainerStatuses) == 0 { return false } for _, cs := range pod.Status.ContainerStatuses { if !cs.Ready { return false } } return true } func unstructuredNested(obj map[string]any, keys ...string) (any, bool, error) { cur := any(obj) for _, k := range keys { m, ok := cur.(map[string]any) if !ok { return nil, false, nil } v, ok := m[k] if !ok { return nil, false, nil } cur = v } return cur, true, nil } func ptrQuantity(s string) *resource.Quantity { q := resource.MustParse(s) return &q }