433 lines
14 KiB
Go
433 lines
14 KiB
Go
package k8s
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
netv1 "k8s.io/api/networking/v1"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
|
|
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
|
|
)
|
|
|
|
// SessionHostSpec captures the defaults + per-install knobs. The values
|
|
// are chosen to match web-tui's k8s/session-host manifests.
|
|
type SessionHostSpec struct {
|
|
// Image is the dev-container image (golden image). Defaults to the
|
|
// value baked into the ConfigFromEnv loader.
|
|
Image string
|
|
// PortWatchImage is the sidecar image built from web-tui/cmd/port-watch.
|
|
PortWatchImage string
|
|
// ApexDomain is the root domain used to form per-user hostnames
|
|
// (e.g. "spinoff.dev" → "*.alice.spinoff.dev").
|
|
ApexDomain string
|
|
// ClusterIssuer is the cert-manager ClusterIssuer used to obtain the
|
|
// per-user wildcard cert via DNS-01 (e.g. "letsencrypt-cloudflare-dns01").
|
|
ClusterIssuer string
|
|
// GatewayService is the in-cluster Service name that the per-user
|
|
// ingress forwards <port>.<user>.<apex>/* traffic to.
|
|
GatewayService string
|
|
// GatewayServiceNamespace is where GatewayService lives (e.g. "web-tui").
|
|
GatewayServiceNamespace string
|
|
}
|
|
|
|
// EnsureSessionHost is idempotent: guarantees the user has a
|
|
// long-lived session-host pod + supporting PVCs, Service, Ingress, and
|
|
// per-user wildcard cert. Re-running on an already-provisioned user is
|
|
// a no-op that reports current status.
|
|
func (c *Client) EnsureSessionHost(ctx context.Context, user string, spec SessionHostSpec) (SessionHostStatus, error) {
|
|
ns := model.NamespaceName(user)
|
|
|
|
if err := c.EnsureNamespace(ctx, user); err != nil {
|
|
return SessionHostStatus{}, err
|
|
}
|
|
|
|
if err := c.ensureSessionHostPVCs(ctx, ns); err != nil {
|
|
return SessionHostStatus{}, err
|
|
}
|
|
|
|
if err := c.ensureSessionHostServiceAccount(ctx, ns); err != nil {
|
|
return SessionHostStatus{}, err
|
|
}
|
|
|
|
pod, err := c.ensureSessionHostPod(ctx, ns, user, spec)
|
|
if err != nil {
|
|
return SessionHostStatus{}, err
|
|
}
|
|
|
|
if err := c.ensureSessionHostService(ctx, ns); err != nil {
|
|
return SessionHostStatus{}, err
|
|
}
|
|
|
|
if err := c.ensureSessionHostCertificate(ctx, ns, user, spec); err != nil {
|
|
return SessionHostStatus{}, err
|
|
}
|
|
|
|
if err := c.ensureSessionHostIngress(ctx, ns, user, spec); err != nil {
|
|
return SessionHostStatus{}, err
|
|
}
|
|
|
|
return c.readSessionHostStatus(ctx, ns, user, pod)
|
|
}
|
|
|
|
// SessionHostStatusForUser returns the current status of the user's
|
|
// session-host pod without creating anything new.
|
|
func (c *Client) SessionHostStatusForUser(ctx context.Context, user string) (SessionHostStatus, error) {
|
|
ns := model.NamespaceName(user)
|
|
pod, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, "session-host", metav1.GetOptions{})
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
return SessionHostStatus{Ready: false}, nil
|
|
}
|
|
return SessionHostStatus{}, fmt.Errorf("get session-host pod: %w", err)
|
|
}
|
|
return c.readSessionHostStatus(ctx, ns, user, pod)
|
|
}
|
|
|
|
// ---- helpers ----
|
|
|
|
const (
|
|
sessionHostPodName = "session-host"
|
|
sessionHostServiceName = "session-host"
|
|
sessionHostSA = "session-host"
|
|
workspacePVC = "workspace"
|
|
dotfilesPVC = "dotfiles"
|
|
)
|
|
|
|
func (c *Client) ensureSessionHostPVCs(ctx context.Context, ns string) error {
|
|
pvcs := []struct {
|
|
name string
|
|
size string
|
|
}{
|
|
{workspacePVC, "20Gi"},
|
|
{dotfilesPVC, "1Gi"},
|
|
}
|
|
for _, p := range pvcs {
|
|
want := &corev1.PersistentVolumeClaim{
|
|
ObjectMeta: metav1.ObjectMeta{Name: p.name},
|
|
Spec: corev1.PersistentVolumeClaimSpec{
|
|
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
|
|
Resources: corev1.VolumeResourceRequirements{
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceStorage: resource.MustParse(p.size),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
_, err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Create(ctx, want, metav1.CreateOptions{})
|
|
if err != nil && !errors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("create pvc %s/%s: %w", ns, p.name, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *Client) ensureSessionHostServiceAccount(ctx context.Context, ns string) error {
|
|
sa := &corev1.ServiceAccount{
|
|
ObjectMeta: metav1.ObjectMeta{Name: sessionHostSA},
|
|
}
|
|
_, err := c.Clientset.CoreV1().ServiceAccounts(ns).Create(ctx, sa, metav1.CreateOptions{})
|
|
if err != nil && !errors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("create sa %s/%s: %w", ns, sessionHostSA, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *Client) ensureSessionHostPod(ctx context.Context, ns, user string, spec SessionHostSpec) (*corev1.Pod, error) {
|
|
existing, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, sessionHostPodName, metav1.GetOptions{})
|
|
if err == nil {
|
|
return existing, nil
|
|
}
|
|
if !errors.IsNotFound(err) {
|
|
return nil, fmt.Errorf("get pod %s/%s: %w", ns, sessionHostPodName, err)
|
|
}
|
|
pod := sessionHostPodManifest(user, spec)
|
|
pod, err = c.Clientset.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{})
|
|
if err != nil {
|
|
if errors.IsAlreadyExists(err) {
|
|
return c.Clientset.CoreV1().Pods(ns).Get(ctx, sessionHostPodName, metav1.GetOptions{})
|
|
}
|
|
return nil, fmt.Errorf("create pod %s/%s: %w", ns, sessionHostPodName, err)
|
|
}
|
|
return pod, nil
|
|
}
|
|
|
|
func sessionHostPodManifest(user string, spec SessionHostSpec) *corev1.Pod {
|
|
one := int64(1000)
|
|
return &corev1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: sessionHostPodName,
|
|
Labels: map[string]string{
|
|
"app.kubernetes.io/name": "session-host",
|
|
"app.kubernetes.io/component": "dev-pod",
|
|
"web-tui.spinoff.dev/user": user,
|
|
"managed-by": "dev-pod-api",
|
|
},
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
RestartPolicy: corev1.RestartPolicyAlways,
|
|
ServiceAccountName: sessionHostSA,
|
|
SecurityContext: &corev1.PodSecurityContext{FSGroup: &one},
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "dev",
|
|
Image: spec.Image,
|
|
ImagePullPolicy: corev1.PullIfNotPresent,
|
|
Command: []string{"/usr/local/bin/dev-entrypoint"},
|
|
TTY: true,
|
|
Stdin: true,
|
|
Env: []corev1.EnvVar{
|
|
{Name: "HOME", Value: "/home/dev"},
|
|
{Name: "USER", Value: "dev"},
|
|
{Name: "WEBTUI_USER", Value: user},
|
|
},
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{Name: "workspace", MountPath: "/home/dev/workspace"},
|
|
{Name: "dotfiles", MountPath: "/home/dev/.config"},
|
|
{Name: "tmp", MountPath: "/tmp"},
|
|
},
|
|
Resources: corev1.ResourceRequirements{
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("250m"),
|
|
corev1.ResourceMemory: resource.MustParse("256Mi"),
|
|
},
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("1"),
|
|
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "port-watch",
|
|
Image: spec.PortWatchImage,
|
|
ImagePullPolicy: corev1.PullIfNotPresent,
|
|
Args: []string{"-listen=:9100", "-uploads=/home/dev/uploads"},
|
|
Ports: []corev1.ContainerPort{
|
|
{ContainerPort: 9100, Name: "portwatch"},
|
|
},
|
|
ReadinessProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromInt32(9100)},
|
|
},
|
|
PeriodSeconds: 5,
|
|
},
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{Name: "workspace", MountPath: "/home/dev/workspace", ReadOnly: true},
|
|
},
|
|
Resources: corev1.ResourceRequirements{
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("20m"),
|
|
corev1.ResourceMemory: resource.MustParse("32Mi"),
|
|
},
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("100m"),
|
|
corev1.ResourceMemory: resource.MustParse("64Mi"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []corev1.Volume{
|
|
{Name: "workspace", VolumeSource: corev1.VolumeSource{
|
|
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: workspacePVC}}},
|
|
{Name: "dotfiles", VolumeSource: corev1.VolumeSource{
|
|
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: dotfilesPVC}}},
|
|
{Name: "tmp", VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: ptrQuantity("1Gi")}}},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (c *Client) ensureSessionHostService(ctx context.Context, ns string) error {
|
|
svc := &corev1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{Name: sessionHostServiceName},
|
|
Spec: corev1.ServiceSpec{
|
|
Selector: map[string]string{"app.kubernetes.io/name": "session-host"},
|
|
Ports: []corev1.ServicePort{
|
|
{Name: "portwatch", Port: 9100, TargetPort: intstr.FromInt32(9100)},
|
|
},
|
|
},
|
|
}
|
|
_, err := c.Clientset.CoreV1().Services(ns).Create(ctx, svc, metav1.CreateOptions{})
|
|
if err != nil && !errors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("create svc: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *Client) ensureSessionHostCertificate(ctx context.Context, ns, user string, spec SessionHostSpec) error {
|
|
name := "wildcard-" + user
|
|
gvr := schema.GroupVersionResource{Group: "cert-manager.io", Version: "v1", Resource: "certificates"}
|
|
wildcardHost := "*." + user + "." + spec.ApexDomain
|
|
obj := &unstructured.Unstructured{Object: map[string]any{
|
|
"apiVersion": "cert-manager.io/v1",
|
|
"kind": "Certificate",
|
|
"metadata": map[string]any{
|
|
"name": name,
|
|
},
|
|
"spec": map[string]any{
|
|
"secretName": name + "-tls",
|
|
"dnsNames": []string{wildcardHost},
|
|
"duration": "2160h",
|
|
"renewBefore": "720h",
|
|
"privateKey": map[string]any{"algorithm": "ECDSA", "size": int64(256)},
|
|
"issuerRef": map[string]any{
|
|
"name": spec.ClusterIssuer,
|
|
"kind": "ClusterIssuer",
|
|
"group": "cert-manager.io",
|
|
},
|
|
},
|
|
}}
|
|
_, err := c.Dynamic.Resource(gvr).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{})
|
|
if err != nil && !errors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("create certificate: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *Client) ensureSessionHostIngress(ctx context.Context, ns, user string, spec SessionHostSpec) error {
|
|
host := "*." + user + "." + spec.ApexDomain
|
|
pathType := netv1.PathTypePrefix
|
|
className := "traefik"
|
|
ing := &netv1.Ingress{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "preview-" + user,
|
|
Annotations: map[string]string{
|
|
"cert-manager.io/cluster-issuer": spec.ClusterIssuer,
|
|
"traefik.ingress.kubernetes.io/router.tls": "true",
|
|
},
|
|
},
|
|
Spec: netv1.IngressSpec{
|
|
IngressClassName: &className,
|
|
TLS: []netv1.IngressTLS{
|
|
{Hosts: []string{host}, SecretName: "wildcard-" + user + "-tls"},
|
|
},
|
|
Rules: []netv1.IngressRule{
|
|
{
|
|
Host: host,
|
|
IngressRuleValue: netv1.IngressRuleValue{
|
|
HTTP: &netv1.HTTPIngressRuleValue{
|
|
Paths: []netv1.HTTPIngressPath{
|
|
{
|
|
Path: "/",
|
|
PathType: &pathType,
|
|
Backend: netv1.IngressBackend{
|
|
Service: &netv1.IngressServiceBackend{
|
|
Name: spec.GatewayService,
|
|
Port: netv1.ServiceBackendPort{Name: "http"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
_, err := c.Clientset.NetworkingV1().Ingresses(ns).Create(ctx, ing, metav1.CreateOptions{})
|
|
if err != nil && !errors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("create ingress: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *Client) readSessionHostStatus(ctx context.Context, ns, user string, pod *corev1.Pod) (SessionHostStatus, error) {
|
|
ready := pod.Status.Phase == corev1.PodRunning && allContainersReady(pod)
|
|
certReady, certReason := c.readCertStatus(ctx, ns, "wildcard-"+user)
|
|
return SessionHostStatus{
|
|
Ready: ready,
|
|
PodName: pod.Name,
|
|
Namespace: ns,
|
|
CertReady: certReady,
|
|
CertPendingReason: certReason,
|
|
CreatedAt: pod.CreationTimestamp.Time,
|
|
}, nil
|
|
}
|
|
|
|
func (c *Client) readCertStatus(ctx context.Context, ns, name string) (bool, string) {
|
|
gvr := schema.GroupVersionResource{Group: "cert-manager.io", Version: "v1", Resource: "certificates"}
|
|
obj, err := c.Dynamic.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return false, "not-found"
|
|
}
|
|
conds, ok, _ := unstructuredNested(obj.Object, "status", "conditions")
|
|
if !ok {
|
|
return false, "no-status"
|
|
}
|
|
list, ok := conds.([]any)
|
|
if !ok {
|
|
return false, "bad-status"
|
|
}
|
|
for _, c := range list {
|
|
m, ok := c.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
if m["type"] == "Ready" {
|
|
if m["status"] == "True" {
|
|
return true, ""
|
|
}
|
|
if reason, ok := m["reason"].(string); ok {
|
|
return false, reason
|
|
}
|
|
return false, "not-ready"
|
|
}
|
|
}
|
|
return false, "pending"
|
|
}
|
|
|
|
// SessionHostStatus is duplicated here so the k8s package doesn't need
|
|
// to import the api package (avoids an import cycle). The api package
|
|
// mirrors the fields.
|
|
type SessionHostStatus struct {
|
|
Ready bool
|
|
PodName string
|
|
Namespace string
|
|
CertReady bool
|
|
CertPendingReason string
|
|
AtchSessionCount int
|
|
CreatedAt time.Time
|
|
}
|
|
|
|
// ---- tiny helpers ----
|
|
|
|
func allContainersReady(pod *corev1.Pod) bool {
|
|
if len(pod.Status.ContainerStatuses) == 0 {
|
|
return false
|
|
}
|
|
for _, cs := range pod.Status.ContainerStatuses {
|
|
if !cs.Ready {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func unstructuredNested(obj map[string]any, keys ...string) (any, bool, error) {
|
|
cur := any(obj)
|
|
for _, k := range keys {
|
|
m, ok := cur.(map[string]any)
|
|
if !ok {
|
|
return nil, false, nil
|
|
}
|
|
v, ok := m[k]
|
|
if !ok {
|
|
return nil, false, nil
|
|
}
|
|
cur = v
|
|
}
|
|
return cur, true, nil
|
|
}
|
|
|
|
func ptrQuantity(s string) *resource.Quantity {
|
|
q := resource.MustParse(s)
|
|
return &q
|
|
}
|