build source
This commit is contained in:
build
commit
ee1fec43ed
4171 changed files with 1351288 additions and 0 deletions
433
internal/k8s/sessionhost.go
Normal file
433
internal/k8s/sessionhost.go
Normal file
|
|
@ -0,0 +1,433 @@
|
|||
package k8s
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
netv1 "k8s.io/api/networking/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
|
||||
"github.com/iliaivanov/spec-kit-remote/cmd/dev-pod-api/internal/model"
|
||||
)
|
||||
|
||||
// SessionHostSpec captures the defaults + per-install knobs. The values
|
||||
// are chosen to match web-tui's k8s/session-host manifests.
|
||||
type SessionHostSpec struct {
|
||||
// Image is the dev-container image (golden image). Defaults to the
|
||||
// value baked into the ConfigFromEnv loader.
|
||||
Image string
|
||||
// PortWatchImage is the sidecar image built from web-tui/cmd/port-watch.
|
||||
PortWatchImage string
|
||||
// ApexDomain is the root domain used to form per-user hostnames
|
||||
// (e.g. "spinoff.dev" → "*.alice.spinoff.dev").
|
||||
ApexDomain string
|
||||
// ClusterIssuer is the cert-manager ClusterIssuer used to obtain the
|
||||
// per-user wildcard cert via DNS-01 (e.g. "letsencrypt-cloudflare-dns01").
|
||||
ClusterIssuer string
|
||||
// GatewayService is the in-cluster Service name that the per-user
|
||||
// ingress forwards <port>.<user>.<apex>/* traffic to.
|
||||
GatewayService string
|
||||
// GatewayServiceNamespace is where GatewayService lives (e.g. "web-tui").
|
||||
GatewayServiceNamespace string
|
||||
}
|
||||
|
||||
// EnsureSessionHost is idempotent: guarantees the user has a
|
||||
// long-lived session-host pod + supporting PVCs, Service, Ingress, and
|
||||
// per-user wildcard cert. Re-running on an already-provisioned user is
|
||||
// a no-op that reports current status.
|
||||
func (c *Client) EnsureSessionHost(ctx context.Context, user string, spec SessionHostSpec) (SessionHostStatus, error) {
|
||||
ns := model.NamespaceName(user)
|
||||
|
||||
if err := c.EnsureNamespace(ctx, user); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
if err := c.ensureSessionHostPVCs(ctx, ns); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
if err := c.ensureSessionHostServiceAccount(ctx, ns); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
pod, err := c.ensureSessionHostPod(ctx, ns, user, spec)
|
||||
if err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
if err := c.ensureSessionHostService(ctx, ns); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
if err := c.ensureSessionHostCertificate(ctx, ns, user, spec); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
if err := c.ensureSessionHostIngress(ctx, ns, user, spec); err != nil {
|
||||
return SessionHostStatus{}, err
|
||||
}
|
||||
|
||||
return c.readSessionHostStatus(ctx, ns, user, pod)
|
||||
}
|
||||
|
||||
// SessionHostStatusForUser returns the current status of the user's
|
||||
// session-host pod without creating anything new.
|
||||
func (c *Client) SessionHostStatusForUser(ctx context.Context, user string) (SessionHostStatus, error) {
|
||||
ns := model.NamespaceName(user)
|
||||
pod, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, "session-host", metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return SessionHostStatus{Ready: false}, nil
|
||||
}
|
||||
return SessionHostStatus{}, fmt.Errorf("get session-host pod: %w", err)
|
||||
}
|
||||
return c.readSessionHostStatus(ctx, ns, user, pod)
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
const (
|
||||
sessionHostPodName = "session-host"
|
||||
sessionHostServiceName = "session-host"
|
||||
sessionHostSA = "session-host"
|
||||
workspacePVC = "workspace"
|
||||
dotfilesPVC = "dotfiles"
|
||||
)
|
||||
|
||||
func (c *Client) ensureSessionHostPVCs(ctx context.Context, ns string) error {
|
||||
pvcs := []struct {
|
||||
name string
|
||||
size string
|
||||
}{
|
||||
{workspacePVC, "20Gi"},
|
||||
{dotfilesPVC, "1Gi"},
|
||||
}
|
||||
for _, p := range pvcs {
|
||||
want := &corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: p.name},
|
||||
Spec: corev1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
|
||||
Resources: corev1.VolumeResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceStorage: resource.MustParse(p.size),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := c.Clientset.CoreV1().PersistentVolumeClaims(ns).Create(ctx, want, metav1.CreateOptions{})
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create pvc %s/%s: %w", ns, p.name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) ensureSessionHostServiceAccount(ctx context.Context, ns string) error {
|
||||
sa := &corev1.ServiceAccount{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: sessionHostSA},
|
||||
}
|
||||
_, err := c.Clientset.CoreV1().ServiceAccounts(ns).Create(ctx, sa, metav1.CreateOptions{})
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create sa %s/%s: %w", ns, sessionHostSA, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) ensureSessionHostPod(ctx context.Context, ns, user string, spec SessionHostSpec) (*corev1.Pod, error) {
|
||||
existing, err := c.Clientset.CoreV1().Pods(ns).Get(ctx, sessionHostPodName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
return existing, nil
|
||||
}
|
||||
if !errors.IsNotFound(err) {
|
||||
return nil, fmt.Errorf("get pod %s/%s: %w", ns, sessionHostPodName, err)
|
||||
}
|
||||
pod := sessionHostPodManifest(user, spec)
|
||||
pod, err = c.Clientset.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
if errors.IsAlreadyExists(err) {
|
||||
return c.Clientset.CoreV1().Pods(ns).Get(ctx, sessionHostPodName, metav1.GetOptions{})
|
||||
}
|
||||
return nil, fmt.Errorf("create pod %s/%s: %w", ns, sessionHostPodName, err)
|
||||
}
|
||||
return pod, nil
|
||||
}
|
||||
|
||||
func sessionHostPodManifest(user string, spec SessionHostSpec) *corev1.Pod {
|
||||
one := int64(1000)
|
||||
return &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: sessionHostPodName,
|
||||
Labels: map[string]string{
|
||||
"app.kubernetes.io/name": "session-host",
|
||||
"app.kubernetes.io/component": "dev-pod",
|
||||
"web-tui.spinoff.dev/user": user,
|
||||
"managed-by": "dev-pod-api",
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: corev1.RestartPolicyAlways,
|
||||
ServiceAccountName: sessionHostSA,
|
||||
SecurityContext: &corev1.PodSecurityContext{FSGroup: &one},
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "dev",
|
||||
Image: spec.Image,
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
Command: []string{"/usr/local/bin/dev-entrypoint"},
|
||||
TTY: true,
|
||||
Stdin: true,
|
||||
Env: []corev1.EnvVar{
|
||||
{Name: "HOME", Value: "/home/dev"},
|
||||
{Name: "USER", Value: "dev"},
|
||||
{Name: "WEBTUI_USER", Value: user},
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{Name: "workspace", MountPath: "/home/dev/workspace"},
|
||||
{Name: "dotfiles", MountPath: "/home/dev/.config"},
|
||||
{Name: "tmp", MountPath: "/tmp"},
|
||||
},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("250m"),
|
||||
corev1.ResourceMemory: resource.MustParse("256Mi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "port-watch",
|
||||
Image: spec.PortWatchImage,
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
Args: []string{"-listen=:9100", "-uploads=/home/dev/uploads"},
|
||||
Ports: []corev1.ContainerPort{
|
||||
{ContainerPort: 9100, Name: "portwatch"},
|
||||
},
|
||||
ReadinessProbe: &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromInt32(9100)},
|
||||
},
|
||||
PeriodSeconds: 5,
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{Name: "workspace", MountPath: "/home/dev/workspace", ReadOnly: true},
|
||||
},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("20m"),
|
||||
corev1.ResourceMemory: resource.MustParse("32Mi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("100m"),
|
||||
corev1.ResourceMemory: resource.MustParse("64Mi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Volumes: []corev1.Volume{
|
||||
{Name: "workspace", VolumeSource: corev1.VolumeSource{
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: workspacePVC}}},
|
||||
{Name: "dotfiles", VolumeSource: corev1.VolumeSource{
|
||||
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: dotfilesPVC}}},
|
||||
{Name: "tmp", VolumeSource: corev1.VolumeSource{
|
||||
EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: ptrQuantity("1Gi")}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) ensureSessionHostService(ctx context.Context, ns string) error {
|
||||
svc := &corev1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: sessionHostServiceName},
|
||||
Spec: corev1.ServiceSpec{
|
||||
Selector: map[string]string{"app.kubernetes.io/name": "session-host"},
|
||||
Ports: []corev1.ServicePort{
|
||||
{Name: "portwatch", Port: 9100, TargetPort: intstr.FromInt32(9100)},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := c.Clientset.CoreV1().Services(ns).Create(ctx, svc, metav1.CreateOptions{})
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create svc: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) ensureSessionHostCertificate(ctx context.Context, ns, user string, spec SessionHostSpec) error {
|
||||
name := "wildcard-" + user
|
||||
gvr := schema.GroupVersionResource{Group: "cert-manager.io", Version: "v1", Resource: "certificates"}
|
||||
wildcardHost := "*." + user + "." + spec.ApexDomain
|
||||
obj := &unstructured.Unstructured{Object: map[string]any{
|
||||
"apiVersion": "cert-manager.io/v1",
|
||||
"kind": "Certificate",
|
||||
"metadata": map[string]any{
|
||||
"name": name,
|
||||
},
|
||||
"spec": map[string]any{
|
||||
"secretName": name + "-tls",
|
||||
"dnsNames": []string{wildcardHost},
|
||||
"duration": "2160h",
|
||||
"renewBefore": "720h",
|
||||
"privateKey": map[string]any{"algorithm": "ECDSA", "size": int64(256)},
|
||||
"issuerRef": map[string]any{
|
||||
"name": spec.ClusterIssuer,
|
||||
"kind": "ClusterIssuer",
|
||||
"group": "cert-manager.io",
|
||||
},
|
||||
},
|
||||
}}
|
||||
_, err := c.Dynamic.Resource(gvr).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{})
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create certificate: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) ensureSessionHostIngress(ctx context.Context, ns, user string, spec SessionHostSpec) error {
|
||||
host := "*." + user + "." + spec.ApexDomain
|
||||
pathType := netv1.PathTypePrefix
|
||||
className := "traefik"
|
||||
ing := &netv1.Ingress{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "preview-" + user,
|
||||
Annotations: map[string]string{
|
||||
"cert-manager.io/cluster-issuer": spec.ClusterIssuer,
|
||||
"traefik.ingress.kubernetes.io/router.tls": "true",
|
||||
},
|
||||
},
|
||||
Spec: netv1.IngressSpec{
|
||||
IngressClassName: &className,
|
||||
TLS: []netv1.IngressTLS{
|
||||
{Hosts: []string{host}, SecretName: "wildcard-" + user + "-tls"},
|
||||
},
|
||||
Rules: []netv1.IngressRule{
|
||||
{
|
||||
Host: host,
|
||||
IngressRuleValue: netv1.IngressRuleValue{
|
||||
HTTP: &netv1.HTTPIngressRuleValue{
|
||||
Paths: []netv1.HTTPIngressPath{
|
||||
{
|
||||
Path: "/",
|
||||
PathType: &pathType,
|
||||
Backend: netv1.IngressBackend{
|
||||
Service: &netv1.IngressServiceBackend{
|
||||
Name: spec.GatewayService,
|
||||
Port: netv1.ServiceBackendPort{Name: "http"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := c.Clientset.NetworkingV1().Ingresses(ns).Create(ctx, ing, metav1.CreateOptions{})
|
||||
if err != nil && !errors.IsAlreadyExists(err) {
|
||||
return fmt.Errorf("create ingress: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) readSessionHostStatus(ctx context.Context, ns, user string, pod *corev1.Pod) (SessionHostStatus, error) {
|
||||
ready := pod.Status.Phase == corev1.PodRunning && allContainersReady(pod)
|
||||
certReady, certReason := c.readCertStatus(ctx, ns, "wildcard-"+user)
|
||||
return SessionHostStatus{
|
||||
Ready: ready,
|
||||
PodName: pod.Name,
|
||||
Namespace: ns,
|
||||
CertReady: certReady,
|
||||
CertPendingReason: certReason,
|
||||
CreatedAt: pod.CreationTimestamp.Time,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) readCertStatus(ctx context.Context, ns, name string) (bool, string) {
|
||||
gvr := schema.GroupVersionResource{Group: "cert-manager.io", Version: "v1", Resource: "certificates"}
|
||||
obj, err := c.Dynamic.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return false, "not-found"
|
||||
}
|
||||
conds, ok, _ := unstructuredNested(obj.Object, "status", "conditions")
|
||||
if !ok {
|
||||
return false, "no-status"
|
||||
}
|
||||
list, ok := conds.([]any)
|
||||
if !ok {
|
||||
return false, "bad-status"
|
||||
}
|
||||
for _, c := range list {
|
||||
m, ok := c.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if m["type"] == "Ready" {
|
||||
if m["status"] == "True" {
|
||||
return true, ""
|
||||
}
|
||||
if reason, ok := m["reason"].(string); ok {
|
||||
return false, reason
|
||||
}
|
||||
return false, "not-ready"
|
||||
}
|
||||
}
|
||||
return false, "pending"
|
||||
}
|
||||
|
||||
// SessionHostStatus is duplicated here so the k8s package doesn't need
|
||||
// to import the api package (avoids an import cycle). The api package
|
||||
// mirrors the fields.
|
||||
type SessionHostStatus struct {
|
||||
Ready bool
|
||||
PodName string
|
||||
Namespace string
|
||||
CertReady bool
|
||||
CertPendingReason string
|
||||
AtchSessionCount int
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// ---- tiny helpers ----
|
||||
|
||||
func allContainersReady(pod *corev1.Pod) bool {
|
||||
if len(pod.Status.ContainerStatuses) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, cs := range pod.Status.ContainerStatuses {
|
||||
if !cs.Ready {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func unstructuredNested(obj map[string]any, keys ...string) (any, bool, error) {
|
||||
cur := any(obj)
|
||||
for _, k := range keys {
|
||||
m, ok := cur.(map[string]any)
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
v, ok := m[k]
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur = v
|
||||
}
|
||||
return cur, true, nil
|
||||
}
|
||||
|
||||
func ptrQuantity(s string) *resource.Quantity {
|
||||
q := resource.MustParse(s)
|
||||
return &q
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue