From 82d0fe3e0d050d986b3929135d2e85a0b3738bb2 Mon Sep 17 00:00:00 2001 From: dmgnr Date: Thu, 3 Sep 2026 11:28:30 +0700 Subject: [PATCH] feat: v2 --- .dockerignore | 7 + .kuberrc.ts | 165 ++ Dockerfile.server | 20 + README.md | 472 ++++-- build.ts | 35 - command/audit.ts | 59 + command/auth.ts | 136 ++ command/db.ts | 22 +- command/down.ts | 67 +- command/exec.ts | 115 +- command/logs.ts | 284 +--- command/main.ts | 10 + command/operations.ts | 97 ++ command/ps.ts | 33 +- command/restart.ts | 54 +- command/rollback.ts | 82 +- command/s3.ts | 28 +- command/stop.ts | 38 +- command/up.ts | 341 +++-- command/users.ts | 253 ++++ compose.yml | 70 + const.ts | 2 + example/README.md | 6 +- index.ts | 5 + lib/api.ts | 288 ++++ lib/apply.ts | 21 +- lib/build.ts | 863 ++++------- lib/config.ts | 32 - lib/convert.ts | 76 +- lib/exec-api.ts | 226 +++ lib/render.ts | 19 +- lib/session.ts | 109 ++ lib/workspace.ts | 357 +++++ package.json | 5 +- server/app.ts | 2024 +++++++++++++++++++++++++ server/audit-store.ts | 143 ++ server/auth.ts | 204 +++ server/authorization.ts | 43 + server/build-controller.ts | 661 ++++++++ server/build-job.ts | 218 +++ server/build-kubernetes.ts | 429 ++++++ server/build-store.ts | 188 +++ server/cas.ts | 99 ++ server/exec-service.ts | 729 +++++++++ server/index.ts | 257 ++++ server/kubernetes-exec.ts | 265 ++++ server/kubernetes-logs.ts | 152 ++ server/kubernetes-state.ts | 1129 ++++++++++++++ server/kubernetes-store.ts | 381 +++++ server/log-service.ts | 575 +++++++ server/management.ts | 677 +++++++++ server/materialize.ts | 161 ++ server/operation-store.ts | 398 +++++ server/redact.ts | 19 + server/registry.ts | 175 +++ server/workspace-store.ts | 377 +++++ shared/api.ts | 245 +++ shared/artifacts.ts | 226 +++ shared/build-protocol.ts | 63 + tests/command/administration.test.ts | 179 +++ tests/command/api-migration.test.ts | 141 ++ tests/command/exec.test.ts | 57 + tests/command/logs.test.ts | 84 + tests/command/metadata.test.ts | 10 +- tests/command/up-api.test.ts | 116 ++ tests/lib/api.test.ts | 270 ++++ tests/lib/apply.test.ts | 10 + tests/lib/build-api.test.ts | 178 +++ tests/lib/config.test.ts | 8 - tests/lib/convert-artifacts.test.ts | 191 +++ tests/lib/exec-api.test.ts | 163 ++ tests/lib/render-api.test.ts | 34 + tests/lib/session.test.ts | 71 + tests/lib/workspace.test.ts | 192 +++ tests/server/app.test.ts | 471 ++++++ tests/server/audit-store.test.ts | 78 + tests/server/auth.test.ts | 151 ++ tests/server/build-controller.test.ts | 483 ++++++ tests/server/build-job.test.ts | 169 +++ tests/server/build-store.test.ts | 110 ++ tests/server/cas.test.ts | 71 + tests/server/exec-service.test.ts | 373 +++++ tests/server/exec-websocket.test.ts | 347 +++++ tests/server/kubernetes-state.test.ts | 836 ++++++++++ tests/server/kubernetes-store.test.ts | 261 ++++ tests/server/log-service.test.ts | 410 +++++ tests/server/management.test.ts | 240 +++ tests/server/materialize.test.ts | 143 ++ tests/server/operation-store.test.ts | 181 +++ tests/server/registry.test.ts | 146 ++ tests/server/workspace-store.test.ts | 74 + tests/shared/build-protocol.test.ts | 32 + types.d.ts | 7 - 93 files changed, 19237 insertions(+), 1285 deletions(-) create mode 100644 .dockerignore create mode 100644 .kuberrc.ts create mode 100644 Dockerfile.server delete mode 100644 build.ts create mode 100644 command/audit.ts create mode 100644 command/auth.ts create mode 100644 command/operations.ts create mode 100644 command/users.ts create mode 100644 compose.yml create mode 100644 lib/api.ts create mode 100644 lib/exec-api.ts create mode 100644 lib/session.ts create mode 100644 lib/workspace.ts create mode 100644 server/app.ts create mode 100644 server/audit-store.ts create mode 100644 server/auth.ts create mode 100644 server/authorization.ts create mode 100644 server/build-controller.ts create mode 100644 server/build-job.ts create mode 100644 server/build-kubernetes.ts create mode 100644 server/build-store.ts create mode 100644 server/cas.ts create mode 100644 server/exec-service.ts create mode 100644 server/index.ts create mode 100644 server/kubernetes-exec.ts create mode 100644 server/kubernetes-logs.ts create mode 100644 server/kubernetes-state.ts create mode 100644 server/kubernetes-store.ts create mode 100644 server/log-service.ts create mode 100644 server/management.ts create mode 100644 server/materialize.ts create mode 100644 server/operation-store.ts create mode 100644 server/redact.ts create mode 100644 server/registry.ts create mode 100644 server/workspace-store.ts create mode 100644 shared/api.ts create mode 100644 shared/artifacts.ts create mode 100644 shared/build-protocol.ts create mode 100644 tests/command/administration.test.ts create mode 100644 tests/command/api-migration.test.ts create mode 100644 tests/command/exec.test.ts create mode 100644 tests/command/logs.test.ts create mode 100644 tests/command/up-api.test.ts create mode 100644 tests/lib/api.test.ts create mode 100644 tests/lib/build-api.test.ts create mode 100644 tests/lib/convert-artifacts.test.ts create mode 100644 tests/lib/exec-api.test.ts create mode 100644 tests/lib/render-api.test.ts create mode 100644 tests/lib/session.test.ts create mode 100644 tests/lib/workspace.test.ts create mode 100644 tests/server/app.test.ts create mode 100644 tests/server/audit-store.test.ts create mode 100644 tests/server/auth.test.ts create mode 100644 tests/server/build-controller.test.ts create mode 100644 tests/server/build-job.test.ts create mode 100644 tests/server/build-store.test.ts create mode 100644 tests/server/cas.test.ts create mode 100644 tests/server/exec-service.test.ts create mode 100644 tests/server/exec-websocket.test.ts create mode 100644 tests/server/kubernetes-state.test.ts create mode 100644 tests/server/kubernetes-store.test.ts create mode 100644 tests/server/log-service.test.ts create mode 100644 tests/server/management.test.ts create mode 100644 tests/server/materialize.test.ts create mode 100644 tests/server/operation-store.test.ts create mode 100644 tests/server/registry.test.ts create mode 100644 tests/server/workspace-store.test.ts create mode 100644 tests/shared/build-protocol.test.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d4b0653 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.gitignore +.next +dist +node_modules +example/node_modules +*.log diff --git a/.kuberrc.ts b/.kuberrc.ts new file mode 100644 index 0000000..cbbd160 --- /dev/null +++ b/.kuberrc.ts @@ -0,0 +1,165 @@ +import type { KuberConfig } from "./types"; +import { LABELS } from "./const"; + +export default { + project: "kuber-system", + + postRender(resources, context) { + const metadata = { + namespace: context.project, + labels: LABELS, + }; + + resources.push( + { + apiVersion: "v1", + kind: "ServiceAccount", + metadata: { ...metadata, name: "kuber-server" }, + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "Role", + metadata: { ...metadata, name: "kuber-server-auth" }, + rules: [ + { + apiGroups: [""], + resources: ["secrets", "configmaps", "pods", "pods/log"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + { + apiGroups: ["batch"], + resources: ["jobs"], + verbs: ["get", "list", "watch", "create", "delete"], + }, + { + apiGroups: ["coordination.k8s.io"], + resources: ["leases"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "RoleBinding", + metadata: { ...metadata, name: "kuber-server-auth" }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "Role", + name: "kuber-server-auth", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "kuber-server", + namespace: context.project, + }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "ClusterRole", + metadata: { name: "kuber-server-manager", labels: LABELS }, + rules: [ + { + apiGroups: [""], + resources: ["namespaces"], + verbs: ["get", "list", "create", "patch", "delete"], + }, + { + apiGroups: [""], + resources: ["services", "configmaps", "secrets", "persistentvolumeclaims"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + { + apiGroups: [""], + resources: ["pods", "pods/log"], + verbs: ["get", "list", "watch"], + }, + { + apiGroups: [""], + resources: ["pods/exec"], + verbs: ["get", "create"], + }, + { + apiGroups: ["apps"], + resources: ["deployments", "replicasets", "statefulsets", "daemonsets"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + { + apiGroups: ["batch"], + resources: ["jobs", "cronjobs"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + { + apiGroups: ["networking.k8s.io"], + resources: ["ingresses", "networkpolicies"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + { + apiGroups: ["autoscaling"], + resources: ["horizontalpodautoscalers"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + { + apiGroups: ["policy"], + resources: ["poddisruptionbudgets"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + { + apiGroups: ["storage.k8s.io"], + resources: ["storageclasses"], + verbs: ["get", "list", "create", "patch"], + }, + { + apiGroups: ["traefik.io"], + resources: ["ingressroutes"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + { + apiGroups: ["postgresql.cnpg.io"], + resources: ["clusters"], + verbs: ["get", "list", "patch"], + }, + { + apiGroups: ["postgresql.cnpg.io"], + resources: ["databases"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + { + apiGroups: ["garage.rajsingh.info"], + resources: ["garagebuckets", "garagekeys"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }, + ], + }, + { + apiVersion: "rbac.authorization.k8s.io/v1", + kind: "ClusterRoleBinding", + metadata: { name: "kuber-server-manager", labels: LABELS }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "ClusterRole", + name: "kuber-server-manager", + }, + subjects: [ + { + kind: "ServiceAccount", + name: "kuber-server", + namespace: context.project, + }, + ], + }, + ); + + const password = process.env.KUBER_BOOTSTRAP_PASSWORD; + if (password) { + resources.push({ + apiVersion: "v1", + kind: "Secret", + metadata: { ...metadata, name: "kuber-bootstrap" }, + type: "Opaque", + stringData: { password }, + }); + } + }, +} satisfies KuberConfig; diff --git a/Dockerfile.server b/Dockerfile.server new file mode 100644 index 0000000..d6a4fff --- /dev/null +++ b/Dockerfile.server @@ -0,0 +1,20 @@ +FROM oven/bun:1-alpine AS build +WORKDIR /app + +COPY package.json bun.lock ./ +RUN bun install --frozen-lockfile + +COPY const.ts tsconfig.json ./ +COPY lib ./lib +COPY schema ./schema +COPY server ./server +COPY shared ./shared +RUN bun build server/index.ts --target bun --minify --outfile dist/kuber-server.js + +FROM oven/bun:1-alpine +WORKDIR /app +COPY --from=build /app/dist/kuber-server.js ./kuber-server.js + +USER bun +EXPOSE 3000 +CMD ["bun", "kuber-server.js"] diff --git a/README.md b/README.md index 6eef6a4..d05c71b 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,280 @@ # kuber -`kuber` is Astral's internal Docker Compose to Kubernetes translation layer. +`kuber` is a Docker Compose to Kubernetes translation layer backed by a +self-hosted management service (`kuber-server`). It reads a local Compose file, +renders Kubernetes resources, and applies them to the cluster through an +authenticated v2 API. Image builds happen inside the cluster with rootless +BuildKit, so the workstation needs neither Docker nor `kubectl`. -It reads a local Compose file, renders Kubernetes resources, applies them to the cluster, and can build images remotely from your working tree without requiring Docker on your machine. +## Architecture -## What It Does +``` +workstation (kuber CLI) ──HTTPS──▶ kuber-server (in-cluster pod) + │ + ├─ CAS (content-addressed blobs on RWX PVC) + ├─ BuildKit Jobs (rootless) ──▶ registry + └─ Kubernetes API (RBAC-scoped) +``` -- Reads `compose.yml` or `docker-compose.yml` -- Converts supported Compose services into Kubernetes resources -- Applies those resources into a namespace named after the current directory -- Turns `env_file` entries into Kubernetes Secrets and mounts them through `envFrom` -- Translates file mounts into ConfigMaps and directory/volume mounts into PVC-backed volumes -- Builds images on the remote builder by syncing: - - committed git state - - tracked local diffs - - untracked files - - ignored `.env*` files -- Supports host-based `ports` syntax that renders Kubernetes `Ingress` rules -- Supports managed Postgres claims through special pseudo-volumes such as `postgresql:app` -- Supports managed S3 buckets and credentials through pseudo-volumes such as `s3:app` +The CLI authenticates to the server with a Bearer token acquired by `kuber +login`. It snapshots the repository into a content-addressed workspace, uploads +only the blobs the server is missing, and submits a build request. The server +materializes that workspace onto a shared RWX PVC and runs a rootless BuildKit +Job that builds and pushes the image, then the CLI pins the resulting immutable +registry digest into the rendered Deployment. + +The server also owns reconciliation: it plans, applies, and prunes resources in +a per-project namespace, reconciles managed Postgres and S3 claims, rolls +deployments back, streams logs, and exposes interactive `exec` sessions over a +WebSocket. ## Environment Assumptions -This tool is built for Astral's cluster and workstation setup. It is not intended to work unchanged outside that environment. +kuber targets a specific self-hosted cluster and workstation setup. It is not +intended to run unchanged against an arbitrary Kubernetes environment. The +expected setup: -Expected local setup: - -- Tailscale access to the remote builder and Kubernetes network -- a working `~/.kube/config` -- `ssh` available locally +- an account on the `kuber-server` management API +- a working ClusterRole/Role (see [Server Deployment](#server-deployment)) +- a registry the in-cluster BuildKit can push to Not required locally: - Docker - `kubectl` -Docker is not required because builds happen on the remote builder after `kuber` syncs your repo state there. `kubectl` is not required because cluster access is handled through the bundled Kubernetes client. +## API Origin + +The v2 management API has one hard-coded origin: + +```text +https://kuber.astrxl.dev/api/v2 +``` + +## Authentication + +```bash +kuber login dmgnr +kuber login dmgnr --persist +kuber whoami +kuber logout +``` + +The default login is stored with mode `0600` under +`$XDG_RUNTIME_DIR/kuber/session.json` and disappears with the user runtime +directory. `--persist` instead uses `$XDG_CONFIG_HOME/kuber/session.json`, or +`~/.config/kuber/session.json` when `XDG_CONFIG_HOME` is unset. Password input +is never echoed. Runtime sessions last 24 hours and persistent sessions last 30 +days; `logout` revokes the server-side session. `readSession` falls back to the +persistent file when no runtime session exists. + +The default login is scoped to the current user and the authenticated identity +is available to every v2 command. `login`, `logout`, and `whoami` are the only +commands that run without a loaded project configuration. + +### Roles and Authorization + +The server grants capabilities through three roles: + +- `viewer` — read-only cluster access (`kubernetes:read`) +- `operator` — `viewer` plus `kubernetes:write` and `kubernetes:exec` +- `admin` — all capabilities, including user administration + (`users:read`, `users:write`, `sessions:revoke`) + +Administer users with the `users` command tree: + +```bash +kuber users ls +kuber users add dmgnr --roles admin +kuber users update dmgnr --roles operator +kuber users update dmgnr --password +kuber users disable dmgnr +kuber users enable dmgnr +kuber users delete dmgnr +kuber users revoke dmgnr +``` + +`users add`, `update --password`, and `delete` prompt for password / written +confirmation on an interactive terminal. Passwords are hashed with Argon2id on +the server and never stored in plaintext. Updating a user's roles or password +revokes all of that user's active sessions. + +Inspect server-side operations and the audit trail: + +```bash +kuber operations ls +kuber operations get +kuber audit ls +``` + +## Server Deployment + +The repository's `compose.yml` owns the `kuber-system` namespace, the +`kuber-server` image, its Deployment, Service, and Ingress. `.kuberrc.ts` +extends the rendered manifests with the server's ServiceAccount and RBAC: + +- a namespaced `Role`/`RoleBinding` (`kuber-server-auth`) for the `kuber-system` + Secrets, ConfigMaps, Pods, Jobs, and Leases the server itself reads and writes +- a `ClusterRole`/`ClusterRoleBinding` (`kuber-server-manager`) granting the + cross-namespace verbs it needs to manage user projects + +`compose.yml` runs the server as a non-root user (`runAsUser`/`runAsGroup` +`1000`), drops all Linux capabilities, uses a read-only root filesystem with a +`RuntimeDefault` seccomp profile, and backs `/data` with a Longhorn PVC +(`kuber-build-data`). The PVC hosts the CAS, materialized workspaces, and +resumable upload bytes, and is shared with BuildKit Jobs. + +Deploy the server with kuber itself: + +```bash +KUBER_BOOTSTRAP_PASSWORD='replace-me' kuber up +``` + +The server creates an `admin` user (from `KUBER_BOOTSTRAP_USERNAME`, default +`dmgnr`) on first boot only if that user does not already exist. The bootstrap +Secret is only rendered while `KUBER_BOOTSTRAP_PASSWORD` is set. After logging +in successfully, reconcile without that variable and restart once so kuber +removes the stale bootstrap Secret and the password leaves the pod environment: + +```bash +kuber up --no-b +kuber restart kuber-server +``` + +See [Account Recovery](#account-recovery) for what to do if you are locked out. + +### Security Constraints + +Because the server's ServiceAccount is scoped by the RBAC in `.kuberrc.ts`, it +can only act on the resources kuber manages. The management layer additionally +enforces ownership, so the server refuses to mutate resources that do not carry +kuber's workspace labels. Deleting a resource requires its UID as an optimistic +concurrency precondition, and workspace deletion also requires the namespace +UID. Namespaces that are not owned by kuber, or owned by a different workspace, +are never mutated (see [Workspaces and Migration](#workspaces-and-migration)). + +## Workspaces and Migration + +Each project maps to a Kubernetes namespace derived from the current working +directory. The CLI records a "workspace" on the server keyed by project name and +borrows the namespace UID to guarantee it owns the namespace before reconciling. +Adopting existing resources relabels them (Server-Side Apply) only when they are +already managed by kuber and not owned by another workspace. + +`up` refuses to mutate a namespace when: + +- the namespace already exists but does not carry kuber's managed-by label + (`external`), or +- the namespace is labeled for a different workspace UID + (`different-workspace`) + +In those cases the CLI prints a hint to `POST +/workspaces//adopt` with the namespace UID to complete a safe, +explicit adoption. The platform namespace (`kuber-system`) is adopted through +the admin-only platform adoption route. + +Workspace state, revisions, operations, and audit events are stored as Secrets +and ConfigMaps in `kuber-system` keyed by kuber's `kuber.astrxl.dev/type` +label. Workspace updates are optimistic (If-Match on resource version) and +immutable revisions are recorded so history survives. Expired sessions are +cleaned up on an interval, and stale operations are marked failed on server +startup recovery. + +### Migration Behavior + +`kuber up` is idempotent and safe to re-run. Each run: + +1. snapshots the workspace and uploads missing blobs to the server CAS, +2. ensures the workspace record (creating or updating it with an optimistic + If-Match), +3. adopts the namespace and its kuber-managed resources, +4. reconciles managed Postgres and S3 claims, +5. renders manifests, plans the diff, applies desired resources, waits for + rollout, and deletes stale resources. + +Because resource references are immutable digests, re-running `up` only restarts +deployments whose image content actually changed. `start` re-resolves published +digests without building. + +## Registry Authentication + +Building and pushing images from inside the cluster typically requires +credentials for the target registry. These are read from a Docker config file +(`KUBER_REGISTRY_CONFIG`, default `/etc/kuber/registry/config.json`) and mounted +as an image pull secret named by `KUBER_REGISTRY_SECRET`. + +Registry authentication is optional. If `KUBER_REGISTRY_SECRET` is unset, the +server warns at startup and BuildKit uses anonymous registry access. This is +intended for registries that allow anonymous push/pull. The same credentials are +used when the server resolves a published image digest (`start` / `export`). + +The server supports both standard registry bearer-token (`WWW-Authenticate: +Bearer`) and pre-emptive Basic auth when resolving digests. + +### Build Registry Environment + +The server's registry behavior is driven by a few closely related environment +variables: + +- `KUBER_BUILD_REGISTRY` (default `registry.neko-piranha.ts.net`): the registry + the in-cluster BuildKit pushes built images to and `kuber` uses as the image + namespace. It also supplies the registry host for authentication. +- `KUBER_INTERNAL_REGISTRY_HOST`: the host of an internal registry (for example + the in-cluster distribution service) used for the BuildKit cache image and, + when set, the image direct push target. When unset, push and cache fall back + to `KUBER_BUILD_REGISTRY`. +- `KUBER_INTERNAL_REGISTRY_INSECURE`: set to `"true"` to push to the internal + registry over plain HTTP instead of HTTPS. Only meaningful when + `KUBER_INTERNAL_REGISTRY_HOST` is set. +- `KUBER_PUSH_IMAGE_PREFIX` (default `kuber/`): a prefix applied to project + images pushed to the internal registry when `KUBER_INTERNAL_REGISTRY_HOST` is + configured. +- `KUBER_REGISTRY_RESOLVE_ORIGIN`: an explicit origin used to resolve a + published image digest (used by `start` / `export`). Useful when the digest + must be resolved from a different endpoint than the build/push registry, such + as an internal HTTP registry. +- `KUBER_REGISTRY_CONFIG`: path to the Docker config file with registry + credentials (see above). +- `KUBER_REGISTRY_SECRET`: the image pull Secret mounted for BuildKit's + registry access. +- `KUBER_BUILDKIT_IMAGE`: override the BuildKit runner image used for builds. +- `KUBER_BUILD_DATA_CLAIM`: the PVC claim backing builds. + +See [Server Deployment](#server-deployment) for how `compose.yml` wires the +internal registry variables for the kuber-server pod. + +## Account Recovery + +If you lose your credentials and cannot log in: + +1. Recreate the bootstrap admin by deploying with + `KUBER_BOOTSTRAP_PASSWORD` set again: + ```bash + KUBER_BOOTSTRAP_PASSWORD='new-password' kuber up --no-b + kuber restart kuber-server + ``` +2. The server only creates the bootstrap user if the account does not already + exist, so a fresh `kuber login ` with the new password works, or + use the newly created admin to reset other accounts: + ```bash + kuber users update --password + ``` +3. After recovering, reconcile without `KUBER_BOOTSTRAP_PASSWORD` and restart + so the bootstrap Secret is removed and the password leaves the pod + environment. + +Because user records and session hashes are stored as Secrets in +`kuber-system`, recovery relies on cluster administrators being able to redeploy +the server with bootstrap credentials. `users revoke ` forcibly logs a +user out across all devices. ## Next.js Example -[`example/`](example/) contains a documented deployment template for adding kuber to an existing Bun-powered Next.js project without initializing or bundling an application in this repository. It includes a standalone-output Dockerfile, `.dockerignore`, `compose.yml`, and the required Next.js configuration. +[`example/`](example/) contains a documented deployment template for adding +kuber to an existing Bun-powered Next.js project without initializing or +bundling an application in this repository. It includes a standalone-output +Dockerfile, `.dockerignore`, `compose.yml`, and the required Next.js +configuration. ## Running @@ -72,16 +307,23 @@ bun run index.ts db ls bun run index.ts s3 ls bun run index.ts s3 creds app bun run index.ts s3 ui app +bun run index.ts login dmgnr +bun run index.ts users ls +bun run index.ts operations ls +bun run index.ts audit ls ``` All commands accept `--config` to use a configuration file other than `.kuberrc.ts`: ```bash -kuber --config deploy/production.kuberrc.ts up -kuber up --config deploy/production.kuberrc.ts +kuber --config production.kuberrc.ts up +kuber up --config production.kuberrc.ts ``` +`login`, `logout`, and `whoami` are context-free and do not require a +configuration. + ### Shell Completion Generate and load completions for your shell: @@ -91,27 +333,43 @@ source <(kuber complete zsh) source <(kuber complete bash) ``` -For a permanent setup, write the generated script to a file and source it from your shell configuration. Fish and PowerShell are also supported through `kuber complete fish` and `kuber complete powershell`. +For a permanent setup, write the generated script to a file and source it from +your shell configuration. Fish and PowerShell are also supported through +`kuber complete fish` and `kuber complete powershell`. ## Commands -- `up`: build images if needed, pin matching registry digests, render manifests, apply them, and wait for rollout -- `start`: same as `up` but resolves the currently published image digests instead of building +- `up [--no-b]`: build images if needed, ensure the workspace, reconcile managed + Postgres/S3, render manifests, apply them, wait for rollout, and delete stale + resources +- `start`: like `up` but re-resolves the currently published image digests + instead of building +- `login [username] [--persist]`: authenticate with the kuber API +- `logout`: revoke and remove the current API session +- `whoami`: show the authenticated API user and roles +- `users`: administer user accounts and roles +- `operations`: inspect server-side reconciliation operations +- `audit`: inspect the audit trail +- `ps [-a]`: print an ANSI graph of the current project namespace, hiding stopped + deployments by default +- `logs [deployment] [-f]`: print (or follow) logs for one deployment or all + managed deployments +- `exec `: execute a command inside a running + deployment pod over an interactive WebSocket +- `restart [deployment]`: roll out a restart across managed deployments - `stop`: scale managed deployments to zero -- `restart`: roll out a restart across managed deployments -- `rollback` (alias `fuck`): roll one deployment back to its previous release, or all managed deployments when no name is given -- `down`: delete managed resources while keeping ingress, PVCs, managed databases, and managed S3 storage -- `down -f`: also delete ingress, PVCs, managed database and S3 resources, and the namespace -- `ps`: print an ANSI tree of the current project namespace, hiding stopped deployments by default -- `ps -a`: include stopped deployments and stale ReplicaSets in the tree -- `logs [deployment]`: print logs for one deployment or all managed deployments -- `logs -f [deployment]`: follow logs continuously -- `exec `: execute a command inside a running deployment pod -- `db ls`: list managed Postgres claims declared in the current Compose file -- `db creds `: reconcile and print the generated connection details for a managed Postgres claim -- `s3 ls`: list managed S3 claims declared in the current Compose file -- `s3 creds `: print all generated S3 environment variables for a service -- `s3 ui `: print the Garage UI object-browser URL for a service bucket +- `rollback` (alias `fuck`) `[deployment]`: roll one deployment back to its + previous release, or all managed deployments when no name is given +- `down [-f]`: delete managed resources while keeping ingress, PVCs, managed + databases, and managed S3 storage; `-f` also deletes those and the namespace +- `db ls` / `db creds `: list or print credentials for managed Postgres + claims +- `s3 ls` / `s3 creds ` / `s3 ui `: list managed S3 claims, + print their credentials, or print the Garage UI URL for a bucket +- `export [-o file]`: render manifests to a YAML file without applying them + +Lifecycle commands (`restart`, `stop`, `rollback`, `down`) are idempotent: +identical requests are deduplicated server-side and tracked as operations. ## Configuration @@ -125,11 +383,6 @@ export default { project: "my-app", composeFile: "compose.production.yml", registry: "registry.example.com", - builders: { - amd64: "kuber@amd-builder", - arm64: "kuber@arm-builder", - remoteRoot: "kuber-build", - }, rolloutTimeoutMs: 10 * 60_000, async compose(compose) { @@ -147,37 +400,50 @@ export default { Operational defaults: -- `project`: Compose top-level `name`, falling back to the current working directory name +- `project`: Compose top-level `name`, falling back to the current working + directory name - `composeFile`: the first recognized Compose filename in the working directory - `registry`: `registry.neko-piranha.ts.net` -- `builders.amd64`: `kuber@astral-th` -- `builders.arm64`: `kuber@astral` -- `builders.remoteRoot`: `kuber-build` - `rolloutTimeoutMs`: `300000` Project-name precedence is `.kuberrc.ts project`, Compose top-level `name`, then the current working directory name. +The `registry` value controls which registry the CLI requests build images from +and which the server uses to resolve published digests. `rolloutTimeoutMs` +bounds how long `up` and `rollback` wait for a Deployment rollout. + Configuration hooks can be synchronous or asynchronous and receive mutable values: -- `compose(compose, context)`: once after parsing and validation; affects every command that reads Compose -- `preBuild(compose, context)`: before build eligibility is evaluated when builds are enabled -- `postBuild(result, context)`: after images are built; `result` contains `built` and `changed` service names -- `postRender(resources, context)`: after rendering and before reconciliation planning; also runs for `export` -- `postApply(resources, context)`: after desired resources are successfully applied +- `compose(compose, context)`: once after parsing and validation; affects every + command that reads Compose +- `preBuild(compose, context)`: before build eligibility is evaluated when + builds are enabled +- `postBuild(result, context)`: after images are built; `result` contains + `built` and `changed` service names +- `postRender(resources, context)`: after rendering and before reconciliation + planning; also runs for `export` +- `postApply(resources, context)`: after desired resources are successfully + applied Hook context contains the resolved `cwd`, `project`, `composeFile`, and optional `configFile`. A hook error aborts the command and is reported by the normal CLI error handler. +Registry and rollout configuration remain part of the CLI-facing contract. +Image builder selection is not configurable: builds are scheduled, executed, and +owned entirely by the server. + ## Compose Conventions -`kuber` supports a few project-specific Compose conventions on top of normal service translation. +`kuber` supports a few project-specific Compose conventions on top of normal +service translation. ### Host-Based Ports -If a `ports` entry uses a hostname instead of a numeric published port, `kuber` treats it as an ingress host and routes traffic to the target container port. +If a `ports` entry uses a hostname instead of a numeric published port, `kuber` +treats it as an ingress host and routes traffic to the target container port. Example: @@ -188,9 +454,11 @@ services: - somedomain.astrxl.dev:3000 ``` -That produces a Kubernetes `Ingress` rule for `somedomain.astrxl.dev` pointing at the service port for container port `3000`. +That produces a Kubernetes `Ingress` rule for `somedomain.astrxl.dev` pointing +at the service port for container port `3000`. -Single-level wildcard subdomains are supported. Quote wildcard entries so YAML does not treat the leading `*` as an alias: +Single-level wildcard subdomains are supported. Quote wildcard entries so YAML +does not treat the leading `*` as an alias: ```yml services: @@ -200,7 +468,8 @@ services: - "*.secure.astrxl.dev:3001:protected" ``` -Protected routes use the kuber dialect and render Traefik `IngressRoute` resources instead of plain Kubernetes `Ingress`: +Protected routes use the kuber dialect and render Traefik `IngressRoute` +resources instead of plain Kubernetes `Ingress`: ```yml services: @@ -254,7 +523,8 @@ services: - s3:app ``` -This creates `GarageBucket/app` and `GarageKey/app` in `garage-system`. To use different key and bucket names: +This creates `GarageBucket/app` and `GarageKey/app` in `garage-system`. To use +different key and bucket names: ```yml services: @@ -263,7 +533,8 @@ services: - s3:app-key/shared-assets ``` -The Garage operator generates the credentials. `kuber` reads its generated Secret and injects these values into the service's `-env` Secret: +The Garage operator generates the credentials. `kuber` reads its generated +Secret and injects these values into the service's `-env` Secret: - `AWS_ACCESS_KEY_ID` - `AWS_SECRET_ACCESS_KEY` @@ -271,7 +542,9 @@ The Garage operator generates the credentials. `kuber` reads its generated Secre - `AWS_REGION` - `S3_BUCKET` -One service can declare both `postgresql:...` and `s3:...`; all generated values are merged into the same service Secret. Managed Garage buckets and keys are retained by normal `down` and deleted by `down -f`. +One service can declare both `postgresql:...` and `s3:...`; all generated +values are merged into the same service Secret. Managed Garage buckets and keys +are retained by normal `down` and deleted by `down -f`. Inspect a claim, print its generated credentials, or get its Garage UI URL: @@ -287,9 +560,11 @@ assignments. `s3 ui` only prints the URL; it does not open a browser. ### Environment Files -`env_file` entries are read locally and turned into a Kubernetes `Secret` named `-env`. Deployments then consume that secret through `envFrom`. +`env_file` entries are read locally and turned into a Kubernetes `Secret` named +`-env`. Deployments then consume that secret through `envFrom`. -This is also where generated values such as `DATABASE_URL` and the managed S3 environment are injected. +This is also where generated values such as `DATABASE_URL` and the managed S3 +environment are injected. ### Volumes @@ -382,12 +657,27 @@ shared across projects and intentionally retained when a project is removed. ## Building -Image builds run through the selected SSH builder. After a push, the registry's -manifest digest is captured and embedded into the rendered Deployment as an -immutable `:latest@sha256:...` reference, so a changed image naturally triggers -a rollout (there is no separate "restart changed deployments" step). `start` and -`export` look up the currently published digest without rebuilding, and fail if -a buildable service has no published image yet. +When a service declares `build`, the CLI: + +1. snapshots the repository into a content-addressed workspace + (committed git state, tracked changes, untracked files, and ignored `.env*` + files), +2. negotiates with the server and uploads only the blobs it is missing, +3. submits a build request; the server materializes the workspace from the CAS + onto a shared RWX PVC and runs a rootless BuildKit Job that builds and pushes + the configured image with registry cache. + +Build containers are restricted: they run as non-root (`runAsUser`/`runAsGroup` +`1000`), do not mount the service account token, and only read the workspace +(read-only mount) and a writable BuildKit state `emptyDir`. After a push, the +registry's manifest digest is captured and embedded into the rendered Deployment +as an immutable `:latest@sha256:...` reference, so a changed image naturally +triggers a rollout. `start` and `export` look up the currently published digest +without rebuilding, and fail if a buildable service has no published image yet. + +`export` is side-effect-free: it refuses to render managed Postgres or S3 claims +(because it cannot call the server to generate credentials) and instead tells +you to run `kuber up` or remove the managed provider claims. ## Rollback @@ -407,23 +697,35 @@ Notes and limitations: - A later `kuber up` or `kuber start` re-resolves `:latest` and returns the Deployment to the current desired state anyway, so rollback is the right tool for responding to a bad deploy, not for permanently pinning an old version. -- Rollback only considers Deployments managed by kuber (`app.kubernetes.io/managed-by=kuber`). +- Rollback only considers Deployments managed by kuber + (`app.kubernetes.io/managed-by=kuber`) and only runs within a workspace whose + namespace kuber owns. -To build distributable binaries: +## Workspace State and Operations + +The server persists per-workspace state, revisions, operations, and audit +events as Secrets/ConfigMaps in `kuber-system`. Mutations are idempotent: + +- operations carry an idempotency key so retries do not double-apply, +- resource deletion requires a UID precondition, +- workspace replacement is guarded by If-Match. + +The CLI is built with the normal `build` script for distribution and local +use: ```bash -bun run build.ts -``` - -If you only want a plain JavaScript bundle for quick local use in another workspace, build `index.ts` directly: - -```bash -bun build index.ts --target bun --minify --sourcemap --outdir dist +bun run build ``` ## Notes - Resource names and namespaces are derived from the current working directory. -- The remote build flow is optimized for local iteration, not for producing a perfectly clean export of the repository. -- Managed database support is Kubernetes-only. It injects `DATABASE_URL` into the generated app secret and does not rewrite local `.env` files. -- `kuber` operates on managed resources in the namespace matching the current directory name. +- The workspace snapshot is optimized for local iteration, not for producing a + perfectly clean export of the repository. +- Managed database support is Kubernetes-only. It injects `DATABASE_URL` into + the generated app secret and does not rewrite local `.env` files. +- `kuber` operates on managed resources in the namespace matching the current + directory name. +- Builds are scheduled, executed, and owned by the server. There is no local + SSH/daemon builder configuration; `registry` and `rolloutTimeoutMs` remain + CLI-facing settings. diff --git a/build.ts b/build.ts deleted file mode 100644 index 2291f20..0000000 --- a/build.ts +++ /dev/null @@ -1,35 +0,0 @@ -// oxlint-disable no-unused-expressions -import { build } from "bun"; -import { createLogger } from "./lib/logger"; -import { readdir, rename, rmdir } from "node:fs/promises"; - -async function platform(p: string) { - const log = createLogger( - p - .split("-") - .map((e) => e[0]) - .join(""), - ); - log`Build started`; - await build({ - compile: true, - minify: true, - target: `bun-${p}` as "bun", - entrypoints: ["index.ts"], - outdir: `dist/${p}`, - features: ["prod"], - bytecode: true, - }); - log`Build finished`; - const output = await readdir(`dist/${p}`).then((e) => e[0]!); - const [_, ext] = output.split("."); - await rename( - `dist/${p}/${output}`, - `dist/kuber${p.includes("arm64") ? "-arm64" : ""}${ext ? `.${ext}` : ""}`, - ); - await rmdir(`dist/${p}`); -} - -const PLATFORMS = ["linux-x64", "linux-arm64", "windows-x64"]; - -await Promise.all(PLATFORMS.map(platform)); diff --git a/command/audit.ts b/command/audit.ts new file mode 100644 index 0000000..9bea9b9 --- /dev/null +++ b/command/audit.ts @@ -0,0 +1,59 @@ +import { defineCommand } from "citty"; +import { apiRequest, type ApiRequestInit } from "../lib/api"; +import { toTable } from "../lib/format"; + +export type AuditApiRequest = ( + path: string, + init?: ApiRequestInit, +) => Promise; + +type AuditEvent = { + metadata: { name: string; creationTimestamp: string }; + spec: { + actor: { username: string }; + action: string; + workspaceId?: string; + operationId?: string; + outcome: string; + }; +}; + +export async function listAuditEvents( + workspace?: string, + request: AuditApiRequest = apiRequest, +): Promise { + const path = workspace + ? `/audit?${new URLSearchParams({ workspaceId: workspace })}` + : "/audit"; + const response = await request<{ items: AuditEvent[] }>(path); + if (response.items.length === 0) return "No audit events"; + return toTable( + response.items.map((event) => ({ + timestamp: event.metadata.creationTimestamp, + actor: event.spec.actor.username, + action: event.spec.action, + workspace: event.spec.workspaceId, + outcome: event.spec.outcome, + operation: event.spec.operationId, + })), + ); +} + +const list = defineCommand({ + meta: { name: "ls", description: "List audit events" }, + args: { + workspace: { + type: "string", + alias: "w", + description: "Filter by workspace", + }, + }, + async run({ args }) { + console.log(await listAuditEvents(args.workspace ?? args._[0])); + }, +}); + +export const audit = defineCommand({ + meta: { name: "audit", description: "Inspect the audit log" }, + subCommands: { ls: list }, +}); diff --git a/command/auth.ts b/command/auth.ts new file mode 100644 index 0000000..b76d083 --- /dev/null +++ b/command/auth.ts @@ -0,0 +1,136 @@ +import { defineCommand } from "citty"; +import { createInterface } from "node:readline/promises"; +import { stdin, stdout } from "node:process"; +import { apiRequest } from "../lib/api"; +import { + readSession, + removeSessions, + writeSession, + type KuberSession, +} from "../lib/session"; + +type LoginResponse = KuberSession; + +async function promptUsername(): Promise { + const readline = createInterface({ input: stdin, output: stdout }); + try { + return (await readline.question("Username: ")).trim(); + } finally { + readline.close(); + } +} + +async function promptPassword(): Promise { + if (!stdin.isTTY || !stdin.setRawMode) { + throw new Error("Password input requires an interactive terminal"); + } + + stdout.write("Password: "); + stdin.setRawMode(true); + stdin.resume(); + return new Promise((resolve, reject) => { + let password = ""; + const cleanup = () => { + stdin.off("data", onData); + stdin.setRawMode(false); + stdin.pause(); + stdout.write("\n"); + }; + const onData = (chunk: Buffer) => { + const value = chunk.toString("utf8"); + if (value === "\u0003") { + cleanup(); + reject(new Error("Login cancelled")); + return; + } + if (value === "\r" || value === "\n") { + cleanup(); + resolve(password); + return; + } + if (value === "\u007f" || value === "\b") { + password = password.slice(0, -1); + return; + } + password += value; + }; + stdin.on("data", onData); + }); +} + +export async function loginUser( + username: string, + password: string, + persistent: boolean, +): Promise { + const session = await apiRequest( + "/login", + { + method: "POST", + body: JSON.stringify({ username, password, persistent }), + }, + { authenticated: false }, + ); + await writeSession(session, persistent); + return session; +} + +export const login = defineCommand({ + meta: { + name: "login", + description: "Log in to kuber.astrxl.dev", + }, + args: { + persist: { + type: "boolean", + description: "Keep the login across reboots", + }, + }, + async run({ args }) { + const username = String(args._[0] ?? "").trim() || (await promptUsername()); + if (!username) throw new Error("Username is required"); + const session = await loginUser( + username, + await promptPassword(), + Boolean(args.persist), + ); + console.log(`Logged in as ${session.user.username}`); + }, +}); + +export const logout = defineCommand({ + meta: { + name: "logout", + description: "Log out of kuber.astrxl.dev", + }, + async run() { + const session = await readSession(); + if (session) { + try { + await apiRequest( + "/logout", + { + method: "POST", + }, + { session }, + ); + } finally { + await removeSessions(); + } + } else { + await removeSessions(); + } + console.log("Logged out"); + }, +}); + +export const whoami = defineCommand({ + meta: { + name: "whoami", + description: "Show the current kuber user", + }, + async run() { + const result = await apiRequest("/me"); + console.log(`${result.username} (${result.roles.join(", ")})`); + }, +}); diff --git a/command/db.ts b/command/db.ts index 45e1860..cdcebb9 100644 --- a/command/db.ts +++ b/command/db.ts @@ -1,14 +1,32 @@ import { defineCommand } from "citty"; +import { apiRequest, type ApiRequestInit } from "../lib/api"; import { ctx } from "../lib/context"; import { DATABASE_HOST, DATABASE_PORT, buildDatabaseUrl, getComposePostgresClaims, - reconcilePostgresClaim, + type PostgresClaim, + type RoleCredentials, } from "../lib/database"; import { toTable } from "../lib/format"; +export type DbApiRequest = ( + path: string, + init?: ApiRequestInit, +) => Promise; + +export function getDatabaseCredentials( + project: string, + claim: PostgresClaim, + request: DbApiRequest = apiRequest, +): Promise { + return request( + `/workspaces/${encodeURIComponent(project)}/databases/credentials`, + { method: "POST", json: { claim } }, + ); +} + const list = defineCommand({ meta: { name: "ls", @@ -53,7 +71,7 @@ const creds = defineCommand({ ); } - const credentials = await reconcilePostgresClaim(context.project, claim); + const credentials = await getDatabaseCredentials(context.project, claim); const url = buildDatabaseUrl(claim, credentials); console.log( diff --git a/command/down.ts b/command/down.ts index 02ec099..d2d44e0 100644 --- a/command/down.ts +++ b/command/down.ts @@ -1,14 +1,28 @@ import { defineCommand } from "citty"; import { Listr } from "listr2"; +import { apiRequest, type ApiRequestInit } from "../lib/api"; import { ctx } from "../lib/context"; -import { - assertManagedNamespace, - deleteResource, - listManagedResources, - sortResources, -} from "../lib/apply"; -import { listManagedDatabaseResources } from "../lib/database"; -import { listManagedStorageResources } from "../lib/storage"; + +export type DownResult = { + full: boolean; + retained: Array<{ kind: string; name: string }>; + delete: Array<{ kind: string; name: string }>; +}; +export type DownApiRequest = ( + path: string, + init?: ApiRequestInit, +) => Promise; + +export function runDown( + project: string, + full = false, + request: DownApiRequest = apiRequest, +): Promise { + return request( + `/workspaces/${encodeURIComponent(project)}/down`, + { method: "POST", json: { full } }, + ); +} export const down = defineCommand({ meta: { @@ -24,43 +38,12 @@ export const down = defineCommand({ }, async run({ args }) { const { project } = ctx(); - await assertManagedNamespace(project); - const resources = sortResources(await listManagedResources(project)).filter( - (resource) => - args.full || - (resource.kind !== "Ingress" && - resource.kind !== "PersistentVolumeClaim"), - ); - - if (args.full) { - resources.push(...(await listManagedDatabaseResources(project))); - resources.push(...(await listManagedStorageResources(project))); - } - - if (args.full) { - resources.push({ - apiVersion: "v1", - kind: "Namespace", - metadata: { name: project }, - }); - } - - if (resources.length === 0) return; - await new Listr([ { title: "Delete resources", - task: (_ctx, task) => { - const ordered = sortResources(resources).reverse(); - - task.output = `${ordered.length} resources queued`; - return task.newListr( - ordered.map((resource) => ({ - title: `${resource.kind} ${resource.metadata?.name}`, - task: () => deleteResource(resource), - })), - { concurrent: false, exitOnError: true }, - ); + task: async (_ctx, task) => { + const result = await runDown(project, args.full); + task.output = `${result.delete.length} resources queued`; }, }, ]).run(); diff --git a/command/exec.ts b/command/exec.ts index 104c96a..fea3634 100644 --- a/command/exec.ts +++ b/command/exec.ts @@ -1,18 +1,66 @@ import { defineCommand } from "citty"; -import { getPodContainerName, getPodForDeployment } from "../lib/shared"; -import { execClient } from "../lib/k8s"; +import { ctx } from "../lib/context"; +import { + openExecSession, + type ExecApiSession, + type OpenExecOptions, +} from "../lib/exec-api"; -function prepareInteractiveStdin(): (() => void) | undefined { - if (!process.stdin.isTTY) return; +export type ExecSessionOpener = typeof openExecSession; - process.stdin.setRawMode?.(true); - process.stdin.resume(); - return () => { - process.stdin.setRawMode?.(false); - process.stdin.pause(); +function terminalSize(): { columns?: number; rows?: number } { + return { + columns: process.stdout.columns, + rows: process.stdout.rows, }; } +export async function runExec( + project: string, + deployment: string, + command: string[], + signal: AbortSignal, + opener: ExecSessionOpener = openExecSession, + options?: OpenExecOptions, +): Promise { + if (!deployment) throw new Error("Deployment name is required"); + if (command.length === 0) throw new Error("Command is required"); + const tty = Boolean(process.stdin.isTTY && process.stdout.isTTY); + const session: ExecApiSession = await opener( + project, + { deployment, command, tty, ...(tty ? terminalSize() : {}) }, + signal, + options, + ); + const onData = (chunk: Buffer | string) => + session.sendStdin( + typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk, + ); + const onEnd = () => session.sendStdin(new Uint8Array(), true); + const onResize = () => { + const { columns, rows } = terminalSize(); + if (columns && rows) session.resize(columns, rows); + }; + process.stdin.on("data", onData); + process.stdin.once("end", onEnd); + if (tty) process.stdout.on("resize", onResize); + try { + for await (const frame of session) { + if (frame.type === "stdout") process.stdout.write(frame.data); + else if (frame.type === "stderr") process.stderr.write(frame.data); + else if (frame.type === "error") throw new Error(frame.message); + else return frame.exitCode; + } + if (signal.aborted) return 0; + throw new Error("Exec connection closed without an exit status"); + } finally { + process.stdin.off("data", onData); + process.stdin.off("end", onEnd); + process.stdout.off("resize", onResize); + session.close(); + } +} + export const exec = defineCommand({ meta: { name: "exec", @@ -22,39 +70,32 @@ export const exec = defineCommand({ const [deployment, ...command] = args._; if (!deployment) throw new Error("Deployment name is required"); if (command.length === 0) throw new Error("Command is required"); - - const pod = await getPodForDeployment(deployment); - if (pod.status?.phase !== "Running") { - throw new Error( - `Deployment ${deployment} has no running pod to exec into (${pod.metadata?.name ?? "unknown pod"} is ${pod.status?.phase ?? "not ready"})`, - ); + const controller = new AbortController(); + const abort = () => controller.abort(); + process.on("SIGINT", abort); + process.on("SIGTERM", abort); + const raw = Boolean(process.stdin.isTTY); + if (raw) { + process.stdin.setRawMode?.(true); + process.stdin.resume(); } - - const namespace = pod.metadata?.namespace ?? "default"; - const podName = pod.metadata?.name; - if (!podName) throw new Error(`No pod name found for deployment ${deployment}`); - - const restoreStdin = prepareInteractiveStdin(); - try { - const socket = await execClient.exec( - namespace, - podName, - getPodContainerName(pod), + const exitCode = await runExec( + ctx().project, + deployment, command, - process.stdout, - process.stderr, - process.stdin, - Boolean(process.stdin.isTTY), + controller.signal, ); - - await new Promise((resolve, reject) => { - socket.onclose = () => resolve(); - socket.onerror = (event: { error?: unknown }) => - reject(event.error ?? new Error("Exec failed")); - }); + if (exitCode !== 0) process.exitCode = exitCode; + } catch (error) { + if (!controller.signal.aborted) throw error; } finally { - restoreStdin?.(); + process.off("SIGINT", abort); + process.off("SIGTERM", abort); + if (raw) { + process.stdin.setRawMode?.(false); + process.stdin.pause(); + } } }, }); diff --git a/command/logs.ts b/command/logs.ts index 3c2742c..7da43c3 100644 --- a/command/logs.ts +++ b/command/logs.ts @@ -1,210 +1,81 @@ import { defineCommand } from "citty"; -import https from "node:https"; -import { createLogger } from "../lib/logger"; -import { core, kc } from "../lib/k8s"; -import { delay, parseRetryAfter } from "../lib/k8s-http"; import { - getPodContainerName, - listManagedDeployments, - listPodsForDeployment, -} from "../lib/shared"; + apiStreamNdjson, + type ApiRequestInit, + type ApiRequestOptions, +} from "../lib/api"; +import { ctx } from "../lib/context"; +import { createLogger } from "../lib/logger"; -function formatError(error: unknown): string { - if (error instanceof Error) return error.message; - return String(error); -} - -async function streamResponseLines( - stream: NodeJS.ReadableStream, - onLine: (line: string) => void, -) { - let buffer = ""; - - for await (const chunk of stream) { - buffer += chunk.toString(); - let newline = buffer.indexOf("\n"); - - while (newline !== -1) { - const line = buffer.slice(0, newline).replace(/\r$/, ""); - buffer = buffer.slice(newline + 1); - if (line) onLine(line); - newline = buffer.indexOf("\n"); +export type LogApiEvent = + | { + type: "log"; + targetName: string; + pod: string; + container: string; + message: string; } - } + | { type: "heartbeat"; timestamp: string } + | { + type: "error"; + message: string; + pod?: string; + container?: string; + retryable: boolean; + }; - const line = buffer.replace(/\r$/, ""); - if (line) onLine(line); -} +export type LogsApiStream = ( + path: string, + init?: ApiRequestInit, + options?: ApiRequestOptions, +) => AsyncIterable; -async function followPodLogs( - namespace: string, - podName: string, - containerName: string, - writeLine: (line: string) => void, - signal: AbortSignal, -) { - const cluster = kc.getCurrentCluster(); - if (!cluster) throw new Error("No currently active cluster"); +export type LogEventWriter = ( + prefix: string, + message: string, + error: boolean, +) => void; - const requestURL = new URL( - `${cluster.server}/api/v1/namespaces/${namespace}/pods/${podName}/log`, - ); - requestURL.searchParams.set("container", containerName); - requestURL.searchParams.set("follow", "true"); - - const options: https.RequestOptions = { - method: "GET", - protocol: requestURL.protocol, - hostname: requestURL.hostname, - port: requestURL.port, - path: `${requestURL.pathname}${requestURL.search}`, - signal, +function defaultWriter(): LogEventWriter { + const loggers = new Map>(); + return (prefix, message, error) => { + let logger = loggers.get(prefix); + if (!logger) { + logger = createLogger(prefix); + loggers.set(prefix, logger); + } + if (error) logger.warn`${message}`; + else logger`${message}`; }; - await kc.applyToHTTPSOptions(options); - - for (let attempt = 0; ; attempt += 1) { - const result = await new Promise<{ - statusCode: number; - retryAfter?: string; - body?: string; - }>((resolve, reject) => { - const request = https.request(options, async (response) => { - const statusCode = response.statusCode ?? 0; - if (statusCode < 200 || statusCode > 299) { - const chunks: Buffer[] = []; - response.on("data", (chunk) => { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - }); - response.on("end", () => { - resolve({ - statusCode, - retryAfter: response.headers["retry-after"], - body: Buffer.concat(chunks).toString("utf8"), - }); - }); - return; - } - - try { - await streamResponseLines(response, writeLine); - resolve({ statusCode }); - } catch (error) { - reject(error); - } - }); - - request.on("error", reject); - request.end(); - }); - - if (result.statusCode >= 200 && result.statusCode <= 299) return; - if (result.statusCode !== 429 || attempt >= 3) { - throw new Error(result.body || `HTTP ${result.statusCode}`); - } - - const retryAfter = parseRetryAfter(result.retryAfter); - await delay(retryAfter ?? 250 * 2 ** attempt, signal); - } } -async function followDeploymentLogs(name: string, signal: AbortSignal) { - const logger = createLogger(name); - const activePods = new Set(); +export async function runLogs( + project: string, + service: string | undefined, + follow: boolean, + signal?: AbortSignal, + stream: LogsApiStream = apiStreamNdjson, + write: LogEventWriter = defaultWriter(), +): Promise { + const query = new URLSearchParams(); + if (service) query.set("service", service); + if (follow) query.set("follow", "true"); + const suffix = query.size ? `?${query}` : ""; - while (!signal.aborted) { - const pods = await listPodsForDeployment(name); - - for (const pod of pods) { - const podName = pod.metadata?.name; - if ( - !podName || - activePods.has(podName) || - pod.status?.phase !== "Running" - ) { - continue; - } - - activePods.add(podName); - void followPodLogs( - pod.metadata?.namespace ?? "default", - podName, - getPodContainerName(pod), - (line) => logger`${line}`, - signal, - ) - .catch((error) => { - if (!signal.aborted) logger.warn`${podName}: ${formatError(error)}`; - }) - .finally(() => { - activePods.delete(podName); - }); + for await (const event of stream( + `/workspaces/${encodeURIComponent(project)}/logs${suffix}`, + { signal }, + { timeoutMs: follow ? 0 : undefined }, + )) { + if (event.type === "heartbeat") continue; + if (event.type === "error") { + write(service ?? event.pod ?? project, event.message, true); + continue; } - - await delay(2000, signal); + write(event.targetName || service || project, event.message, false); } } -async function logDeployment(name: string, follow: boolean) { - const logger = createLogger(name); - - if (!follow) { - const pods = await listPodsForDeployment(name); - if (pods.length === 0) - throw new Error(`No pods found for deployment ${name}`); - - for (const pod of pods) { - const podName = pod.metadata?.name; - if (!podName) continue; - - try { - const text = await core.readNamespacedPodLog({ - namespace: pod.metadata?.namespace ?? "default", - name: podName, - container: getPodContainerName(pod), - }); - - for (const line of text.split(/\r?\n/)) { - if (line) logger`${line}`; - } - } catch (error) { - logger.warn`${podName}: ${formatError(error)}`; - } - } - return; - } - - const controller = new AbortController(); - await new Promise((resolve, reject) => { - const cleanup = () => { - process.off("SIGINT", abort); - process.off("SIGTERM", abort); - }; - - const abort = () => { - controller.abort(); - cleanup(); - resolve(); - }; - - process.on("SIGINT", abort); - process.on("SIGTERM", abort); - void followDeploymentLogs(name, controller.signal) - .then(() => { - cleanup(); - resolve(); - }) - .catch((error) => { - cleanup(); - if (controller.signal.aborted) { - resolve(); - return; - } - controller.abort(); - reject(error); - }); - }); -} - export const logs = defineCommand({ meta: { name: "logs", @@ -218,18 +89,23 @@ export const logs = defineCommand({ }, }, async run({ args }) { - const [deployment] = args._; - const names = deployment - ? [deployment] - : (await listManagedDeployments()) - .map((item) => item.metadata?.name) - .filter((item): item is string => Boolean(item)); - - if (args.follow) { - await Promise.all(names.map((name) => logDeployment(name, true))); - return; + const service = args._[0]; + const controller = new AbortController(); + const abort = () => controller.abort(); + process.on("SIGINT", abort); + process.on("SIGTERM", abort); + try { + await runLogs( + ctx().project, + service, + Boolean(args.follow), + controller.signal, + ); + } catch (error) { + if (!controller.signal.aborted) throw error; + } finally { + process.off("SIGINT", abort); + process.off("SIGTERM", abort); } - - for (const name of names) await logDeployment(name, false); }, }); diff --git a/command/main.ts b/command/main.ts index ceb27c8..77ad53c 100644 --- a/command/main.ts +++ b/command/main.ts @@ -1,10 +1,13 @@ import tab from "@bomb.sh/tab/citty"; import { defineCommand } from "citty"; +import { audit } from "./audit"; +import { login, logout, whoami } from "./auth"; import { db } from "./db"; import { down } from "./down"; import { exec } from "./exec"; import { exportCommand } from "./export"; import { logs } from "./logs"; +import { operations } from "./operations"; import { ps } from "./ps"; import { restart } from "./restart"; import { rollback } from "./rollback"; @@ -12,6 +15,7 @@ import { s3 } from "./s3"; import { start } from "./start"; import { stop } from "./stop"; import { up } from "./up"; +import { users } from "./users"; export const main = defineCommand({ meta: { @@ -26,11 +30,15 @@ export const main = defineCommand({ }, }, subCommands: { + audit, db, down, export: exportCommand, exec, logs, + login, + logout, + operations, ps, restart, rollback, @@ -38,6 +46,8 @@ export const main = defineCommand({ start, stop, up, + users, + whoami, }, }); diff --git a/command/operations.ts b/command/operations.ts new file mode 100644 index 0000000..4e3288e --- /dev/null +++ b/command/operations.ts @@ -0,0 +1,97 @@ +import { defineCommand } from "citty"; +import { apiRequest, type ApiRequestInit } from "../lib/api"; +import { toTable } from "../lib/format"; + +export type OperationsApiRequest = ( + path: string, + init?: ApiRequestInit, +) => Promise; + +type Operation = { + metadata: { name: string; creationTimestamp: string }; + spec: { workspaceId: string; action: string }; + status: { + state: string; + startedAt?: string; + finishedAt?: string; + result?: unknown; + error?: { code: string; message: string }; + }; +}; + +function operationPath(workspace?: string): string { + return workspace + ? `/operations?${new URLSearchParams({ workspaceId: workspace })}` + : "/operations"; +} + +function renderOperations(items: Operation[]): string { + if (items.length === 0) return "No operations"; + return toTable( + items.map((operation) => ({ + id: operation.metadata.name, + workspace: operation.spec.workspaceId, + action: operation.spec.action, + state: operation.status.state, + created: operation.metadata.creationTimestamp, + finished: operation.status.finishedAt, + })), + ); +} + +export async function listOperations( + workspace?: string, + request: OperationsApiRequest = apiRequest, +): Promise { + const response = await request<{ items: Operation[] }>( + operationPath(workspace), + ); + return renderOperations(response.items); +} + +export async function getOperation( + id: string, + request: OperationsApiRequest = apiRequest, +): Promise { + const operation = await request( + `/operations/${encodeURIComponent(id)}`, + ); + const output = [renderOperations([operation])]; + if (operation.status.error) { + output.push( + `Error: ${operation.status.error.code}: ${operation.status.error.message}`, + ); + } + if (operation.status.result !== undefined) { + output.push(`Result: ${JSON.stringify(operation.status.result)}`); + } + return output.join("\n"); +} + +const list = defineCommand({ + meta: { name: "ls", description: "List operations" }, + args: { + workspace: { + type: "string", + alias: "w", + description: "Filter by workspace", + }, + }, + async run({ args }) { + console.log(await listOperations(args.workspace ?? args._[0])); + }, +}); + +const get = defineCommand({ + meta: { name: "get", description: "Get an operation" }, + async run({ args }) { + const id = String(args._[0] ?? "").trim(); + if (!id) throw new Error("Operation ID is required"); + console.log(await getOperation(id)); + }, +}); + +export const operations = defineCommand({ + meta: { name: "operations", description: "Inspect server operations" }, + subCommands: { get, ls: list }, +}); diff --git a/command/ps.ts b/command/ps.ts index 01cd70d..ca6e2be 100644 --- a/command/ps.ts +++ b/command/ps.ts @@ -1,10 +1,24 @@ import { defineCommand } from "citty"; -import { - buildNamespaceGraphs, - fetchNamespaceObjects, - renderNamespaceGraphs, -} from "../lib/graph"; -import { getProject } from "../lib/shared"; +import { apiRequest, type ApiRequestInit } from "../lib/api"; +import { renderNamespaceGraphs, type NamespaceGraph } from "../lib/graph"; +import { ctx } from "../lib/context"; + +export type PsApiRequest = ( + path: string, + init?: ApiRequestInit, +) => Promise; + +export async function runPs( + project: string, + includeIdle = false, + request: PsApiRequest = apiRequest, +): Promise { + const query = includeIdle ? "?includeIdle=true" : ""; + const graphs = await request( + `/workspaces/${encodeURIComponent(project)}/status${query}`, + ); + return renderNamespaceGraphs(graphs); +} export const ps = defineCommand({ meta: { @@ -20,11 +34,6 @@ export const ps = defineCommand({ }, }, async run({ args: { all } }) { - const objects = await fetchNamespaceObjects(getProject()); - console.log( - renderNamespaceGraphs( - buildNamespaceGraphs(objects, { includeIdle: all }), - ), - ); + console.log(await runPs(ctx().project, all)); }, }); diff --git a/command/restart.ts b/command/restart.ts index 596369f..59734eb 100644 --- a/command/restart.ts +++ b/command/restart.ts @@ -1,34 +1,50 @@ import { defineCommand } from "citty"; import { Listr } from "listr2"; -import { assertManagedNamespace } from "../lib/apply"; -import { - getProject, - listManagedDeployments, - restartDeployment, -} from "../lib/shared"; +import { apiRequest, type ApiRequestInit } from "../lib/api"; +import { ctx } from "../lib/context"; + +type LifecycleResponse = { deployments: string[] }; +export type RestartApiRequest = ( + path: string, + init?: ApiRequestInit, +) => Promise; + +export function runRestart( + project: string, + name?: string, + request: RestartApiRequest = apiRequest, +): Promise { + return request( + `/workspaces/${encodeURIComponent(project)}/lifecycle`, + { + method: "POST", + json: { action: "restart", ...(name && { services: [name] }) }, + }, + ); +} export const restart = defineCommand({ meta: { name: "restart", description: "Roll out a restart for managed deployments", }, - async run() { - await assertManagedNamespace(getProject()); - const deployments = await listManagedDeployments(); - if (deployments.length === 0) return; - + args: { + deployment: { + type: "positional", + description: "Deployment name to restart (defaults to all managed)", + required: false, + }, + }, + async run({ args }) { await new Listr([ { title: "Restart deployments", - task: (_ctx, task) => { - task.output = `${deployments.length} deployments queued`; - return task.newListr( - deployments.map((deployment) => ({ - title: `Deployment ${deployment.metadata?.name}`, - task: () => restartDeployment(deployment.metadata!.name!), - })), - { concurrent: false, exitOnError: true }, + task: async (_ctx, task) => { + const { deployments } = await runRestart( + ctx().project, + args.deployment, ); + task.output = `${deployments.length} deployments queued`; }, }, ]).run(); diff --git a/command/rollback.ts b/command/rollback.ts index 589af07..6f8f192 100644 --- a/command/rollback.ts +++ b/command/rollback.ts @@ -1,9 +1,31 @@ import { defineCommand } from "citty"; import { Listr } from "listr2"; -import { assertManagedNamespace } from "../lib/apply"; +import { apiRequest, type ApiRequestInit } from "../lib/api"; import { ctx } from "../lib/context"; -import { planRollback, rollbackDeployment } from "../lib/rollback"; -import { waitForDeploymentRollout } from "../lib/shared"; + +export type RollbackResult = { deployments: string[] }; +export type RollbackApiRequest = ( + path: string, + init?: ApiRequestInit, +) => Promise; + +export function runRollback( + project: string, + name: string | undefined, + timeoutMs: number, + request: RollbackApiRequest = apiRequest, +): Promise { + return request( + `/workspaces/${encodeURIComponent(project)}/rollback`, + { + method: "POST", + json: { + ...(name && { services: [name] }), + timeoutMs, + }, + }, + ); +} export const rollback = defineCommand({ meta: { @@ -20,48 +42,30 @@ export const rollback = defineCommand({ }, async run({ args }) { const { project, config } = ctx(); - await assertManagedNamespace(project); - - const names = args.deployment ? [args.deployment] : undefined; - const candidates = await planRollback(names); - - if (candidates.length === 0) { - if (names) { - console.log(`${names[0]} has no previous release to roll back to`); - } else { - console.log("No deployments have a previous release to roll back to"); - } - return; - } + let deployments: string[] = []; await new Listr([ { title: "Roll back deployments", - task: (taskCtx, task) => { - task.output = `${candidates.length} deployment${candidates.length === 1 ? "" : "s"} queued`; - return task.newListr( - candidates.map((candidate) => ({ - title: `Deployment ${candidate.name}`, - task: () => rollbackDeployment(candidate), - })), - { concurrent: false, exitOnError: true }, - ); - }, - }, - { - title: "Wait for rollout", - task: (taskCtx, task) => { - task.output = `${candidates.length} deployment${candidates.length === 1 ? "" : "s"} queued`; - return task.newListr( - candidates.map((candidate) => ({ - title: `Deployment ${candidate.name}`, - task: () => - waitForDeploymentRollout(candidate.name, config.rolloutTimeoutMs), - })), - { concurrent: false, exitOnError: true }, - ); + task: async (_taskCtx, task) => { + ({ deployments } = await runRollback( + project, + args.deployment, + config.rolloutTimeoutMs, + )); + task.output = `${deployments.length} deployment${deployments.length === 1 ? "" : "s"} queued`; }, }, ]).run(); + + if (deployments.length === 0) { + if (args.deployment) { + console.log( + `${args.deployment} has no previous release to roll back to`, + ); + } else { + console.log("No deployments have a previous release to roll back to"); + } + } }, }); diff --git a/command/s3.ts b/command/s3.ts index c83c942..0c6fe5c 100644 --- a/command/s3.ts +++ b/command/s3.ts @@ -1,14 +1,27 @@ import { defineCommand } from "citty"; +import { apiRequest, type ApiRequestInit } from "../lib/api"; import { ctx } from "../lib/context"; import { toTable } from "../lib/format"; -import { - getComposeS3Claims, - getS3Credentials, - type S3Claim, -} from "../lib/storage"; +import { getComposeS3Claims, type S3Claim } from "../lib/storage"; const GARAGE_UI_URL = "http://garage-ui.garage-system.svc.cluster.local"; +export type S3ApiRequest = ( + path: string, + init?: ApiRequestInit, +) => Promise; + +export function getRemoteS3Credentials( + project: string, + claim: S3Claim, + request: S3ApiRequest = apiRequest, +): Promise> { + return request>( + `/workspaces/${encodeURIComponent(project)}/storage/credentials`, + { method: "POST", json: { claim } }, + ); +} + export function getS3UiUrl(bucket: string): string { return `${GARAGE_UI_URL}/buckets/${encodeURIComponent(bucket)}/objects`; } @@ -56,7 +69,10 @@ const creds = defineCommand({ const service = args._[0]; if (!service) throw new Error("Service name is required"); - const credentials = await getS3Credentials(await getServiceClaim(service)); + const credentials = await getRemoteS3Credentials( + ctx().project, + await getServiceClaim(service), + ); for (const [name, value] of Object.entries(credentials)) { console.log(`${name}=${value}`); } diff --git a/command/stop.ts b/command/stop.ts index ea59aae..88c4236 100644 --- a/command/stop.ts +++ b/command/stop.ts @@ -1,11 +1,23 @@ import { defineCommand } from "citty"; import { Listr } from "listr2"; -import { assertManagedNamespace } from "../lib/apply"; -import { - getProject, - listManagedDeployments, - scaleDeployment, -} from "../lib/shared"; +import { apiRequest, type ApiRequestInit } from "../lib/api"; +import { ctx } from "../lib/context"; + +type LifecycleResponse = { deployments: string[] }; +export type StopApiRequest = ( + path: string, + init?: ApiRequestInit, +) => Promise; + +export function runStop( + project: string, + request: StopApiRequest = apiRequest, +): Promise { + return request( + `/workspaces/${encodeURIComponent(project)}/lifecycle`, + { method: "POST", json: { action: "stop" } }, + ); +} export const stop = defineCommand({ meta: { @@ -13,22 +25,12 @@ export const stop = defineCommand({ description: "Scale managed deployments to zero", }, async run() { - await assertManagedNamespace(getProject()); - const deployments = await listManagedDeployments(); - if (deployments.length === 0) return; - await new Listr([ { title: "Scale deployments", - task: (_ctx, task) => { + task: async (_ctx, task) => { + const { deployments } = await runStop(ctx().project); task.output = `${deployments.length} deployments queued`; - return task.newListr( - deployments.map((deployment) => ({ - title: `Deployment ${deployment.metadata?.name}`, - task: () => scaleDeployment(deployment.metadata!.name!, 0), - })), - { concurrent: false, exitOnError: true }, - ); }, }, ]).run(); diff --git a/command/up.ts b/command/up.ts index aeb4700..5941d72 100644 --- a/command/up.ts +++ b/command/up.ts @@ -2,39 +2,59 @@ import { defineCommand } from "citty"; import { Listr } from "listr2"; import type { ComposeSpecification } from "../schema/docker.d"; import type { KuberResource } from "../types"; -import { ctx } from "../lib/context"; +import { apiRequest, KuberApiError, type ApiRequestInit } from "../lib/api"; import { - assertManagedNamespace, - applyResource, - deleteResource, - getStaleResources, - sortResources, -} from "../lib/apply"; -import { buildServices, resolveBuildImages } from "../lib/build"; + buildServices, + getRepoRoot, + resolveBuildImages, + type ApiRequester, +} from "../lib/build"; import { composeToKubernetes, type KubernetesResource } from "../lib/convert"; -import { - getComposePostgresClaims, - reconcilePostgresClaims, -} from "../lib/database"; -import { getComposeS3Claims, reconcileS3Claims } from "../lib/storage"; -import { waitForDeploymentRollout } from "../lib/shared"; +import { ctx } from "../lib/context"; +import { getComposePostgresClaims } from "../lib/database"; +import { getComposeS3Claims } from "../lib/storage"; +import { enumerateWorkspace, type WorkspaceSnapshot } from "../lib/workspace"; + +type Workspace = { + metadata: { name: string; uid: string; resourceVersion: string }; +}; + +type ResourceIdentity = { + apiVersion: string; + kind: string; + name: string; + namespace?: string; + uid: string; + workspaceUid: string; +}; + +type ResourcePlan = { + desired: KubernetesResource[]; + stale: ResourceIdentity[]; +}; type UpContext = { compose?: ComposeSpecification; + snapshot?: WorkspaceSnapshot; buildImages?: Record; serviceEnv?: Record>; resources?: KubernetesResource[]; - staleResources?: KubernetesResource[]; + plan?: ResourcePlan; }; +export const WORKSPACE_ADOPTION_METHOD = "POST"; + +export function workspaceAdoptionRoute(project: string): string { + return `/workspaces/${encodeURIComponent(project)}/adopt`; +} + export function mergeServiceEnv( current: Record> | undefined, next: Record>, ): Record> { const merged = { ...current }; - for (const [service, environment] of Object.entries(next)) { + for (const [service, environment] of Object.entries(next)) merged[service] = { ...merged[service], ...environment }; - } return merged; } @@ -45,9 +65,141 @@ export function getDeploymentNames(resources: KubernetesResource[]): string[] { .filter((name): name is string => Boolean(name)); } -export async function runUp(build: boolean) { +function workspaceInput( + compose: ComposeSpecification, + snapshot: WorkspaceSnapshot, +) { + return { + source: { + uri: `cas://${snapshot.digest}`, + digest: snapshot.digest, + }, + config: { compose }, + }; +} + +export async function ensureWorkspace( + project: string, + compose: ComposeSpecification, + snapshot: WorkspaceSnapshot, + request: ApiRequester = apiRequest, +): Promise { + const path = `/workspaces/${encodeURIComponent(project)}`; + const input = workspaceInput(compose, snapshot); + let current: Workspace; + try { + current = await request(path); + } catch (error) { + if (!(error instanceof KuberApiError) || error.status !== 404) throw error; + return request("/workspaces", { + method: "POST", + json: { id: project, ...input }, + }); + } + return request(path, { + method: "PUT", + headers: { "if-match": `"${current.metadata.resourceVersion}"` }, + json: input, + }); +} + +function operationEnvironment( + response: Record, +): Record> { + return Object.fromEntries( + Object.entries(response).filter( + ([key, value]) => + key !== "operation" && + key !== "operationId" && + value !== null && + typeof value === "object" && + !Array.isArray(value), + ), + ) as Record>; +} + +function adoptionHint(project: string, error: unknown): Error { + const message = error instanceof Error ? error.message : String(error); + if (!/namespace .* (?:external|different-workspace)/i.test(message)) + return error instanceof Error ? error : new Error(message); + return new Error( + `${message}. Safe adoption requires ${WORKSPACE_ADOPTION_METHOD} ${workspaceAdoptionRoute(project)} with namespace UID and ownership preconditions.`, + { cause: error }, + ); +} + +async function managementRequest( + project: string, + request: ApiRequester, + path: string, + init: ApiRequestInit, +): Promise { + try { + return await request(path, init); + } catch (error) { + throw adoptionHint(project, error); + } +} + +export async function reconcileResources( + project: string, + resources: KubernetesResource[], + rolloutTimeoutMs: number, + postApply: + | ((resources: KubernetesResource[]) => void | Promise) + | undefined, + request: ApiRequester = apiRequest, +): Promise { + const workspacePath = `/workspaces/${encodeURIComponent(project)}`; + const plan = await managementRequest( + project, + request, + `${workspacePath}/resources/plan`, + { method: "POST", json: { resources } }, + ); + await managementRequest( + project, + request, + `${workspacePath}/resources/apply`, + { + method: "POST", + json: { resources: plan.desired }, + }, + ); + await postApply?.(plan.desired); + + const deployments = getDeploymentNames(plan.desired); + if (deployments.length > 0) { + await managementRequest( + project, + request, + `${workspacePath}/resources/wait`, + { + method: "POST", + json: { deployments, timeoutMs: rolloutTimeoutMs }, + }, + ); + } + if (plan.stale.length > 0) { + await managementRequest( + project, + request, + `${workspacePath}/resources/delete`, + { + method: "POST", + json: { resources: plan.stale }, + }, + ); + } + return plan; +} + +export async function runUp( + build: boolean, + request: ApiRequester = apiRequest, +) { const { project, compose, cwd, config, hookContext: getHookContext } = ctx(); - await assertManagedNamespace(project); + const workspacePath = `/workspaces/${encodeURIComponent(project)}`; const taskCtx = await new Listr( [ @@ -55,18 +207,21 @@ export async function runUp(build: boolean) { title: "Read compose", task: async (taskCtx, task) => { taskCtx.compose = await compose(); - if (build) { + if (build) await config.preBuild?.(taskCtx.compose, await getHookContext()); - } task.output = `${Object.keys(taskCtx.compose.services ?? {}).length} services`; }, }, { - title: "Build images", - rendererOptions: { - outputBar: 10, - persistentOutput: true, + title: "Snapshot workspace", + task: async (taskCtx, task) => { + taskCtx.snapshot = await enumerateWorkspace(await getRepoRoot(cwd)); + task.output = `${taskCtx.snapshot.manifest.files.length} files`; }, + }, + { + title: "Build images", + rendererOptions: { outputBar: 10, persistentOutput: true }, enabled: async () => build && Object.values((await compose()).services ?? {}).some( @@ -83,11 +238,11 @@ export async function runUp(build: boolean) { }, stream: task.stdout(), }, - config, + { ...config, request, snapshot: taskCtx.snapshot }, ); taskCtx.buildImages = result.images; await config.postBuild?.(result, await getHookContext()); - task.output = `Built ${result.built.length} image${result.built.length === 1 ? "" : "s"}, ${result.changed.length} changed`; + task.output = `Built ${result.built.length} image${result.built.length === 1 ? "" : "s"}`; }, }, { @@ -101,9 +256,33 @@ export async function runUp(build: boolean) { taskCtx.buildImages = await resolveBuildImages( project, taskCtx.compose!, - config, + { + ...config, + request, + }, ); - task.output = `${Object.keys(taskCtx.buildImages).length} image${Object.keys(taskCtx.buildImages).length === 1 ? "" : "s"}`; + task.output = `${Object.keys(taskCtx.buildImages).length} images`; + }, + }, + { + title: "Update workspace", + task: async (taskCtx, task) => { + const workspace = await ensureWorkspace( + project, + taskCtx.compose!, + taskCtx.snapshot!, + request, + ); + const adopted = await request<{ resourcesAdopted: number }>( + workspaceAdoptionRoute(project), + { + method: WORKSPACE_ADOPTION_METHOD, + json: { workspaceUid: workspace.metadata.uid }, + }, + ); + task.output = adopted.resourcesAdopted + ? `Adopted ${adopted.resourcesAdopted} existing resources` + : "Workspace ready"; }, }, { @@ -111,22 +290,34 @@ export async function runUp(build: boolean) { enabled: async () => getComposePostgresClaims(await compose()).length > 0, task: async (taskCtx, task) => { + const response = await managementRequest>( + project, + request, + `${workspacePath}/databases`, + { method: "POST", json: { compose: taskCtx.compose } }, + ); taskCtx.serviceEnv = mergeServiceEnv( taskCtx.serviceEnv, - await reconcilePostgresClaims(project, taskCtx.compose!), + operationEnvironment(response), ); - task.output = `${Object.keys(taskCtx.serviceEnv).length} service${Object.keys(taskCtx.serviceEnv).length === 1 ? "" : "s"}`; + task.output = `${Object.keys(taskCtx.serviceEnv).length} services`; }, }, { title: "Reconcile S3 storage", enabled: async () => getComposeS3Claims(await compose()).length > 0, task: async (taskCtx, task) => { + const response = await managementRequest>( + project, + request, + `${workspacePath}/storage`, + { method: "POST", json: { compose: taskCtx.compose } }, + ); taskCtx.serviceEnv = mergeServiceEnv( taskCtx.serviceEnv, - await reconcileS3Claims(project, taskCtx.compose!), + operationEnvironment(response), ); - task.output = `${Object.keys(taskCtx.serviceEnv).length} service${Object.keys(taskCtx.serviceEnv).length === 1 ? "" : "s"}`; + task.output = `${Object.keys(taskCtx.serviceEnv).length} services`; }, }, { @@ -147,89 +338,33 @@ export async function runUp(build: boolean) { }, }, { - title: "Plan reconciliation", + title: "Reconcile resources", task: async (taskCtx, task) => { - taskCtx.staleResources = (await getStaleResources( + taskCtx.plan = await reconcileResources( project, taskCtx.resources!, - )) as KubernetesResource[]; - task.output = `${taskCtx.staleResources.length} stale resource${taskCtx.staleResources.length === 1 ? "" : "s"}`; - }, - }, - { - title: "Apply resources", - task: (taskCtx, task) => { - const resources = sortResources(taskCtx.resources!); - - task.output = `${resources.length} resources queued`; - return task.newListr( - resources.map((resource) => ({ - title: `${resource.kind} ${resource.metadata?.name}`, - task: () => applyResource(resource), - })), - { concurrent: false, exitOnError: true }, - ); - }, - }, - { - title: "Run post-apply hook", - enabled: () => Boolean(config.postApply), - task: async (taskCtx) => { - await config.postApply?.( - taskCtx.resources! as KuberResource[], - await getHookContext(), + config.rolloutTimeoutMs, + config.postApply + ? async (resources) => + config.postApply?.( + resources as KuberResource[], + await getHookContext(), + ) + : undefined, + request, ); + task.output = `${taskCtx.plan.desired.length} applied, ${taskCtx.plan.stale.length} stale deleted`; }, }, ], { rendererOptions: { collapseErrors: false } }, ).run(); - const deployments = getDeploymentNames(taskCtx.resources!); - if (deployments.length > 0) { - await new Listr([ - { - title: "Wait for rollout", - task: (_taskCtx, task) => { - task.output = `${deployments.length} deployments queued`; - return task.newListr( - deployments.map((name) => ({ - title: `Deployment ${name}`, - task: () => - waitForDeploymentRollout(name, config.rolloutTimeoutMs), - })), - { concurrent: false, exitOnError: true }, - ); - }, - }, - ]).run(); - } - - if (taskCtx.staleResources && taskCtx.staleResources.length > 0) { - const resources = sortResources(taskCtx.staleResources).reverse(); - await new Listr([ - { - title: "Delete stale resources", - task: (_taskCtx, task) => { - task.output = `${resources.length} resources queued`; - return task.newListr( - resources.map((resource) => ({ - title: `${resource.kind} ${resource.metadata?.name}`, - task: () => deleteResource(resource), - })), - { concurrent: false, exitOnError: true }, - ); - }, - }, - ]).run(); - } + return taskCtx; } export const up = defineCommand({ - meta: { - name: "up", - description: "Create and start deployments", - }, + meta: { name: "up", description: "Create and start deployments" }, args: { build: { type: "boolean", diff --git a/command/users.ts b/command/users.ts new file mode 100644 index 0000000..b17de2e --- /dev/null +++ b/command/users.ts @@ -0,0 +1,253 @@ +import { stdin, stdout } from "node:process"; +import { createInterface } from "node:readline/promises"; +import { defineCommand } from "citty"; +import { apiRequest, type ApiRequestInit } from "../lib/api"; +import { toTable } from "../lib/format"; +import type { + CreateUserRequest, + ListUsersResponse, + UpdateUserRequest, + User, + UserResponse, + UserRole, +} from "../shared/api"; + +export type UsersApiRequest = ( + path: string, + init?: ApiRequestInit, +) => Promise; + +function requireUsername(value: unknown): string { + const username = String(value ?? "").trim(); + if (!username) throw new Error("Username is required"); + return username; +} + +function parseRoles(value: unknown): UserRole[] { + const roles = String(value ?? "") + .split(",") + .map((role) => role.trim()) + .filter(Boolean); + if (roles.length === 0) { + throw new Error( + "At least one role is required (for example: --roles admin)", + ); + } + const invalid = roles.find( + (role) => !["viewer", "operator", "admin"].includes(role), + ); + if (invalid) throw new Error(`Unknown role: ${invalid}`); + return [...new Set(roles)]; +} + +function renderUsers(users: User[]): string { + if (users.length === 0) return "No users"; + return toTable( + users.map((user) => ({ + username: user.username, + roles: user.roles.join(","), + disabled: user.disabled ? "yes" : "no", + updated: user.updatedAt, + })), + ); +} + +async function promptPassword(label = "Password: "): Promise { + if (!stdin.isTTY || !stdin.setRawMode) { + throw new Error("Password input requires an interactive terminal"); + } + + stdout.write(label); + stdin.setRawMode(true); + stdin.resume(); + return new Promise((resolve, reject) => { + let password = ""; + const cleanup = () => { + stdin.off("data", onData); + stdin.setRawMode(false); + stdin.pause(); + stdout.write("\n"); + }; + const finish = (error?: Error) => { + cleanup(); + if (error) reject(error); + else if (!password) reject(new Error("Password is required")); + else resolve(password); + }; + const onData = (chunk: Buffer | string) => { + for (const value of chunk.toString()) { + if (value === "\u0003" || value === "\u0004") { + finish(new Error("Password input cancelled")); + return; + } + if (value === "\r" || value === "\n") { + finish(); + return; + } + if (value === "\u007f" || value === "\b") + password = password.slice(0, -1); + else password += value; + } + }; + stdin.on("data", onData); + }); +} + +async function confirmDeletion(username: string): Promise { + if (!stdin.isTTY) { + throw new Error("Confirmation requires an interactive terminal; use --yes"); + } + const readline = createInterface({ input: stdin, output: stdout }); + try { + const answer = await readline.question(`Delete user '${username}'? [y/N] `); + return answer.trim().toLowerCase() === "y"; + } finally { + readline.close(); + } +} + +export async function listUsers( + request: UsersApiRequest = apiRequest, +): Promise { + const response = await request("/users"); + return renderUsers(response.items); +} + +export async function addUser( + username: string, + password: string, + roles: UserRole[], + request: UsersApiRequest = apiRequest, +): Promise { + const body: CreateUserRequest = { username, password, roles }; + const user = await request("/users", { + method: "POST", + json: body, + }); + return renderUsers([user]); +} + +export async function updateUser( + username: string, + update: UpdateUserRequest, + request: UsersApiRequest = apiRequest, +): Promise { + if (Object.keys(update).length === 0) + throw new Error("No user updates specified"); + const user = await request( + `/users/${encodeURIComponent(username)}`, + { method: "PATCH", json: update }, + ); + return renderUsers([user]); +} + +export function setUserDisabled( + username: string, + disabled: boolean, + request: UsersApiRequest = apiRequest, +): Promise { + return updateUser(username, { disabled }, request); +} + +export async function deleteUser( + username: string, + confirmed: boolean, + request: UsersApiRequest = apiRequest, +): Promise { + if (!confirmed) return "Deletion cancelled"; + await request(`/users/${encodeURIComponent(username)}`, { + method: "DELETE", + }); + return `Deleted user ${username}`; +} + +export async function revokeUserSessions( + username: string, + request: UsersApiRequest = apiRequest, +): Promise { + const result = await request<{ username: string; revoked: number }>( + `/users/${encodeURIComponent(username)}/sessions/revoke`, + { method: "POST" }, + ); + return `Revoked ${result.revoked} session${result.revoked === 1 ? "" : "s"} for ${result.username}`; +} + +const list = defineCommand({ + meta: { name: "ls", description: "List users" }, + async run() { + console.log(await listUsers()); + }, +}); + +const add = defineCommand({ + meta: { name: "add", description: "Add a user" }, + args: { + roles: { type: "string", description: "Comma-separated roles" }, + }, + async run({ args }) { + const username = requireUsername(args._[0]); + console.log( + await addUser(username, await promptPassword(), parseRoles(args.roles)), + ); + }, +}); + +const update = defineCommand({ + meta: { name: "update", description: "Update a user's roles or password" }, + args: { + roles: { type: "string", description: "Comma-separated roles" }, + password: { type: "boolean", description: "Prompt for a new password" }, + }, + async run({ args }) { + const username = requireUsername(args._[0]); + const body: UpdateUserRequest = {}; + if (args.roles !== undefined) body.roles = parseRoles(args.roles); + if (args.password) body.password = await promptPassword("New password: "); + console.log(await updateUser(username, body)); + }, +}); + +function disabledCommand(name: "disable" | "enable", disabled: boolean) { + return defineCommand({ + meta: { name, description: `${disabled ? "Disable" : "Enable"} a user` }, + async run({ args }) { + console.log(await setUserDisabled(requireUsername(args._[0]), disabled)); + }, + }); +} + +const remove = defineCommand({ + meta: { name: "delete", description: "Delete a user" }, + args: { + yes: { + type: "boolean", + alias: "y", + description: "Delete without confirmation", + }, + }, + async run({ args }) { + const username = requireUsername(args._[0]); + const confirmed = Boolean(args.yes) || (await confirmDeletion(username)); + console.log(await deleteUser(username, confirmed)); + }, +}); + +const revoke = defineCommand({ + meta: { name: "revoke", description: "Revoke all sessions for a user" }, + async run({ args }) { + console.log(await revokeUserSessions(requireUsername(args._[0]))); + }, +}); + +export const users = defineCommand({ + meta: { name: "users", description: "Administer users" }, + subCommands: { + add, + delete: remove, + disable: disabledCommand("disable", true), + enable: disabledCommand("enable", false), + ls: list, + revoke, + update, + }, +}); diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..e365b7c --- /dev/null +++ b/compose.yml @@ -0,0 +1,70 @@ +name: kuber-system + +services: + kuber-server: + build: + context: . + dockerfile: Dockerfile.server + environment: + PORT: "3000" + KUBER_BOOTSTRAP_USERNAME: dmgnr + KUBER_DATA_ROOT: /data + KUBER_BUILD_DATA_CLAIM: kuber-system-kuber-build-data + KUBER_REGISTRY_RESOLVE_ORIGIN: http://cncf-distribution-svc.registry.svc.cluster.local:5000 + KUBER_INTERNAL_REGISTRY_HOST: cncf-distribution-svc.registry.svc.cluster.local:5000 + KUBER_INTERNAL_REGISTRY_INSECURE: "true" + ports: + - kuber.astrxl.dev:3000 + deploy: + replicas: 1 + volumes: + - kuber-build-data:/data + x-container: + env: + - name: KUBER_BOOTSTRAP_PASSWORD + valueFrom: + secretKeyRef: + name: kuber-bootstrap + key: password + optional: true + readinessProbe: + httpGet: + path: /api/v2/health + port: 3000 + initialDelaySeconds: 2 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /api/v2/health + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + x-deployment: + spec: + template: + spec: + serviceAccountName: kuber-server + +volumes: + kuber-build-data: + x-size: 20Gi + x-diskTag: fast + x-replicaCount: 2 + x-dataLocality: none diff --git a/const.ts b/const.ts index 188bae3..dce3cce 100644 --- a/const.ts +++ b/const.ts @@ -1,4 +1,6 @@ export const IMAGE_REGISTRY = "registry.neko-piranha.ts.net"; +export const KUBER_API_ORIGIN = "https://kuber.astrxl.dev"; +export const KUBER_API_BASE_URL = `${KUBER_API_ORIGIN}/api/v2`; export const LABELS = { "app.kubernetes.io/managed-by": "kuber", }; diff --git a/example/README.md b/example/README.md index 796fdc5..56f203f 100644 --- a/example/README.md +++ b/example/README.md @@ -33,7 +33,11 @@ kuber logs -f app The current directory name determines the Kubernetes namespace and forms part of built image names. The Compose `name` field does not override that kuber behavior. -Docker and `kubectl` are not required on the workstation. kuber uses the configured SSH builder for image builds and the Kubernetes client with `~/.kube/config` for cluster operations. +Docker and `kubectl` are not required on the workstation. Image builds happen +inside the cluster: kuber uploads the repository snapshot to the management +service, which runs a rootless BuildKit Job that builds and pushes the image. +Cluster operations go through the authenticated v2 API, not a local +`~/.kube/config`. ## Image Build diff --git a/index.ts b/index.ts index 6eeeeef..88c8dfb 100644 --- a/index.ts +++ b/index.ts @@ -11,6 +11,11 @@ async function run() { wrapCommandErrors(main); const cli = createMain(main); const { configPath, rawArgs } = extractConfigArgument(process.argv.slice(2)); + const contextFree = new Set(["login", "logout", "whoami"]); + if (rawArgs[0] && contextFree.has(rawArgs[0])) { + await cli({ rawArgs }); + return; + } await provideContext(() => cli({ rawArgs }), configPath); } diff --git a/lib/api.ts b/lib/api.ts new file mode 100644 index 0000000..4c6c92b --- /dev/null +++ b/lib/api.ts @@ -0,0 +1,288 @@ +import { KUBER_API_BASE_URL } from "../const"; +import type { ApiProblemDetails } from "../shared/api"; +import { readSession, type KuberSession } from "./session"; + +export type { ApiProblemDetails } from "../shared/api"; + +export const DEFAULT_API_TIMEOUT_MS = 30_000; + +export type ApiRequestInit = RequestInit & { + /** JSON is serialized here so callers do not need to manage content headers. */ + json?: unknown; +}; + +export type ApiRequestOptions = { + authenticated?: boolean; + baseUrl?: string; + session?: KuberSession; + /** Set to 0 to disable the deadline, primarily for long-lived streams. */ + timeoutMs?: number; + /** Byte offset for resumable binary upload requests. */ + uploadOffset?: number; +}; + +export type ApiUploadOptions = ApiRequestOptions & { + offset: number; + contentType?: string; +}; + +export class KuberApiError extends Error { + readonly code: string; + readonly requestId?: string; + readonly operationId?: string; + readonly problem: ApiProblemDetails; + + constructor(message: string, status: number, problem?: ApiProblemDetails) { + super(message); + this.name = "KuberApiError"; + this.status = status; + this.code = problem?.code ?? `HTTP_${status}`; + this.requestId = problem?.requestId; + this.operationId = problem?.operationId; + this.problem = { + ...problem, + status, + title: problem?.title ?? message, + code: this.code, + }; + } + + readonly status: number; +} + +type RequestDeadline = { + signal: AbortSignal; + clear: () => void; +}; + +function requestDeadline( + signal: AbortSignal | null | undefined, + timeoutMs = DEFAULT_API_TIMEOUT_MS, +): RequestDeadline { + if (!Number.isFinite(timeoutMs) || timeoutMs < 0) { + throw new RangeError("timeoutMs must be a finite non-negative number"); + } + + const controller = new AbortController(); + const abortFromCaller = () => controller.abort(signal?.reason); + if (signal?.aborted) abortFromCaller(); + else signal?.addEventListener("abort", abortFromCaller, { once: true }); + + const timer = + timeoutMs === 0 + ? undefined + : setTimeout(() => { + controller.abort( + new DOMException( + `API request timed out after ${timeoutMs}ms`, + "TimeoutError", + ), + ); + }, timeoutMs); + + return { + signal: controller.signal, + clear: () => { + if (timer !== undefined) clearTimeout(timer); + signal?.removeEventListener("abort", abortFromCaller); + }, + }; +} + +type FetchBody = NonNullable; + +function isBinaryBody(body: FetchBody): boolean { + return ( + body instanceof ArrayBuffer || + ArrayBuffer.isView(body) || + (typeof Blob !== "undefined" && body instanceof Blob) || + (typeof ReadableStream !== "undefined" && body instanceof ReadableStream) + ); +} + +async function requestHeaders( + init: ApiRequestInit, + options: ApiRequestOptions, +): Promise { + const headers = new Headers(init.headers); + if (!headers.has("accept")) headers.set("accept", "application/json"); + + if (Object.hasOwn(init, "json")) { + headers.set("content-type", "application/json"); + } else if ( + init.body !== undefined && + init.body !== null && + !headers.has("content-type") && + !isBinaryBody(init.body) + ) { + // Preserve the original apiRequest convention: string bodies are JSON. + headers.set("content-type", "application/json"); + } + + if (options.uploadOffset !== undefined) { + if ( + !Number.isSafeInteger(options.uploadOffset) || + options.uploadOffset < 0 + ) { + throw new RangeError("uploadOffset must be a non-negative safe integer"); + } + headers.set("upload-offset", String(options.uploadOffset)); + } + + if (options.authenticated !== false) { + const session = options.session ?? (await readSession()); + if (!session) throw new Error("Not logged in. Run kuber login first."); + headers.set("authorization", `Bearer ${session.token}`); + } + return headers; +} + +async function responseProblem(response: Response): Promise { + let problem: ApiProblemDetails | undefined; + try { + const value: unknown = JSON.parse(await response.text()); + if (value && typeof value === "object") { + problem = value as ApiProblemDetails; + } + } catch { + // The status and response headers still provide a stable error shape. + } + + return { + ...problem, + status: response.status, + title: problem?.title || response.statusText || `HTTP ${response.status}`, + code: problem?.code || `HTTP_${response.status}`, + requestId: + problem?.requestId ?? response.headers.get("x-request-id") ?? undefined, + operationId: + problem?.operationId ?? + response.headers.get("x-operation-id") ?? + undefined, + }; +} + +async function assertResponseOk(response: Response): Promise { + if (response.ok) return; + const problem = await responseProblem(response); + throw new KuberApiError( + problem.detail || problem.message || problem.title, + response.status, + problem, + ); +} + +async function sendRequest( + path: string, + init: ApiRequestInit, + options: ApiRequestOptions, + signal: AbortSignal, +): Promise { + const headers = await requestHeaders(init, options); + const { json, ...requestInit } = init; + const body = Object.hasOwn(init, "json") ? JSON.stringify(json) : init.body; + return fetch(`${options.baseUrl ?? KUBER_API_BASE_URL}${path}`, { + ...requestInit, + body, + headers, + signal, + }); +} + +export async function apiRequest( + path: string, + init: ApiRequestInit = {}, + options: ApiRequestOptions = {}, +): Promise { + const deadline = requestDeadline(init.signal, options.timeoutMs); + try { + const response = await sendRequest(path, init, options, deadline.signal); + await assertResponseOk(response); + + if ( + init.method?.toUpperCase() === "HEAD" || + response.status === 204 || + response.status === 205 || + response.body === null + ) { + return undefined as T; + } + + const text = await response.text(); + if (!text.trim()) return undefined as T; + return JSON.parse(text) as T; + } finally { + deadline.clear(); + } +} + +export function apiUpload( + path: string, + body: Blob | ArrayBuffer | ArrayBufferView | ReadableStream, + options: ApiUploadOptions, +): Promise { + const { + contentType = "application/octet-stream", + offset, + ...requestOptions + } = options; + return apiRequest( + path, + { + method: "PATCH", + headers: { "content-type": contentType }, + body: body as FetchBody, + }, + { ...requestOptions, uploadOffset: offset }, + ); +} + +/** Parses records as they arrive instead of buffering the complete response. */ +export async function* apiStreamNdjson( + path: string, + init: ApiRequestInit = {}, + options: ApiRequestOptions = {}, +): AsyncGenerator { + const deadline = requestDeadline(init.signal, options.timeoutMs); + try { + const headers = new Headers(init.headers); + headers.set("accept", "application/x-ndjson"); + const response = await sendRequest( + path, + { ...init, headers }, + options, + deadline.signal, + ); + await assertResponseOk(response); + if (!response.body) return; + + const reader = response.body.getReader(); + try { + const decoder = new TextDecoder(); + let buffer = ""; + for (;;) { + const { done, value } = await reader.read(); + buffer += decoder.decode(value, { stream: !done }); + let newline = buffer.indexOf("\n"); + while (newline !== -1) { + const line = buffer.slice(0, newline).replace(/\r$/, "").trim(); + buffer = buffer.slice(newline + 1); + if (line) yield JSON.parse(line) as T; + newline = buffer.indexOf("\n"); + } + if (done) break; + } + const finalLine = buffer.replace(/\r$/, "").trim(); + if (finalLine) yield JSON.parse(finalLine) as T; + } finally { + try { + await reader.cancel(); + } catch { + // Cancellation can race with an upstream abort. + } + reader.releaseLock(); + } + } finally { + deadline.clear(); + } +} diff --git a/lib/apply.ts b/lib/apply.ts index 91cfe4a..3b6e876 100644 --- a/lib/apply.ts +++ b/lib/apply.ts @@ -9,14 +9,19 @@ const ResourceOrder = { Namespace: 0, GarageBucket: 1, GarageKey: 2, - StorageClass: 3, - PersistentVolumeClaim: 4, - Secret: 5, - ConfigMap: 6, - Service: 7, - Deployment: 8, - Ingress: 9, - IngressRoute: 10, + ServiceAccount: 3, + ClusterRole: 4, + Role: 5, + ClusterRoleBinding: 6, + RoleBinding: 7, + StorageClass: 8, + PersistentVolumeClaim: 9, + Secret: 10, + ConfigMap: 11, + Service: 12, + Deployment: 13, + Ingress: 14, + IngressRoute: 15, } as const; const ManagedResources = [ diff --git a/lib/build.ts b/lib/build.ts index e1dbf1c..0f60d6a 100644 --- a/lib/build.ts +++ b/lib/build.ts @@ -1,41 +1,42 @@ -import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; -import { cp, mkdir, mkdtemp, rm } from "node:fs/promises"; -import { hostname, tmpdir } from "node:os"; -import { AsyncLocalStorage } from "node:async_hooks"; +import { randomUUID } from "node:crypto"; +import { execFile } from "node:child_process"; +import { isAbsolute, relative, resolve, sep } from "node:path"; +import { promisify } from "node:util"; import type { Writable } from "node:stream"; import type { ComposeSpecification, Service } from "../schema/docker.d"; -import { getComposeArch, withComposeArch } from "./arch"; -import { DEFAULT_BUILDERS, DEFAULT_REGISTRY } from "./config"; +import { + BUILD_PROTOCOL_VERSION, + type BuildEvent, + type BuildRequest, + type BuildStatus, + type Sha256Digest, +} from "../shared/build-protocol"; +import { resolveComposeArch } from "./arch"; +import { apiRequest, type ApiRequestInit } from "./api"; +import { DEFAULT_REGISTRY } from "./config"; +import { + enumerateWorkspace, + serializeWorkspaceManifest, + type WorkspaceSnapshot, +} from "./workspace"; + +const execFileAsync = promisify(execFile); +const UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024; +const DEFAULT_POLL_INTERVAL_MS = 1_000; + +export type ApiRequester = ( + path: string, + init?: ApiRequestInit, +) => Promise; export type BuildOptions = { registry?: string; - builders?: { - amd64?: string; - arm64?: string; - remoteRoot?: string; - }; + request?: ApiRequester; + pollIntervalMs?: number; + sleep?: (milliseconds: number) => Promise; + snapshot?: WorkspaceSnapshot; }; -type BuildRuntime = { - registry: string; - builders: { - amd64: string; - arm64: string; - remoteRoot: string; - }; -}; - -const buildRuntime = new AsyncLocalStorage(); - -function getBuildRuntime(): BuildRuntime { - return ( - buildRuntime.getStore() ?? { - registry: DEFAULT_REGISTRY, - builders: DEFAULT_BUILDERS, - } - ); -} - type BuildPlan = { name: string; image: string; @@ -57,36 +58,35 @@ export type BuildResult = { images: Record; }; -type SpawnResult = { - exitCode: number; - stdout: string; - stderr: string; +type SnapshotNegotiation = { + workspace: Sha256Digest; + missing: Sha256Digest[]; + ready: boolean; }; -function summarizeCommandFailure( - exitCode: number, - stderrLines: string[], - stdoutLines: string[], +type ImageResult = { + image: string; + digest: Sha256Digest; + reference: string; +}; + +function posixRelative(root: string, path: string): string { + return relative(root, path).split(sep).join("/") || "."; +} + +function assertInsideRepo( + repoRoot: string, + path: string, + description: string, + service: string, ): string { - const lines = (stderrLines.length > 0 ? stderrLines : stdoutLines) - .map((line) => line.trimEnd()) - .filter(Boolean); - if (lines.length === 0) return `Command failed with exit code ${exitCode}`; - - const preview = lines.slice(0, 8).join("\n"); - return lines.length > 8 - ? `${preview}\n... (${lines.length - 8} more lines)` - : preview; -} - -function toReadableStream( - stream: number | ReadableStream | undefined, -): ReadableStream | undefined { - return typeof stream === "number" ? undefined : stream; -} - -function shellQuote(value: string): string { - return `'${value.replaceAll("'", `'"'"'`)}'`; + const value = relative(repoRoot, path); + if (value.startsWith(`..${sep}`) || value === ".." || isAbsolute(value)) { + throw new Error( + `${description} must stay inside the git repo for service ${service}`, + ); + } + return posixRelative(repoRoot, path); } function resolveBuildArgs(service: Service): string[] { @@ -94,12 +94,9 @@ function resolveBuildArgs(service: Service): string[] { !service.build || typeof service.build === "string" || !service.build.args - ) { + ) return []; - } - if (Array.isArray(service.build.args)) return [...service.build.args]; - return Object.entries(service.build.args) .filter(([, value]) => value !== null) .map(([key, value]) => `${key}=${String(value)}`); @@ -111,238 +108,66 @@ function resolveBuildPlan( service: Service, cwd: string, repoRoot: string, - buildRoot: string, + registry: string, ): BuildPlan | undefined { if (!service.build) return; - const build = service.build; const contextInput = typeof build === "string" ? build : (build.context ?? "."); - - if (contextInput.includes("://")) { + if (contextInput.includes("://")) throw new Error( `Remote build context is not supported for service ${name}`, ); - } + if (typeof build !== "string" && build.dockerfile_inline) + throw new Error(`dockerfile_inline is not supported for service ${name}`); const contextPath = resolve(cwd, contextInput); - const contextRelative = relative(repoRoot, contextPath); - if (contextRelative.startsWith("..") || isAbsolute(contextRelative)) { - throw new Error( - `Build context must stay inside the git repo for service ${name}`, - ); - } - - if (typeof build !== "string" && build.dockerfile_inline) { - throw new Error(`dockerfile_inline is not supported for service ${name}`); - } - + const context = assertInsideRepo( + repoRoot, + contextPath, + "Build context", + name, + ); const dockerfilePath = typeof build === "string" || !build.dockerfile ? undefined : resolve(contextPath, build.dockerfile); - const dockerfileRelative = dockerfilePath - ? relative(repoRoot, dockerfilePath) - : undefined; - - if ( - dockerfileRelative?.startsWith("..") || - isAbsolute(dockerfileRelative ?? "") - ) { - throw new Error( - `Dockerfile must stay inside the git repo for service ${name}`, - ); - } return { name, - image: getBuildImageName(project, name), - context: `${buildRoot}/${contextRelative === "" ? "." : contextRelative}`, - dockerfile: dockerfileRelative - ? `${buildRoot}/${dockerfileRelative}` + // The server replaces this requested name with its configured imageName. + image: getBuildImageName(project, name, registry), + context, + dockerfile: dockerfilePath + ? assertInsideRepo(repoRoot, dockerfilePath, "Dockerfile", name) : undefined, target: typeof build === "string" ? undefined : build.target, buildArgs: resolveBuildArgs(service), }; } -function getRemoteBuilder(): string { - const builders = getBuildRuntime().builders; - return getComposeArch() === "amd64" ? builders.amd64 : builders.arm64; -} - -function isOnRemoteBuilder(): boolean { - return hostname() === getRemoteBuilder().split("@").at(-1); -} - -async function pumpStream( - stream: ReadableStream | null | undefined, - onLine: (line: string) => void | Promise, -) { - if (!stream) return; - - const reader = stream.getReader(); - const decoder = new TextDecoder(); - let buffer = ""; - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - - let newline = buffer.indexOf("\n"); - while (newline !== -1) { - const line = buffer.slice(0, newline).replace(/\r$/, ""); - buffer = buffer.slice(newline + 1); - if (line) await onLine(line); - newline = buffer.indexOf("\n"); - } - } - - buffer += decoder.decode(); - const line = buffer.replace(/\r$/, ""); - if (line) await onLine(line); - } finally { - reader.releaseLock(); - } -} - -async function runWithOutput( - command: Bun.Subprocess, - reporter?: BuildReporter, -) { - const stdoutLines: string[] = []; - const stderrLines: string[] = []; - let streamBuffer = ""; - let flushTimer: ReturnType | undefined; - - function flushStreamBuffer() { - if (!reporter?.stream || streamBuffer.length === 0) return; - reporter.stream.write(streamBuffer); - streamBuffer = ""; - } - - function queueStreamLine(line: string) { - streamBuffer += `${line}\n`; - if (streamBuffer.length >= 8192) { - if (flushTimer) { - clearTimeout(flushTimer); - flushTimer = undefined; - } - flushStreamBuffer(); - return; - } - - if (flushTimer) return; - flushTimer = setTimeout(() => { - flushTimer = undefined; - flushStreamBuffer(); - }, 33); - } - - await Promise.all([ - pumpStream(toReadableStream(command.stdout), async (line) => { - stdoutLines.push(line); - if (reporter?.stream) queueStreamLine(line); - else if (reporter?.progress) await reporter.progress(line); - }), - pumpStream(toReadableStream(command.stderr), async (line) => { - stderrLines.push(line); - if (reporter?.stream) queueStreamLine(line); - else if (reporter?.progress) await reporter.progress(line); - }), - ]); - - if (flushTimer) { - clearTimeout(flushTimer); - flushTimer = undefined; - } - flushStreamBuffer(); - - const exitCode = await command.exited; - if (exitCode !== 0) { - throw new Error( - summarizeCommandFailure(exitCode, stderrLines, stdoutLines), - ); - } - - return { - exitCode, - stdout: stdoutLines.join("\n"), - stderr: stderrLines.join("\n"), - } satisfies SpawnResult; -} - -function ssh(script: string) { - const remoteCommand = `bash -lc ${shellQuote(script)}`; - return Bun.spawn(["ssh", getRemoteBuilder(), remoteCommand], { - stdout: "pipe", - stderr: "pipe", - }); -} - -function scp(localPath: string, remotePath: string) { - return Bun.spawn(["scp", localPath, `${getRemoteBuilder()}:${remotePath}`], { - stdout: "pipe", - stderr: "pipe", - }); -} - export function parseImageManifestDigest(output: string): string { const manifest = JSON.parse(output) as { digest?: unknown }; if ( typeof manifest.digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(manifest.digest) - ) { + ) throw new Error("Registry response did not contain a valid image digest"); - } return manifest.digest; } export function toPinnedImage(image: string, digest: string): string { - if (!/^sha256:[a-f0-9]{64}$/.test(digest)) { + if (!/^sha256:[a-f0-9]{64}$/.test(digest)) throw new Error(`Invalid image digest ${digest}`); - } return `${image}@${digest}`; } export function getBuildImageName( project: string, service: string, - registry = getBuildRuntime().registry, + registry = DEFAULT_REGISTRY, ): string { - return `${registry}/kuber/${project}-${service}:latest`; -} - -async function inspectRemoteImageDigest(image: string): Promise { - const inspectCommand = [ - "docker", - "buildx", - "imagetools", - "inspect", - image, - "--format", - "{{json .Manifest}}", - ] - .map(shellQuote) - .join(" "); - - const result = await runWithOutput( - ssh(`set -euo pipefail; ${inspectCommand}`), - ); - return parseImageManifestDigest(result.stdout); -} - -async function tryInspectRemoteImageDigest( - image: string, -): Promise { - try { - return await inspectRemoteImageDigest(image); - } catch { - return; - } + return `${registry.replace(/\/+$/, "")}/kuber/${project}-${service}:latest`; } export function imageDigestChanged( @@ -352,15 +177,126 @@ export function imageDigestChanged( return !before || !after || before !== after; } -function resolveBuildRuntime(options: BuildOptions): BuildRuntime { - return { - registry: options.registry ?? DEFAULT_REGISTRY, - builders: { - amd64: options.builders?.amd64 ?? DEFAULT_BUILDERS.amd64, - arm64: options.builders?.arm64 ?? DEFAULT_BUILDERS.arm64, - remoteRoot: options.builders?.remoteRoot ?? DEFAULT_BUILDERS.remoteRoot, - }, - }; +export async function getRepoRoot(cwd: string): Promise { + const { stdout } = await execFileAsync("git", [ + "-C", + cwd, + "rev-parse", + "--show-toplevel", + ]); + return stdout.trim(); +} + +async function uploadBlob( + digest: Sha256Digest, + data: Uint8Array, + request: ApiRequester, +): Promise { + const path = `/blobs/${encodeURIComponent(digest)}/uploads`; + const progress = await request<{ offset: number; complete: boolean }>(path, { + method: "POST", + json: { size: data.byteLength }, + }); + let offset = progress.offset; + while (!progress.complete && offset < data.byteLength) { + const chunk = data.subarray(offset, offset + UPLOAD_CHUNK_BYTES); + const uploaded = await request<{ offset: number }>(path, { + method: "PATCH", + headers: { + "content-type": "application/octet-stream", + "upload-offset": String(offset), + }, + body: chunk, + }); + if (uploaded.offset <= offset) + throw new Error(`Blob upload for ${digest} made no progress`); + offset = uploaded.offset; + } + if (!progress.complete) { + await request(`${path}/complete`, { method: "POST", json: {} }); + } +} + +export async function uploadWorkspaceSnapshot( + snapshot: WorkspaceSnapshot, + request: ApiRequester = apiRequest, + reporter?: BuildReporter, +): Promise { + const blobs = new Map(snapshot.blobs.map((blob) => [blob.digest, blob.data])); + blobs.set(snapshot.digest, serializeWorkspaceManifest(snapshot.manifest)); + + for (;;) { + const negotiation = await request( + "/snapshots/negotiate", + { + method: "POST", + json: { workspace: snapshot.digest }, + }, + ); + if (negotiation.ready) return; + if (negotiation.missing.length === 0) + throw new Error( + "Snapshot negotiation is incomplete but reported no missing blobs", + ); + for (const digest of negotiation.missing) { + const data = blobs.get(digest); + if (!data) + throw new Error(`Server requested unknown workspace blob ${digest}`); + await reporter?.progress?.(`Uploading ${digest}`); + await uploadBlob(digest, data, request); + } + } +} + +async function reportBuildEvent( + event: BuildEvent, + reporter?: BuildReporter, + reportedStates?: Set, +): Promise { + if (event.type === "status") { + if (!reportedStates?.has(event.status.state)) { + reportedStates?.add(event.status.state); + await reporter?.progress?.(`Build ${event.status.state}`); + } + return 0; + } + if (reporter?.stream) reporter.stream.write(event.message); + else await reporter?.progress?.(event.message.trimEnd()); + return event.sequence; +} + +async function waitForBuild( + id: string, + request: ApiRequester, + reporter: BuildReporter | undefined, + pollIntervalMs: number, + sleep: (milliseconds: number) => Promise, + initial: BuildStatus, +): Promise { + let status = initial; + let sequence = 0; + const reportedStates = new Set(); + for (;;) { + const events = await request( + `/builds/${encodeURIComponent(id)}/events?after=${sequence}`, + ); + for (const event of events) + sequence = Math.max( + sequence, + await reportBuildEvent(event, reporter, reportedStates), + ); + if (status.state === "succeeded" || status.state === "failed") + return status; + status = await request( + `/builds/${encodeURIComponent(id)}/reconcile`, + { + method: "POST", + json: {}, + }, + ); + if (status.state !== "succeeded" && status.state !== "failed") + await sleep(pollIntervalMs); + } } export async function resolveBuildImages( @@ -368,293 +304,25 @@ export async function resolveBuildImages( compose: ComposeSpecification, options: BuildOptions = {}, ): Promise> { - return buildRuntime.run(resolveBuildRuntime(options), () => - withComposeArch(compose, async () => { - const images: Record = {}; - for (const [name, service] of Object.entries(compose.services ?? {})) { - if (!service.build) continue; - const image = getBuildImageName(project, name); - try { - images[name] = toPinnedImage( - image, - await inspectRemoteImageDigest(image), - ); - } catch (error) { - throw new Error( - `Cannot resolve a published image for service ${name}. Run kuber up to build it.`, - { cause: error }, - ); - } - } - return images; - }), - ); -} - -async function getRepoRoot(cwd: string): Promise { - return Bun.$.cwd(cwd)`git rev-parse --show-toplevel` - .text() - .then((e) => e.trim()); -} - -async function getHeadSha(repoRoot: string): Promise { - return Bun.$.cwd(repoRoot)`git rev-parse HEAD`.text().then((e) => e.trim()); -} - -async function getTrackedDiff(repoRoot: string): Promise { - return Bun.$.cwd(repoRoot)`git diff --binary HEAD`.text(); -} - -async function getUntrackedFiles(repoRoot: string): Promise { - const result = Bun.spawn( - ["git", "ls-files", "--others", "--exclude-standard"], - { - cwd: repoRoot, - stdout: "pipe", - stderr: "pipe", - }, - ); - const output = await runWithOutput(result); - - return output.stdout - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); -} - -async function getIgnoredDotenvFiles(repoRoot: string): Promise { - const result = Bun.spawn( - [ - "git", - "ls-files", - "--others", - "--ignored", - "--exclude-standard", - "--", - ".env*", - "**/.env*", - ], - { - cwd: repoRoot, - stdout: "pipe", - stderr: "pipe", - }, - ); - const output = await runWithOutput(result); - - return output.stdout - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); -} - -async function getRemoteHead(remoteRepo: string): Promise { - const result = ssh( - `if [ -d ${shellQuote(`${remoteRepo}/.git`)} ]; then git -C ${shellQuote(remoteRepo)} rev-parse HEAD; fi`, - ); - - const output = await runWithOutput(result); - const head = output.stdout.trim(); - return head || undefined; -} - -async function syncRemoteRepo( - repoRoot: string, - remoteRepo: string, - reporter?: BuildReporter, -) { - const headSha = await getHeadSha(repoRoot); - const remoteHead = await getRemoteHead(remoteRepo); - const tempDir = await mkdtemp(`${tmpdir()}/kuber-build-`); - - try { - if (remoteHead !== headSha) { - await reporter?.progress?.("Transferring latest git bundle"); - const bundlePath = `${tempDir}/repo.bundle`; - const remoteBundle = `${getBuildRuntime().builders.remoteRoot}/${basename(repoRoot)}.bundle`; - - await Bun.$.cwd(repoRoot)`git bundle create ${bundlePath} HEAD`; - await runWithOutput( - ssh(`mkdir -p ${shellQuote(getBuildRuntime().builders.remoteRoot)}`), - reporter, - ); - await runWithOutput(scp(bundlePath, remoteBundle), reporter); - await runWithOutput( - ssh( - [ - "set -euo pipefail", - `mkdir -p ${shellQuote(remoteRepo)}`, - `if [ ! -d ${shellQuote(`${remoteRepo}/.git`)} ]; then git -C ${shellQuote(remoteRepo)} init; fi`, - `git -C ${shellQuote(remoteRepo)} fetch --force "$PWD/${remoteBundle}" HEAD`, - `git -C ${shellQuote(remoteRepo)} reset --hard FETCH_HEAD`, - `git -C ${shellQuote(remoteRepo)} clean -fd`, - ].join("; "), - ), - reporter, - ); - } else { - await runWithOutput( - ssh( - [ - "set -euo pipefail", - `git -C ${shellQuote(remoteRepo)} reset --hard HEAD`, - `git -C ${shellQuote(remoteRepo)} clean -fd`, - ].join("; "), - ), - reporter, + const request = options.request ?? apiRequest; + const images: Record = {}; + for (const [service, definition] of Object.entries(compose.services ?? {})) { + if (!definition.build) continue; + try { + images[service] = ( + await request("/images/resolve", { + method: "POST", + json: { project, service }, + }) + ).reference; + } catch (error) { + throw new Error( + `Cannot resolve a published image for service ${service}. Run kuber up to build it.`, + { cause: error }, ); } - - const diff = await getTrackedDiff(repoRoot); - if (diff.trim().length > 0) { - await reporter?.progress?.("Applying local git diff on remote builder"); - const diffPath = `${tempDir}/repo.diff`; - const remoteDiff = `${getBuildRuntime().builders.remoteRoot}/${basename(repoRoot)}.diff`; - await Bun.write(diffPath, diff); - await runWithOutput(scp(diffPath, remoteDiff), reporter); - await runWithOutput( - ssh( - [ - "set -euo pipefail", - `git -C ${shellQuote(remoteRepo)} apply --allow-binary-replacement "$PWD/${remoteDiff}"`, - ].join("; "), - ), - reporter, - ); - } - - const untrackedFiles = await getUntrackedFiles(repoRoot); - if (untrackedFiles.length > 0) { - await reporter?.progress?.("Syncing untracked files to remote builder"); - const untrackedDir = `${tempDir}/untracked`; - - for (const file of untrackedFiles) { - const source = resolve(repoRoot, file); - const target = resolve(untrackedDir, file); - await mkdir(dirname(target), { recursive: true }); - await cp(source, target, { force: true, recursive: true }); - } - - const untrackedArchive = `${tempDir}/untracked.tar`; - const remoteUntrackedArchive = `${getBuildRuntime().builders.remoteRoot}/${basename(repoRoot)}-untracked.tar`; - await Bun.$`tar -C ${untrackedDir} -cf ${untrackedArchive} .`; - await runWithOutput( - scp(untrackedArchive, remoteUntrackedArchive), - reporter, - ); - await runWithOutput( - ssh( - [ - "set -euo pipefail", - `tar -C ${shellQuote(remoteRepo)} -xf "$PWD/${remoteUntrackedArchive}"`, - ].join("; "), - ), - reporter, - ); - } - - const ignoredDotenvFiles = await getIgnoredDotenvFiles(repoRoot); - await runWithOutput( - ssh( - [ - "set -euo pipefail", - `git -C ${shellQuote(remoteRepo)} clean -fdX -- .env* '**/.env*'`, - ].join("; "), - ), - reporter, - ); - - if (ignoredDotenvFiles.length === 0) return; - - await reporter?.progress?.("Syncing ignored .env* files to remote builder"); - const dotenvDir = `${tempDir}/dotenv`; - - for (const file of ignoredDotenvFiles) { - const source = resolve(repoRoot, file); - const target = resolve(dotenvDir, file); - await mkdir(dirname(target), { recursive: true }); - await cp(source, target, { force: true }); - } - - const dotenvArchive = `${tempDir}/dotenv.tar`; - const remoteDotenvArchive = `${getBuildRuntime().builders.remoteRoot}/${basename(repoRoot)}-dotenv.tar`; - await Bun.$`tar -C ${dotenvDir} -cf ${dotenvArchive} .`; - await runWithOutput(scp(dotenvArchive, remoteDotenvArchive), reporter); - await runWithOutput( - ssh( - [ - "set -euo pipefail", - `tar -C ${shellQuote(remoteRepo)} -xf "$PWD/${remoteDotenvArchive}"`, - ].join("; "), - ), - reporter, - ); - } finally { - await rm(tempDir, { recursive: true, force: true }); } -} - -function getBuildPlans( - project: string, - compose: ComposeSpecification, - cwd: string, - repoRoot: string, - remoteRepo: string, -): BuildPlan[] { - return Object.entries(compose.services ?? {}).flatMap(([name, service]) => { - const plan = resolveBuildPlan( - project, - name, - service, - cwd, - repoRoot, - remoteRepo, - ); - return plan ? [plan] : []; - }); -} - -async function buildRemote(plan: BuildPlan, reporter?: BuildReporter) { - await reporter?.progress?.(`Building ${plan.name}`); - const args = [ - "docker", - "build", - "--push", - "--progress=plain", - "-t", - plan.image, - ...(plan.dockerfile ? ["-f", plan.dockerfile] : []), - ...(plan.target ? ["--target", plan.target] : []), - ...plan.buildArgs.flatMap((arg) => ["--build-arg", arg]), - plan.context, - ]; - - const command = args.map(shellQuote).join(" "); - await runWithOutput(ssh(`set -euo pipefail; ${command}`), reporter); -} - -async function buildLocal(plan: BuildPlan, reporter?: BuildReporter) { - await reporter?.progress?.(`Building ${plan.name}`); - const command = Bun.spawn( - [ - "docker", - "build", - "--push", - "--progress=plain", - "-t", - plan.image, - ...(plan.dockerfile ? ["-f", plan.dockerfile] : []), - ...(plan.target ? ["--target", plan.target] : []), - ...plan.buildArgs.flatMap((arg) => ["--build-arg", arg]), - plan.context, - ], - { - stdout: "pipe", - stderr: "pipe", - }, - ); - - await runWithOutput(command, reporter); + return images; } export async function buildServices( @@ -664,45 +332,70 @@ export async function buildServices( reporter?: BuildReporter, options: BuildOptions = {}, ): Promise { - const runtime = resolveBuildRuntime(options); + if (!Object.values(compose.services ?? {}).some((service) => service.build)) + return { built: [], changed: [], images: {} }; - return buildRuntime.run(runtime, () => - withComposeArch(compose, async () => { - if ( - !Object.values(compose.services ?? {}).some((service) => service.build) - ) { - return { built: [], changed: [], images: {} }; - } - - const repoRoot = await getRepoRoot(cwd); - const localBuilder = isOnRemoteBuilder(); - const buildRoot = localBuilder - ? repoRoot - : `${getBuildRuntime().builders.remoteRoot}/${basename(repoRoot)}`; - const plans = getBuildPlans(project, compose, cwd, repoRoot, buildRoot); - - if (plans.length === 0) return { built: [], changed: [], images: {} }; - - if (!localBuilder) { - await syncRemoteRepo(repoRoot, buildRoot, reporter); - } - - const changed: string[] = []; - const images: Record = {}; - for (const plan of plans) { - const before = await tryInspectRemoteImageDigest(plan.image); - if (localBuilder) await buildLocal(plan, reporter); - else await buildRemote(plan, reporter); - const after = await inspectRemoteImageDigest(plan.image); - images[plan.name] = toPinnedImage(plan.image, after); - if (imageDigestChanged(before, after)) changed.push(plan.name); - } - - return { - built: plans.map((plan) => plan.name), - changed, - images, - }; - }), + const request = options.request ?? apiRequest; + const repoRoot = await getRepoRoot(cwd); + const snapshot = options.snapshot ?? (await enumerateWorkspace(repoRoot)); + const plans = Object.entries(compose.services ?? {}).flatMap( + ([name, service]) => { + const plan = resolveBuildPlan( + project, + name, + service, + cwd, + repoRoot, + options.registry ?? DEFAULT_REGISTRY, + ); + return plan ? [plan] : []; + }, ); + await uploadWorkspaceSnapshot(snapshot, request, reporter); + + const images: Record = {}; + for (const plan of plans) { + await reporter?.progress?.(`Building ${plan.name}`); + const id = randomUUID(); + const buildRequest: BuildRequest = { + version: BUILD_PROTOCOL_VERSION, + id, + project, + service: plan.name, + spec: { + architecture: resolveComposeArch(compose), + image: plan.image, + context: plan.context, + dockerfile: plan.dockerfile, + target: plan.target, + buildArgs: plan.buildArgs, + workspace: snapshot.digest, + }, + }; + const initial = await request("/builds", { + method: "POST", + json: buildRequest, + }); + const status = await waitForBuild( + id, + request, + reporter, + options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS, + options.sleep ?? ((milliseconds) => Bun.sleep(milliseconds)), + initial, + ); + if (status.state !== "succeeded") + throw new Error( + `Build failed for service ${plan.name}: ${status.error ?? "unknown error"}`, + ); + images[plan.name] = ( + await request(`/builds/${encodeURIComponent(id)}/result`) + ).reference; + } + + return { + built: plans.map((plan) => plan.name), + changed: plans.map((plan) => plan.name), + images, + }; } diff --git a/lib/config.ts b/lib/config.ts index cc4c113..13b27fc 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -6,11 +6,6 @@ import { IMAGE_REGISTRY } from "../const"; const DEFAULT_CONFIG_FILE = ".kuberrc.ts"; export const DEFAULT_REGISTRY = IMAGE_REGISTRY; -export const DEFAULT_BUILDERS = { - amd64: "kuber@astral-th", - arm64: "kuber@astral", - remoteRoot: "kuber-build", -} as const; export const DEFAULT_ROLLOUT_TIMEOUT_MS = 300_000; type Hooks = Pick< @@ -23,11 +18,6 @@ export type ResolvedKuberConfig = Hooks & { projectConfigured: boolean; composeFile?: string; registry: string; - builders: { - amd64: string; - arm64: string; - remoteRoot: string; - }; rolloutTimeoutMs: number; configFile?: string; }; @@ -64,22 +54,6 @@ function validateConfig(config: unknown, file: string): KuberConfig { if (value.registry !== undefined) requireNonEmptyString(value.registry, "registry"); - if (value.builders !== undefined) { - if ( - !value.builders || - typeof value.builders !== "object" || - Array.isArray(value.builders) - ) { - throw new Error(`builders in ${file} must be an object`); - } - const builders = value.builders as Record; - for (const field of ["amd64", "arm64", "remoteRoot"]) { - if (builders[field] !== undefined) { - requireNonEmptyString(builders[field], `builders.${field}`); - } - } - } - if ( value.rolloutTimeoutMs !== undefined && (typeof value.rolloutTimeoutMs !== "number" || @@ -148,7 +122,6 @@ export async function loadConfig( configFile, ) : {}; - const builders = raw.builders ?? {}; const composeFile = raw.composeFile?.trim(); const registry = (raw.registry?.trim() ?? DEFAULT_REGISTRY).replace( /\/+$/, @@ -165,11 +138,6 @@ export async function loadConfig( : resolve(cwd, composeFile) : undefined, registry, - builders: { - amd64: builders.amd64?.trim() ?? DEFAULT_BUILDERS.amd64, - arm64: builders.arm64?.trim() ?? DEFAULT_BUILDERS.arm64, - remoteRoot: builders.remoteRoot?.trim() ?? DEFAULT_BUILDERS.remoteRoot, - }, rolloutTimeoutMs: raw.rolloutTimeoutMs ?? DEFAULT_ROLLOUT_TIMEOUT_MS, configFile: exists ? configFile : undefined, compose: raw.compose, diff --git a/lib/convert.ts b/lib/convert.ts index 531c752..dc1c0d4 100644 --- a/lib/convert.ts +++ b/lib/convert.ts @@ -14,9 +14,7 @@ import type { V1VolumeMount, } from "@kubernetes/client-node"; import { createHash } from "node:crypto"; -import { existsSync, readFileSync, statSync } from "node:fs"; -import { readFile } from "node:fs/promises"; -import { basename, resolve } from "node:path"; +import { basename } from "node:path"; import type { ComposeSpecification } from "../schema/docker.d"; import type { Service } from "../schema/docker.d"; import { LABELS } from "../const"; @@ -25,6 +23,10 @@ import { isPostgresVolumeEntry } from "./database"; import { toEnvVars } from "./format"; import { deepMerge } from "./shared"; import { isS3VolumeEntry } from "./storage"; +import { + LocalArtifactProvider, + type ArtifactProvider, +} from "../shared/artifacts"; type NormalizedMount = { name: string; @@ -303,21 +305,17 @@ function isBindSource(source: string): boolean { ); } -function resolveSourcePath(source: string, cwd: string): string { - return source.startsWith("~") - ? resolve(process.env.HOME ?? "", source.slice(1)) - : resolve(cwd, source); -} - -function isConfigFileSource(source: string, cwd: string): boolean { - const path = resolveSourcePath(source, cwd); - return existsSync(path) && statSync(path).isFile(); +function isConfigFileSource( + source: string, + artifacts: ArtifactProvider, +): boolean { + return artifacts.isFile(source, { expandHome: true }); } function parseStringMount( entry: string, index: number, - cwd: string, + artifacts: ArtifactProvider, ): NormalizedMount | undefined { if (isPostgresVolumeEntry(entry) || isS3VolumeEntry(entry)) return; @@ -348,7 +346,7 @@ function parseStringMount( : `volume-${index}`, kind: source && isBindSource(source) - ? isConfigFileSource(source, cwd) + ? isConfigFileSource(source, artifacts) ? "config-file" : "bind" : "volume", @@ -366,9 +364,10 @@ function parseStringMount( function toMount( entry: ServiceVolume, index: number, - cwd: string, + artifacts: ArtifactProvider, ): NormalizedMount | undefined { - if (typeof entry === "string") return parseStringMount(entry, index, cwd); + if (typeof entry === "string") + return parseStringMount(entry, index, artifacts); if (!entry.target) return; if ( @@ -389,7 +388,7 @@ function toMount( : `${entry.type}-${toKubeName(source) || index}` : `${entry.type}-${index}`, kind: - entry.type === "bind" && source && isConfigFileSource(source, cwd) + entry.type === "bind" && source && isConfigFileSource(source, artifacts) ? "config-file" : entry.type, source, @@ -450,12 +449,12 @@ function getTopLevelVolumeDataLocality( function toMounts( service: Service, - cwd = process.cwd(), + artifacts: ArtifactProvider, volumes: ComposeVolumes = {}, ): NormalizedMount[] { const mounts = service.volumes?.flatMap((entry, index) => { - const mount = toMount(entry, index, cwd); + const mount = toMount(entry, index, artifacts); if (!mount) return []; return [ @@ -855,15 +854,13 @@ function parseEnvFile(text: string): Record { async function readEnvFiles( envFile: Service["env_file"], - cwd: string, + artifacts: ArtifactProvider, ): Promise> { const result: Record = {}; for (const entry of toEnvFilePaths(envFile)) { - const path = resolve(cwd, entry.path); - try { - const text = await readFile(path, "utf8"); + const text = artifacts.readText(entry.path); Object.assign(result, parseEnvFile(text)); } catch (error) { if ( @@ -892,8 +889,9 @@ export function serviceToDeployment( extraEnv: Record = {}, volumes: ComposeVolumes = {}, buildImages: Record = {}, + artifacts: ArtifactProvider = new LocalArtifactProvider({ workspace: cwd }), ): V1Deployment { - const mounts = toMounts(service, cwd, volumes); + const mounts = toMounts(service, artifacts, volumes); const ports = toPorts(service); const hasEnvSecret = Boolean(service.env_file) || Object.keys(extraEnv).length > 0; @@ -1007,10 +1005,11 @@ export function volumesToPvc( service: Service, cwd = process.cwd(), volumes: ComposeVolumes = {}, + artifacts: ArtifactProvider = new LocalArtifactProvider({ workspace: cwd }), ): V1PersistentVolumeClaim[] { const claims = new Map(); - for (const mount of toMounts(service, cwd, volumes)) { + for (const mount of toMounts(service, artifacts, volumes)) { const claimName = mount.kind === "bind" ? mount.source @@ -1052,10 +1051,11 @@ export function volumesToStorageClasses( service: Service, cwd = process.cwd(), volumes: ComposeVolumes = {}, + artifacts: ArtifactProvider = new LocalArtifactProvider({ workspace: cwd }), ): LonghornStorageClass[] { const classes = new Map(); - for (const mount of toMounts(service, cwd, volumes)) { + for (const mount of toMounts(service, artifacts, volumes)) { const policy = getLonghornStoragePolicy(mount); if (!policy) continue; @@ -1089,14 +1089,14 @@ export function volumesToConfigMaps( service: Service, cwd = process.cwd(), volumes: ComposeVolumes = {}, + artifacts: ArtifactProvider = new LocalArtifactProvider({ workspace: cwd }), ): V1ConfigMap[] { const configMaps = new Map(); - for (const mount of toMounts(service, cwd, volumes)) { + for (const mount of toMounts(service, artifacts, volumes)) { if (mount.kind !== "config-file" || !mount.source || !mount.configKey) continue; - const sourcePath = resolveSourcePath(mount.source, cwd); configMaps.set(mount.name, { apiVersion: "v1", kind: "ConfigMap", @@ -1106,7 +1106,9 @@ export function volumesToConfigMaps( labels: LABELS, }, data: { - [mount.configKey]: readFileSync(sourcePath, "utf8"), + [mount.configKey]: artifacts.readText(mount.source, { + expandHome: true, + }), }, }); } @@ -1120,9 +1122,10 @@ export async function envFromToSecrets( service: Service, cwd = process.cwd(), extraEnv: Record = {}, + artifacts: ArtifactProvider = new LocalArtifactProvider({ workspace: cwd }), ): Promise { const stringData = { - ...(await readEnvFiles(service.env_file, cwd)), + ...(await readEnvFiles(service.env_file, artifacts)), ...extraEnv, }; if (Object.keys(stringData).length === 0) return []; @@ -1180,6 +1183,7 @@ export async function composeToKubernetes( cwd = process.cwd(), serviceEnv: Record> = {}, buildImages: Record = {}, + artifacts: ArtifactProvider = new LocalArtifactProvider({ workspace: cwd }), ): Promise { const resources = new Map(); @@ -1191,11 +1195,18 @@ export async function composeToKubernetes( service, cwd, compose.volumes, + artifacts, )) { resources.set(getResourceKey(storageClass), storageClass); } - for (const pvc of volumesToPvc(project, service, cwd, compose.volumes)) { + for (const pvc of volumesToPvc( + project, + service, + cwd, + compose.volumes, + artifacts, + )) { resources.set(getResourceKey(pvc), pvc); } @@ -1204,6 +1215,7 @@ export async function composeToKubernetes( service, cwd, compose.volumes, + artifacts, )) { resources.set(getResourceKey(configMap), configMap); } @@ -1214,6 +1226,7 @@ export async function composeToKubernetes( service, cwd, serviceEnv[name] ?? {}, + artifacts, ); for (const secret of envSecrets) { resources.set(getResourceKey(secret), secret); @@ -1231,6 +1244,7 @@ export async function composeToKubernetes( serviceEnv[name] ?? {}, compose.volumes, buildImages, + artifacts, ); const envSecret = envSecrets[0]; if (envSecret) { diff --git a/lib/exec-api.ts b/lib/exec-api.ts new file mode 100644 index 0000000..4e8e3d8 --- /dev/null +++ b/lib/exec-api.ts @@ -0,0 +1,226 @@ +import { KUBER_API_BASE_URL } from "../const"; +import { readSession, type KuberSession } from "./session"; + +export type ExecStartFrame = { + type: "start"; + version: 1; + deployment: string; + command: string[]; + tty: boolean; + container?: string; + columns?: number; + rows?: number; +}; + +export type ExecClientWireFrame = + | ExecStartFrame + | { type: "stdin"; data: string; encoding: "base64"; eof?: boolean } + | { type: "resize"; columns: number; rows: number } + | { type: "close" }; + +export type ExecServerWireFrame = + | { type: "stdout" | "stderr"; data: string; encoding: "base64" } + | { type: "exit"; exitCode: number; reason?: string; message?: string } + | { type: "error"; code: string; message: string }; + +export type ExecOutputFrame = + | { type: "stdout"; data: Uint8Array } + | { type: "stderr"; data: Uint8Array } + | Exclude; + +export interface ExecWebSocket { + binaryType: "arraybuffer" | "blob"; + readyState: number; + send(data: string): void; + close(code?: number, reason?: string): void; + addEventListener(type: string, listener: (event: any) => void): void; + removeEventListener(type: string, listener: (event: any) => void): void; +} + +export type ExecWebSocketFactory = ( + url: string, + headers: Readonly>, +) => ExecWebSocket; + +export type OpenExecOptions = { + baseUrl?: string; + session?: KuberSession; + socketFactory?: ExecWebSocketFactory; +}; + +export interface ExecApiSession extends AsyncIterable { + sendStdin(data: Uint8Array, eof?: boolean): void; + resize(columns: number, rows: number): void; + close(): void; +} + +function websocketUrl(baseUrl: string, path: string): string { + const url = new URL(`${baseUrl}${path}`); + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + return url.toString(); +} + +const defaultSocketFactory: ExecWebSocketFactory = (url, headers) => + new WebSocket(url, { + headers, + } as unknown as string[]) as unknown as ExecWebSocket; + +function parseFrame(value: unknown): ExecOutputFrame { + if (typeof value !== "string") + throw new Error("Exec frame must be JSON text"); + const frame: unknown = JSON.parse(value); + if (!frame || typeof frame !== "object" || !("type" in frame)) + throw new Error("Invalid exec frame"); + const wire = frame as ExecServerWireFrame; + if (wire.type === "stdout" || wire.type === "stderr") { + if (wire.encoding !== "base64" || typeof wire.data !== "string") + throw new Error("Invalid exec output frame"); + return { + type: wire.type, + data: Uint8Array.from(Buffer.from(wire.data, "base64")), + }; + } + if (wire.type === "exit" && Number.isSafeInteger(wire.exitCode)) return wire; + if (wire.type === "error" && typeof wire.message === "string") return wire; + throw new Error("Invalid exec frame"); +} + +export async function openExecSession( + project: string, + start: Omit, + signal: AbortSignal, + options: OpenExecOptions = {}, +): Promise { + if (signal.aborted) + throw signal.reason ?? new DOMException("Aborted", "AbortError"); + const session = options.session ?? (await readSession()); + if (!session) throw new Error("Not logged in. Run kuber login first."); + const path = `/workspaces/${encodeURIComponent(project)}/exec`; + const socket = (options.socketFactory ?? defaultSocketFactory)( + websocketUrl(options.baseUrl ?? KUBER_API_BASE_URL, path), + { + authorization: `Bearer ${session.token}`, + }, + ); + socket.binaryType = "arraybuffer"; + + await new Promise((resolve, reject) => { + const onOpen = () => { + cleanup(); + socket.send(JSON.stringify({ type: "start", version: 1, ...start })); + resolve(); + }; + const onError = (event: { error?: unknown }) => { + cleanup(); + reject(event.error ?? new Error("Exec WebSocket connection failed")); + }; + const onClose = () => { + cleanup(); + reject(new Error("Exec WebSocket closed before opening")); + }; + const onAbort = () => { + cleanup(); + socket.close(1000, "aborted"); + reject(signal.reason ?? new DOMException("Aborted", "AbortError")); + }; + const cleanup = () => { + socket.removeEventListener("open", onOpen); + socket.removeEventListener("error", onError); + socket.removeEventListener("close", onClose); + signal.removeEventListener("abort", onAbort); + }; + socket.addEventListener("open", onOpen); + socket.addEventListener("error", onError); + socket.addEventListener("close", onClose); + signal.addEventListener("abort", onAbort, { once: true }); + }); + + const frames: ExecOutputFrame[] = []; + const readers: Array<{ + resolve: (result: IteratorResult) => void; + reject: (error: unknown) => void; + }> = []; + let closed = false; + let failure: unknown; + let live = false; + const pending: ExecClientWireFrame[] = []; + const finish = (error?: unknown) => { + if (closed) return; + closed = true; + failure = error; + socket.removeEventListener("message", onMessage); + socket.removeEventListener("error", onError); + socket.removeEventListener("close", onClose); + signal.removeEventListener("abort", onAbort); + for (const reader of readers.splice(0)) { + if (error) reader.reject(error); + else reader.resolve({ done: true, value: undefined }); + } + }; + const onMessage = (event: { data: unknown }) => { + try { + if (!live) { + live = true; + for (const frame of pending.splice(0)) socket.send(JSON.stringify(frame)); + } + const frame = parseFrame(event.data); + const reader = readers.shift(); + if (reader) reader.resolve({ done: false, value: frame }); + else frames.push(frame); + } catch (error) { + socket.close(1002, "invalid frame"); + finish(error); + } + }; + const onError = (event: { error?: unknown }) => + finish(event.error ?? new Error("Exec WebSocket failed")); + const onClose = () => finish(); + const onAbort = () => { + socket.close(1000, "aborted"); + finish(); + }; + socket.addEventListener("message", onMessage); + socket.addEventListener("error", onError); + socket.addEventListener("close", onClose); + signal.addEventListener("abort", onAbort, { once: true }); + + const send = (frame: ExecClientWireFrame) => { + if (closed) throw failure ?? new Error("Exec session is closed"); + if (!live) { + pending.push(frame); + return; + } + socket.send(JSON.stringify(frame)); + }; + return { + sendStdin(data, eof) { + send({ + type: "stdin", + data: Buffer.from(data).toString("base64"), + encoding: "base64", + ...(eof ? { eof: true } : {}), + }); + }, + resize(columns, rows) { + send({ type: "resize", columns, rows }); + }, + close() { + if (!closed && socket.readyState === 1) send({ type: "close" }); + socket.close(1000, "client closed"); + finish(); + }, + [Symbol.asyncIterator]() { + return { + next(): Promise> { + const frame = frames.shift(); + if (frame) return Promise.resolve({ done: false, value: frame }); + if (failure) return Promise.reject(failure); + if (closed) return Promise.resolve({ done: true, value: undefined }); + return new Promise((resolve, reject) => + readers.push({ resolve, reject }), + ); + }, + }; + }, + }; +} diff --git a/lib/render.ts b/lib/render.ts index 4629ef9..60df0e6 100644 --- a/lib/render.ts +++ b/lib/render.ts @@ -1,7 +1,7 @@ import type { ComposeSpecification } from "../schema/docker.d"; import { composeToKubernetes, type KubernetesResource } from "./convert"; -import { reconcilePostgresClaims } from "./database"; -import { reconcileS3Claims } from "./storage"; +import { getComposePostgresClaims } from "./database"; +import { getComposeS3Claims } from "./storage"; import { resolveBuildImages, type BuildOptions } from "./build"; export async function renderResources( @@ -10,12 +10,15 @@ export async function renderResources( cwd = process.cwd(), options: BuildOptions = {}, ): Promise { - const postgresEnv = await reconcilePostgresClaims(project, compose); - const s3Env = await reconcileS3Claims(project, compose); - const serviceEnv = { ...postgresEnv }; - for (const [service, environment] of Object.entries(s3Env)) { - serviceEnv[service] = { ...serviceEnv[service], ...environment }; + const providers = [ + ...(getComposePostgresClaims(compose).length ? ["Postgres"] : []), + ...(getComposeS3Claims(compose).length ? ["S3"] : []), + ]; + if (providers.length > 0) { + throw new Error( + `Cannot export without generated ${providers.join(" and ")} credentials. Export is side-effect-free; run kuber up or remove managed provider claims.`, + ); } const buildImages = await resolveBuildImages(project, compose, options); - return composeToKubernetes(project, compose, cwd, serviceEnv, buildImages); + return composeToKubernetes(project, compose, cwd, {}, buildImages); } diff --git a/lib/session.ts b/lib/session.ts new file mode 100644 index 0000000..0d69865 --- /dev/null +++ b/lib/session.ts @@ -0,0 +1,109 @@ +import { + chmod, + mkdir, + readFile, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +export type KuberSession = { + token: string; + expiresAt: string; + user: { + username: string; + roles: string[]; + }; +}; + +function runtimeSessionPath(): string { + const directory = + process.env.XDG_RUNTIME_DIR ?? + join(tmpdir(), `kuber-${process.getuid?.() ?? "user"}`); + return join(directory, "kuber", "session.json"); +} + +function persistentSessionPath(): string { + const directory = + process.env.XDG_CONFIG_HOME ?? + join(process.env.HOME ?? tmpdir(), ".config"); + return join(directory, "kuber", "session.json"); +} + +export function getSessionPath(persistent: boolean): string { + return persistent ? persistentSessionPath() : runtimeSessionPath(); +} + +function isSession(value: unknown): value is KuberSession { + if (!value || typeof value !== "object") return false; + const session = value as Partial; + return ( + typeof session.token === "string" && + typeof session.expiresAt === "string" && + Boolean(session.user) && + typeof session.user?.username === "string" && + Array.isArray(session.user.roles) && + session.user.roles.every((role) => typeof role === "string") + ); +} + +async function readSessionFile( + path: string, +): Promise { + try { + const value: unknown = JSON.parse(await readFile(path, "utf8")); + if (!isSession(value)) return; + if (Date.parse(value.expiresAt) <= Date.now()) { + await rm(path, { force: true }); + return; + } + return value; + } catch (error) { + if ( + error && + typeof error === "object" && + "code" in error && + error.code === "ENOENT" + ) { + return; + } + if (error instanceof SyntaxError) return; + throw error; + } +} + +export async function readSession(): Promise { + return ( + (await readSessionFile(runtimeSessionPath())) ?? + (await readSessionFile(persistentSessionPath())) + ); +} + +export async function writeSession( + session: KuberSession, + persistent: boolean, +): Promise { + const path = getSessionPath(persistent); + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + await chmod(dirname(path), 0o700); + const temporaryPath = `${path}.${randomUUID()}.tmp`; + await writeFile(temporaryPath, `${JSON.stringify(session, null, 2)}\n`, { + flag: "wx", + mode: 0o600, + }); + await rename(temporaryPath, path); + await chmod(path, 0o600); + await rm(getSessionPath(!persistent), { force: true }); + return path; +} + +export async function removeSessions(): Promise { + await Promise.all( + [runtimeSessionPath(), persistentSessionPath()].map((path) => + rm(path, { force: true }), + ), + ); +} diff --git a/lib/workspace.ts b/lib/workspace.ts new file mode 100644 index 0000000..c451eed --- /dev/null +++ b/lib/workspace.ts @@ -0,0 +1,357 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { + chmod, + lstat, + mkdir, + open, + readdir, + readlink, + realpath, + symlink, +} from "node:fs/promises"; +import type { Stats } from "node:fs"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; +import { promisify } from "node:util"; +import { + BUILD_PROTOCOL_VERSION, + assertSha256Digest, + type Sha256Digest, + type WorkspaceFile, + type WorkspaceManifest, +} from "../shared/build-protocol"; + +const execFileAsync = promisify(execFile); + +export type WorkspaceBlob = { + digest: Sha256Digest; + data: Uint8Array; +}; + +export type WorkspaceSnapshot = { + manifest: WorkspaceManifest; + digest: Sha256Digest; + blobs: WorkspaceBlob[]; +}; + +export type BlobReader = + | ((digest: Sha256Digest) => Promise) + | { get(digest: Sha256Digest): Promise }; + +function digest(data: Uint8Array | string): Sha256Digest { + return `sha256:${createHash("sha256").update(data).digest("hex")}`; +} + +export function validateWorkspacePath(path: string): void { + if ( + !path || + path.includes("\0") || + path.includes("\\") || + isAbsolute(path) || + path.split("/").some((part) => !part || part === "." || part === "..") + ) { + throw new Error(`Unsafe workspace path: ${JSON.stringify(path)}`); + } +} + +function isWithin(root: string, candidate: string): boolean { + const path = relative(root, candidate); + return ( + path === "" || + (!path.startsWith(`..${sep}`) && path !== ".." && !isAbsolute(path)) + ); +} + +async function gitFiles(root: string, args: string[]): Promise { + const { stdout } = await execFileAsync( + "git", + ["-C", root, "ls-files", "-z", ...args], + { + encoding: "buffer", + maxBuffer: 64 * 1024 * 1024, + }, + ); + const decoder = new TextDecoder("utf-8", { fatal: true }); + const files: string[] = []; + let start = 0; + for ( + let end = stdout.indexOf(0); + end !== -1; + end = stdout.indexOf(0, start) + ) { + if (end > start) files.push(decoder.decode(stdout.subarray(start, end))); + start = end + 1; + } + return files; +} + +async function selectedFiles(root: string): Promise { + const [normal, dotenv] = await Promise.all([ + gitFiles(root, ["--cached", "--others", "--exclude-standard"]), + gitFiles(root, [ + "--others", + "--ignored", + "--exclude-standard", + "--", + ".env*", + "**/.env*", + ]), + ]); + return [...new Set([...normal, ...dotenv])].sort((a, b) => + Buffer.from(a).compare(Buffer.from(b)), + ); +} + +async function isIgnored(root: string, path: string): Promise { + try { + await execFileAsync("git", ["-C", root, "check-ignore", "-q", "--", path]); + return true; + } catch (error) { + if ((error as { code?: number }).code === 1) return false; + throw error; + } +} + +async function rejectSelectedSpecialFiles( + root: string, + directory = root, +): Promise { + for (const entry of await readdir(directory, { withFileTypes: true })) { + if (directory === root && entry.name === ".git") continue; + const source = resolve(directory, entry.name); + const path = relative(root, source).split(sep).join("/"); + if (entry.isDirectory()) { + if (!(await isIgnored(root, path))) + await rejectSelectedSpecialFiles(root, source); + continue; + } + if (entry.isFile() || entry.isSymbolicLink()) continue; + if (!(await isIgnored(root, path)) || entry.name.startsWith(".env")) + throw new Error(`Special files are not allowed in workspaces: ${path}`); + } +} + +function canonicalManifest(manifest: WorkspaceManifest): string { + return JSON.stringify({ + version: manifest.version, + files: manifest.files.map((file) => ({ + path: file.path, + type: file.type, + digest: file.digest, + size: file.size, + mode: file.mode, + })), + }); +} + +export function serializeWorkspaceManifest( + manifest: WorkspaceManifest, +): Uint8Array { + validateWorkspaceManifest(manifest); + return Buffer.from(canonicalManifest(manifest)); +} + +export function workspaceManifestDigest( + manifest: WorkspaceManifest, +): Sha256Digest { + return digest(serializeWorkspaceManifest(manifest)); +} + +export function validateWorkspaceManifest(manifest: WorkspaceManifest): void { + if ( + manifest.version !== BUILD_PROTOCOL_VERSION || + !Array.isArray(manifest.files) + ) { + throw new Error("Unsupported workspace manifest"); + } + + let previous = ""; + const seen = new Set(); + for (const file of manifest.files) { + validateWorkspacePath(file.path); + assertSha256Digest(file.digest); + if (!Number.isSafeInteger(file.size) || file.size < 0) + throw new Error(`Invalid size for ${file.path}`); + if ( + (file.type === "file" && file.mode !== 0o644 && file.mode !== 0o755) || + (file.type === "symlink" && file.mode !== 0o777) + ) { + throw new Error(`Invalid mode for ${file.path}`); + } + if (file.type !== "file" && file.type !== "symlink") + throw new Error(`Invalid entry type for ${file.path}`); + if (seen.has(file.path)) + throw new Error(`Duplicate workspace path: ${file.path}`); + for (const parent of file.path + .split("/") + .slice(0, -1) + .map((_, index, parts) => parts.slice(0, index + 1).join("/"))) { + if (seen.has(parent)) + throw new Error(`Workspace entry is used as a directory: ${parent}`); + } + if (previous && Buffer.from(previous).compare(Buffer.from(file.path)) >= 0) + throw new Error("Workspace files must be bytewise sorted"); + seen.add(file.path); + previous = file.path; + } +} + +export async function enumerateWorkspace( + root: string, +): Promise { + const repository = await realpath(root); + await rejectSelectedSpecialFiles(repository); + const paths = await selectedFiles(repository); + const files: WorkspaceFile[] = []; + const blobs = new Map(); + + for (const path of paths) { + validateWorkspacePath(path); + const source = resolve(repository, path); + if (!isWithin(repository, source)) + throw new Error(`Workspace path escapes root: ${path}`); + + let stat: Stats; + try { + stat = await lstat(source); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + + let data: Uint8Array; + let entry: WorkspaceFile; + if (stat.isSymbolicLink()) { + const target = await readlink(source); + if ( + isAbsolute(target) || + !isWithin(repository, resolve(dirname(source), target)) + ) + throw new Error(`Symlink escapes workspace: ${path} -> ${target}`); + data = Buffer.from(target); + entry = { + path, + type: "symlink", + digest: digest(data), + size: data.byteLength, + mode: 0o777, + }; + } else if (stat.isFile()) { + let mode: 0o644 | 0o755; + const handle = await open( + source, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const opened = await handle.stat(); + if (!opened.isFile()) throw new Error(`Not a regular file: ${path}`); + mode = opened.mode & 0o111 ? 0o755 : 0o644; + data = await handle.readFile(); + } finally { + await handle.close(); + } + entry = { + path, + type: "file", + digest: digest(data), + size: data.byteLength, + mode, + }; + } else { + throw new Error(`Special files are not allowed in workspaces: ${path}`); + } + files.push(entry); + blobs.set(entry.digest, data); + } + + const manifest = { + version: BUILD_PROTOCOL_VERSION, + files, + } satisfies WorkspaceManifest; + return { + manifest, + digest: workspaceManifestDigest(manifest), + blobs: [...blobs].map(([blobDigest, data]) => ({ + digest: blobDigest, + data, + })), + }; +} + +async function ensureParentDirectories( + root: string, + path: string, +): Promise { + let current = root; + for (const part of path.split("/").slice(0, -1)) { + current = resolve(current, part); + try { + const stat = await lstat(current); + if (!stat.isDirectory()) + throw new Error(`Workspace parent is not a directory: ${path}`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + await mkdir(current, { mode: 0o755 }); + } + } +} + +async function readBlob( + reader: BlobReader, + blobDigest: Sha256Digest, +): Promise { + return typeof reader === "function" + ? reader(blobDigest) + : reader.get(blobDigest); +} + +export async function materializeWorkspace( + destination: string, + manifest: WorkspaceManifest, + reader: BlobReader, +): Promise { + validateWorkspaceManifest(manifest); + await mkdir(destination, { recursive: false, mode: 0o755 }); + const root = await realpath(destination); + + for (const file of manifest.files.filter((entry) => entry.type === "file")) { + await ensureParentDirectories(root, file.path); + const data = await readBlob(reader, file.digest); + if (data.byteLength !== file.size || digest(data) !== file.digest) + throw new Error(`Blob verification failed for ${file.path}`); + const target = resolve(root, file.path); + const handle = await open( + target, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, + file.mode, + ); + try { + await handle.writeFile(data); + await handle.sync(); + } finally { + await handle.close(); + } + await chmod(target, file.mode); + } + + for (const file of manifest.files.filter( + (entry) => entry.type === "symlink", + )) { + await ensureParentDirectories(root, file.path); + const data = await readBlob(reader, file.digest); + if (data.byteLength !== file.size || digest(data) !== file.digest) + throw new Error(`Blob verification failed for ${file.path}`); + const linkTarget = Buffer.from(data).toString("utf8"); + const target = resolve(root, file.path); + if ( + linkTarget.includes("\0") || + isAbsolute(linkTarget) || + !isWithin(root, resolve(dirname(target), linkTarget)) + ) + throw new Error( + `Symlink escapes workspace: ${file.path} -> ${linkTarget}`, + ); + await symlink(linkTarget, target); + } +} diff --git a/package.json b/package.json index d862f79..84d4fe9 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,11 @@ "types": "types.d.ts", "scripts": { "build": "bun build index.ts --target bun --minify --outdir dist", + "build:server": "bun build server/index.ts --target bun --minify --outfile dist/kuber-server.js", + "server": "bun server/index.ts", "prepack": "bun run build", "test": "bun test tests", - "typecheck": "tsc --noEmit", - "prod": "bun run build && cp dist/kuber ~/.bun/bin/ && scp dist/kuber-arm64 root@astral:/usr/bin/kuber" + "typecheck": "tsc --noEmit" }, "devDependencies": { "@bomb.sh/tab": "^0.0.22", diff --git a/server/app.ts b/server/app.ts new file mode 100644 index 0000000..f623ab0 --- /dev/null +++ b/server/app.ts @@ -0,0 +1,2024 @@ +import type { KubernetesObject } from "@kubernetes/client-node"; +import { randomUUID } from "node:crypto"; +import type { ComposeSpecification } from "../schema/docker.d"; +import type { Operation as PublicOperation } from "../shared/api"; +import type { BuildRequest, Sha256Digest } from "../shared/build-protocol"; +import { + createToken, + hashToken, + tokenHashesEqual, + type AuthStore, + type KuberUser, + type SessionRecord, +} from "./auth"; +import { + hasCapability, + isRole, + type Capability, + type Role, +} from "./authorization"; +import type { AuditStore } from "./audit-store"; +import { + BuildConflictError, + BuildNotFoundError, + BuildValidationError, + type BuildController, +} from "./build-controller"; +import { KubernetesLogError, type LogService } from "./log-service"; +import { + ExecService, + type ExecClientFrame, + type ExecServerFrame, +} from "./exec-service"; +import type { + ExecClientWireFrame, + ExecServerWireFrame, + ExecStartFrame, +} from "../lib/exec-api"; +import type { ManagementService, ResourceIdentity } from "./management"; +import { + OperationConflictError, + OperationNotFoundError, + OperationValidationError, + sanitizeOperationResult, + type Operation, + type OperationStore, + type WorkspaceLeaseProvider, +} from "./operation-store"; +import { + WorkspaceConflictError, + WorkspaceNotFoundError, + WorkspaceValidationError, + workspaceEtag, + type CreateWorkspaceInput, + type UpdateWorkspaceInput, + type Workspace, + type WorkspaceStore, +} from "./workspace-store"; + +const API_PREFIX = "/api/v2"; +const RUNTIME_SESSION_MS = 24 * 60 * 60 * 1000; +const PERSISTENT_SESSION_MS = 30 * 24 * 60 * 60 * 1000; +const LOGIN_WINDOW_MS = 5 * 60 * 1000; +const MAX_LOGIN_FAILURES = 5; +const DEFAULT_JSON_LIMIT = 1024 * 1024; + +export interface ApiWorkspaceStore extends WorkspaceStore { + delete?(id: string): Promise; + adopt?( + workspaceId: string, + workspaceUid: string, + ): Promise; + adoptPlatform?(workspaceUid: string): Promise; +} + +export type AppOptions = { + store: AuthStore; + workspaceStore?: ApiWorkspaceStore; + operationStore?: OperationStore; + auditStore?: AuditStore; + management?: ManagementService; + builds?: BuildController; + logs?: LogService; + execService?: ExecService; + resolveImage?: ( + project: string, + service: string, + ) => Promise<{ image: string; digest: Sha256Digest; reference: string }>; + leases?: WorkspaceLeaseProvider; + adoption?: WorkspaceAdoptionService; + allowedOrigins?: readonly string[]; + jsonBodyLimit?: number; + verifyPassword?: (password: string, hash: string) => Promise; + hashPassword?: (password: string) => Promise; + now?: () => number; + requestId?: () => string; +}; + +type LoginFailures = { count: number; resetAt: number }; +type Identity = { user: KuberUser; session: SessionRecord }; + +export type WorkspaceAdoptionResult = { + workspaceId: string; + workspaceUid: string; + resourcesAdopted: number; +}; + +export interface WorkspaceAdoptionService { + adopt( + workspaceId: string, + workspaceUid: string, + ): Promise; + adoptPlatform(workspaceUid: string): Promise; +} + +export class WorkspaceAdoptionError extends Error { + readonly code = "ADOPTION_CONFLICT"; +} + +export async function cleanupExpiredSessions( + store: AuthStore, + now = Date.now(), +): Promise { + return store.deleteExpiredSessions(now); +} + +class HttpError extends Error { + constructor( + readonly status: number, + readonly title: string, + readonly code: string, + detail: string, + readonly headers?: Record, + readonly operationId?: string, + ) { + super(detail); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function bearerToken(request: Request): string | undefined { + return /^Bearer\s+(.+)$/i.exec( + request.headers.get("authorization") ?? "", + )?.[1]; +} + +function publicUser(user: KuberUser) { + return { + username: user.username, + roles: user.roles, + disabled: Boolean(user.disabled), + }; +} + +function workspaceIdentity(workspace: Workspace) { + return { project: workspace.metadata.name, uid: workspace.metadata.uid }; +} + +export async function authenticateRequest( + options: Pick, + request: Request, + now: () => number = options.now ?? Date.now, +): Promise { + const token = bearerToken(request); + if (!token) return; + const tokenHash = hashToken(token); + const session = await options.store.getSession(tokenHash); + if ( + !session || + !tokenHashesEqual(tokenHash, session.tokenHash) || + Date.parse(session.expiresAt) <= now() + ) { + if (session) await options.store.deleteSession(tokenHash); + return; + } + const user = await options.store.getUser(session.username); + if (!user || user.disabled || user.authVersion !== session.authVersion) + return; + return { user, session }; +} + +function pathPart(value: string): string { + try { + return decodeURIComponent(value); + } catch { + throw new HttpError( + 400, + "Bad request", + "INVALID_PATH", + "Invalid URL encoding", + ); + } +} + +export function createApp( + options: AppOptions, +): (request: Request) => Promise { + const verifyPassword = + options.verifyPassword ?? + ((password: string, hash: string) => Bun.password.verify(password, hash)); + const hashPassword = + options.hashPassword ?? + ((password: string) => + Bun.password.hash(password, { algorithm: "argon2id" })); + const now = options.now ?? Date.now; + const makeRequestId = options.requestId ?? randomUUID; + const bodyLimit = options.jsonBodyLimit ?? DEFAULT_JSON_LIMIT; + const allowedOrigins = new Set(options.allowedOrigins ?? []); + const hasOriginConfiguration = options.allowedOrigins !== undefined; + const loginFailures = new Map(); + const requestIds = new WeakMap(); + + function loginKey(request: Request): string { + return ( + request.headers.get("cf-connecting-ip") ?? + request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? + "unknown" + ); + } + + function response( + value: unknown, + status = 200, + headers?: Headers | Record, + ): Response { + return Response.json(value, { + status, + headers: { + "cache-control": "no-store", + ...Object.fromEntries(new Headers(headers)), + }, + }); + } + + function problem(error: HttpError, requestId: string): Response { + const result = response( + { + type: `https://kuber.astrxl.dev/problems/${error.code.toLowerCase()}`, + title: error.title, + status: error.status, + detail: error.message, + code: error.code, + requestId, + ...(error.operationId && { operationId: error.operationId }), + }, + error.status, + error.headers, + ); + result.headers.set("content-type", "application/problem+json"); + return result; + } + + async function readJson( + request: Request, + limit = bodyLimit, + ): Promise> { + const length = Number(request.headers.get("content-length")); + if (Number.isFinite(length) && length > limit) + throw new HttpError( + 413, + "Payload too large", + "BODY_TOO_LARGE", + `JSON body exceeds ${limit} bytes`, + ); + const contentType = request.headers.get("content-type"); + if ( + contentType && + !/^application\/(?:[\w.+-]+\+)?json(?:\s*;|$)/i.test(contentType) + ) + throw new HttpError( + 415, + "Unsupported media type", + "UNSUPPORTED_MEDIA_TYPE", + "Request body must be JSON", + ); + const reader = request.body?.getReader(); + const chunks: Uint8Array[] = []; + let bytes = 0; + if (reader) { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + if (bytes > limit) { + await reader.cancel(); + throw new HttpError( + 413, + "Payload too large", + "BODY_TOO_LARGE", + `JSON body exceeds ${limit} bytes`, + ); + } + chunks.push(value); + } + } + const payload = new Uint8Array(bytes); + let offset = 0; + for (const chunk of chunks) { + payload.set(chunk, offset); + offset += chunk.byteLength; + } + const text = new TextDecoder().decode(payload); + let value: unknown; + try { + value = text ? JSON.parse(text) : {}; + } catch { + throw new HttpError( + 400, + "Invalid JSON", + "INVALID_JSON", + "Request body is not valid JSON", + ); + } + if (!isRecord(value)) + throw new HttpError( + 400, + "Invalid request", + "INVALID_BODY", + "JSON body must be an object", + ); + return value; + } + + async function readBytes( + request: Request, + limit: number, + ): Promise { + const length = Number(request.headers.get("content-length")); + if (Number.isFinite(length) && length > limit) + throw new HttpError( + 413, + "Payload too large", + "BODY_TOO_LARGE", + `Request body exceeds ${limit} bytes`, + ); + const reader = request.body?.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + if (reader) { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > limit) { + await reader.cancel(); + throw new HttpError( + 413, + "Payload too large", + "BODY_TOO_LARGE", + `Request body exceeds ${limit} bytes`, + ); + } + chunks.push(value); + } + } + const result = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; + } + + function requireBuilds(): BuildController { + if (!options.builds) + throw new HttpError( + 503, + "Service unavailable", + "BUILDS_UNAVAILABLE", + "Build service is not configured", + ); + return options.builds; + } + + function nonNegativeInteger( + value: string | null, + name: string, + ): number | undefined { + if (value === null) return; + const number = Number(value); + if (!Number.isSafeInteger(number) || number < 0) + throw new HttpError( + 400, + "Invalid request", + "INVALID_QUERY", + `${name} must be a non-negative integer`, + ); + return number; + } + + function ndjson( + events: AsyncIterable | Iterable, + signal?: AbortSignal, + ): Response { + const iterator = + Symbol.asyncIterator in Object(events) + ? (events as AsyncIterable)[Symbol.asyncIterator]() + : (async function* () { + yield* events as Iterable; + })(); + const encoder = new TextEncoder(); + let reading = false; + return new Response( + new ReadableStream({ + async pull(controller) { + if (reading) return; + reading = true; + try { + const next = await iterator.next(); + if (next.done) controller.close(); + else + controller.enqueue( + encoder.encode(`${JSON.stringify(next.value)}\n`), + ); + } catch (error) { + controller.error(error); + } finally { + reading = false; + } + }, + async cancel(reason) { + await iterator.return?.(reason); + }, + }), + { + headers: { + "content-type": "application/x-ndjson; charset=utf-8", + "cache-control": "no-store", + connection: "keep-alive", + ...(signal ? { "x-accel-buffering": "no" } : {}), + }, + }, + ); + } + + async function authenticate(request: Request): Promise { + return authenticateRequest(options, request, now); + } + + function actor(identity: Identity, request: Request) { + return { + username: identity.user.username, + roles: identity.user.roles, + ip: + request.headers.get("cf-connecting-ip") ?? + request.headers.get("x-forwarded-for")?.split(",")[0]?.trim(), + userAgent: request.headers.get("user-agent") ?? undefined, + }; + } + + async function audit( + identity: Identity, + request: Request, + action: string, + outcome: "success" | "failure" | "denied", + details?: unknown, + workspaceId?: string, + operationId?: string, + ): Promise { + if (!options.auditStore) return; + await options.auditStore.append({ + actor: actor(identity, request), + action, + outcome, + details, + workspaceId, + operationId, + }); + } + + async function requireCapability( + identity: Identity, + request: Request, + capability: Capability, + ): Promise { + if (hasCapability(identity.user.roles, capability)) return; + await audit(identity, request, `authorization.${capability}`, "denied"); + throw new HttpError( + 403, + "Forbidden", + "FORBIDDEN", + `Capability '${capability}' is required`, + ); + } + + function requireWorkspaceStore(): ApiWorkspaceStore { + if (!options.workspaceStore) + throw new HttpError( + 503, + "Service unavailable", + "WORKSPACE_STORE_UNAVAILABLE", + "Workspace storage is not configured", + ); + return options.workspaceStore; + } + + function requireManagement(): ManagementService { + if (!options.management) + throw new HttpError( + 503, + "Service unavailable", + "MANAGEMENT_UNAVAILABLE", + "Kubernetes management is not configured", + ); + return options.management; + } + + function requireAdoption(): WorkspaceAdoptionService { + const workspaceStore = options.workspaceStore; + const adoption = + options.adoption ?? + (workspaceStore?.adopt && workspaceStore.adoptPlatform + ? { + adopt: workspaceStore.adopt.bind(workspaceStore), + adoptPlatform: workspaceStore.adoptPlatform.bind(workspaceStore), + } + : undefined); + if (!adoption) + throw new HttpError( + 503, + "Service unavailable", + "ADOPTION_UNAVAILABLE", + "Workspace adoption is not configured", + ); + return adoption; + } + + async function getWorkspace(id: string): Promise { + const workspace = await requireWorkspaceStore().get(id); + if (!workspace) + throw new HttpError( + 404, + "Not found", + "WORKSPACE_NOT_FOUND", + `Workspace '${id}' not found`, + ); + return workspace; + } + + function idempotencyKey(request: Request): string { + const key = request.headers.get("idempotency-key")?.trim(); + return key || requestIds.get(request) || makeRequestId(); + } + + function publicOperation(operation: Operation): PublicOperation { + return { + apiVersion: operation.apiVersion, + kind: operation.kind, + metadata: operation.metadata, + spec: { + workspaceId: operation.spec.workspaceId, + action: operation.spec.action, + }, + status: { + ...operation.status, + ...(operation.status.result !== undefined && { + result: sanitizeOperationResult( + operation.status.result, + operation.spec.action, + ), + }), + }, + }; + } + + function operationBody(operation: Operation, result: unknown) { + const visible = publicOperation(operation); + return isRecord(result) + ? { ...result, operationId: operation.metadata.name, operation: visible } + : Array.isArray(result) + ? { + deployments: result, + operationId: operation.metadata.name, + operation: visible, + } + : { result, operationId: operation.metadata.name, operation: visible }; + } + + async function runOperation( + identity: Identity, + request: Request, + workspace: Workspace, + action: string, + input: unknown, + execute: () => Promise, + ): Promise { + if (!options.operationStore) + throw new HttpError( + 503, + "Service unavailable", + "OPERATION_STORE_UNAVAILABLE", + "Operation storage is not configured", + ); + const operation = await options.operationStore.create({ + workspaceId: workspace.metadata.name, + workspaceUid: workspace.metadata.uid, + action, + idempotencyKey: idempotencyKey(request), + request: input, + }); + if (operation.status.state === "failed") + throw new HttpError( + operation.status.error?.code === "WORKSPACE_BUSY" ? 409 : 500, + "Operation failed", + operation.status.error?.code ?? "OPERATION_FAILED", + operation.status.error?.message ?? "Operation failed", + undefined, + operation.metadata.name, + ); + if (operation.status.state === "cancelled") + throw new HttpError( + 409, + "Operation cancelled", + "OPERATION_CANCELLED", + "The idempotent operation was cancelled", + undefined, + operation.metadata.name, + ); + if (operation.status.state === "running") + return response( + { + operationId: operation.metadata.name, + operation: publicOperation(operation), + }, + 202, + { location: `${API_PREFIX}/operations/${operation.metadata.name}` }, + ); + if (operation.status.state === "succeeded") + return response(operationBody(operation, operation.status.result), 200, { + location: `${API_PREFIX}/operations/${operation.metadata.name}`, + }); + + const lease = options.leases + ? await options.leases.acquire( + workspace.metadata.name, + operation.metadata.name, + ) + : undefined; + if (options.leases && !lease) { + await options.operationStore.transition( + operation.metadata.name, + "failed", + { + error: { + code: "WORKSPACE_BUSY", + message: "Another workspace operation is running", + }, + }, + ); + throw new HttpError( + 409, + "Conflict", + "WORKSPACE_BUSY", + "Another workspace operation is running", + undefined, + operation.metadata.name, + ); + } + await options.operationStore.transition(operation.metadata.name, "running"); + try { + const result = await execute(); + const completed = await options.operationStore.transition( + operation.metadata.name, + "succeeded", + { result }, + ); + try { + await audit( + identity, + request, + action, + "success", + undefined, + workspace.metadata.name, + operation.metadata.name, + ); + } catch (error) { + console.error( + "Failed to append successful operation audit event", + error, + ); + } + return response(operationBody(completed, result), 200, { + location: `${API_PREFIX}/operations/${operation.metadata.name}`, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await options.operationStore.transition( + operation.metadata.name, + "failed", + { + error: { code: "OPERATION_FAILED", message }, + }, + ); + try { + await audit( + identity, + request, + action, + "failure", + { error: message }, + workspace.metadata.name, + operation.metadata.name, + ); + } catch (auditError) { + console.error( + "Failed to append failed operation audit event", + auditError, + ); + } + throw new HttpError( + 500, + "Operation failed", + "OPERATION_FAILED", + message, + undefined, + operation.metadata.name, + ); + } finally { + try { + await lease?.release(); + } catch (error) { + console.error("Failed to release workspace operation lease", error); + } + } + } + + async function handleLogin(request: Request): Promise { + const key = loginKey(request); + const previous = loginFailures.get(key); + if ( + previous && + previous.resetAt > now() && + previous.count >= MAX_LOGIN_FAILURES + ) { + const retryAfter = Math.ceil((previous.resetAt - now()) / 1000); + throw new HttpError( + 429, + "Too many requests", + "LOGIN_RATE_LIMITED", + "Too many failed login attempts", + { "retry-after": String(retryAfter) }, + ); + } + if (previous && previous.resetAt <= now()) loginFailures.delete(key); + const body = await readJson(request, 16_384); + const username = + typeof body.username === "string" ? body.username.trim() : ""; + const password = typeof body.password === "string" ? body.password : ""; + if (!username || !password) + throw new HttpError( + 400, + "Invalid login request", + "INVALID_LOGIN", + "Username and password are required", + ); + const user = await options.store.getUser(username); + if ( + !user || + user.disabled || + !(await verifyPassword(password, user.passwordHash)) + ) { + const failures = loginFailures.get(key); + loginFailures.set(key, { + count: (failures?.count ?? 0) + 1, + resetAt: failures?.resetAt ?? now() + LOGIN_WINDOW_MS, + }); + throw new HttpError( + 401, + "Unauthorized", + "INVALID_CREDENTIALS", + "Invalid username or password", + ); + } + loginFailures.delete(key); + const token = createToken(); + const expiresAt = new Date( + now() + + (body.persistent === true ? PERSISTENT_SESSION_MS : RUNTIME_SESSION_MS), + ).toISOString(); + await options.store.putSession({ + tokenHash: hashToken(token), + username: user.username, + authVersion: user.authVersion, + expiresAt, + }); + return response({ + token, + expiresAt, + user: { username: user.username, roles: user.roles }, + }); + } + + async function handleAuthenticated( + request: Request, + url: URL, + identity: Identity, + ): Promise { + const path = url.pathname; + if (request.method === "GET" && path === `${API_PREFIX}/me`) + return response({ + username: identity.user.username, + roles: identity.user.roles, + }); + if (request.method === "POST" && path === `${API_PREFIX}/logout`) { + await options.store.deleteSession(identity.session.tokenHash); + return new Response(null, { status: 204 }); + } + + if (path === `${API_PREFIX}/builds` && request.method === "POST") { + await requireCapability(identity, request, "kubernetes:write"); + return response( + await requireBuilds().submitBuild( + (await readJson(request)) as unknown as BuildRequest, + ), + 202, + ); + } + + if ( + path === `${API_PREFIX}/snapshots/negotiate` && + request.method === "POST" + ) { + await requireCapability(identity, request, "kubernetes:write"); + const body = await readJson(request); + return response( + await requireBuilds().negotiateSnapshot(body.workspace as Sha256Digest), + ); + } + + if (path === `${API_PREFIX}/images/resolve` && request.method === "POST") { + await requireCapability(identity, request, "kubernetes:write"); + if (!options.resolveImage) + throw new HttpError( + 503, + "Service unavailable", + "BUILDS_UNAVAILABLE", + "Image resolution is not configured", + ); + const body = await readJson(request); + if (typeof body.project !== "string" || typeof body.service !== "string") + throw new HttpError( + 400, + "Invalid request", + "BUILD_INVALID", + "project and service are required", + ); + return response(await options.resolveImage(body.project, body.service)); + } + + let buildMatch = new RegExp( + `^${API_PREFIX}/builds/([^/]+)(?:/(events|reconcile|cancel|cleanup|result))?$`, + ).exec(path); + if (buildMatch) { + const id = pathPart(buildMatch[1]!); + const action = buildMatch[2]; + await requireCapability(identity, request, "kubernetes:write"); + const builds = requireBuilds(); + if (!action && request.method === "GET") + return response(await builds.getBuildStatus(id)); + if (action === "events" && request.method === "GET") + return response( + await builds.getBuildEvents( + id, + nonNegativeInteger(url.searchParams.get("after"), "after") ?? 0, + ), + ); + if (action === "reconcile" && request.method === "POST") + return response(await builds.reconcileBuild(id)); + if (action === "cancel" && request.method === "POST") + return response(await builds.cancelBuild(id)); + if (action === "cleanup" && request.method === "DELETE") { + await builds.cleanupBuild(id); + return new Response(null, { status: 204 }); + } + if (action === "result" && request.method === "GET") + return response(await builds.getBuildResult(id)); + } + + const blobMatch = new RegExp( + `^${API_PREFIX}/blobs/(sha256%3A|sha256:)([a-f0-9]{64})/uploads(?:/(complete))?$`, + "i", + ).exec(path); + if (blobMatch) { + await requireCapability(identity, request, "kubernetes:write"); + const digest = `sha256:${blobMatch[2]!.toLowerCase()}` as Sha256Digest; + if (blobMatch[3] === "complete" && request.method === "POST") + return response(await requireBuilds().completeBlobUpload(digest)); + if (request.method === "POST") { + const body = await readJson(request); + return response( + await requireBuilds().beginBlobUpload(digest, Number(body.size)), + 201, + ); + } + if (request.method === "PATCH") { + const offset = nonNegativeInteger( + request.headers.get("upload-offset"), + "Upload-Offset", + ); + if (offset === undefined) + throw new HttpError( + 400, + "Invalid request", + "BUILD_INVALID", + "Upload-Offset header is required", + ); + return response( + await requireBuilds().uploadBlobChunk( + digest, + offset, + await readBytes(request, 8 * 1024 * 1024), + ), + ); + } + } + + if (path === `${API_PREFIX}/users`) { + if (request.method === "GET") { + await requireCapability(identity, request, "users:read"); + return response({ + items: (await options.store.listUsers()).map(publicUser), + }); + } + if (request.method === "POST") { + await requireCapability(identity, request, "users:write"); + const body = await readJson(request); + if ( + typeof body.username !== "string" || + body.username !== body.username.trim() || + !body.username || + typeof body.password !== "string" || + !body.password || + !Array.isArray(body.roles) || + body.roles.length === 0 || + !body.roles.every(isRole) + ) + throw new HttpError( + 400, + "Invalid user", + "USER_INVALID", + "Username, password, and roles are required", + ); + const user = await options.store.createUser({ + username: body.username, + passwordHash: await hashPassword(body.password), + roles: body.roles as Role[], + }); + await audit(identity, request, "user.create", "success", { + username: user.username, + }); + return response(publicUser(user), 201, { + location: `${API_PREFIX}/users/${encodeURIComponent(user.username)}`, + }); + } + } + + let match = new RegExp( + `^${API_PREFIX}/users/([^/]+)(?:/(sessions/revoke))?$`, + ).exec(path); + if (match) { + const username = pathPart(match[1]!); + if (match[2]) { + if (request.method !== "POST") + throw new HttpError( + 405, + "Method not allowed", + "METHOD_NOT_ALLOWED", + "This endpoint only accepts POST", + { allow: "POST" }, + ); + await requireCapability(identity, request, "sessions:revoke"); + const revoked = await options.store.revokeUserSessions(username); + await audit(identity, request, "sessions.revoke", "success", { + username, + revoked, + }); + return response({ username, revoked }); + } + if (request.method === "GET") { + await requireCapability(identity, request, "users:read"); + const user = await options.store.getUser(username); + if (!user) + throw new HttpError( + 404, + "Not found", + "USER_NOT_FOUND", + `User '${username}' not found`, + ); + return response(publicUser(user)); + } + if (request.method === "PATCH" || request.method === "PUT") { + await requireCapability(identity, request, "users:write"); + const body = await readJson(request); + if ( + (body.roles !== undefined && + (!Array.isArray(body.roles) || + body.roles.length === 0 || + !body.roles.every(isRole))) || + (body.password !== undefined && + (typeof body.password !== "string" || !body.password)) || + (body.disabled !== undefined && typeof body.disabled !== "boolean") + ) + throw new HttpError( + 400, + "Invalid user", + "USER_INVALID", + "Invalid user update fields", + ); + const updated = await options.store.updateUser(username, { + ...(typeof body.password === "string" && + body.password && { + passwordHash: await hashPassword(body.password), + }), + ...(Array.isArray(body.roles) && { roles: body.roles as Role[] }), + ...(typeof body.disabled === "boolean" && { + disabled: body.disabled, + }), + }); + if (!updated) + throw new HttpError( + 404, + "Not found", + "USER_NOT_FOUND", + `User '${username}' not found`, + ); + const revoked = await options.store.revokeUserSessions(username); + await audit(identity, request, "user.update", "success", { + username, + revoked, + }); + return response(publicUser(updated)); + } + if (request.method === "DELETE") { + await requireCapability(identity, request, "users:write"); + if (!(await options.store.deleteUser(username))) + throw new HttpError( + 404, + "Not found", + "USER_NOT_FOUND", + `User '${username}' not found`, + ); + await audit(identity, request, "user.delete", "success", { username }); + return new Response(null, { status: 204 }); + } + } + + if (path === `${API_PREFIX}/workspaces`) { + if (request.method === "GET") { + await requireCapability(identity, request, "kubernetes:read"); + return response({ items: await requireWorkspaceStore().list() }); + } + if (request.method === "POST") { + await requireCapability(identity, request, "kubernetes:write"); + const body = await readJson(request); + const created = await requireWorkspaceStore().create( + body as unknown as CreateWorkspaceInput, + ); + await audit( + identity, + request, + "workspace.create", + "success", + undefined, + created.metadata.name, + ); + return response(created, 201, { + etag: workspaceEtag(created), + location: `${API_PREFIX}/workspaces/${created.metadata.name}`, + }); + } + } + + if ( + path === `${API_PREFIX}/platform/kuber-system/adopt` && + request.method === "POST" + ) { + if (!identity.user.roles.includes("admin")) + throw new HttpError( + 403, + "Forbidden", + "FORBIDDEN", + "The kuber-system platform adoption route requires the admin role", + ); + const body = await readJson(request); + if (typeof body.workspaceUid !== "string" || !body.workspaceUid.trim()) + throw new HttpError( + 400, + "Invalid request", + "ADOPTION_INVALID", + "workspaceUid is required", + ); + const adopted = await requireAdoption().adoptPlatform(body.workspaceUid); + await audit( + identity, + request, + "platform.adopt", + "success", + undefined, + "kuber-system", + ); + return response(adopted); + } + + match = new RegExp(`^${API_PREFIX}/workspaces/([^/]+)(?:/(.*))?$`).exec( + path, + ); + if (match) { + const id = pathPart(match[1]!); + const subpath = match[2]; + if (!subpath) { + if (request.method === "GET") { + await requireCapability(identity, request, "kubernetes:read"); + const workspace = await getWorkspace(id); + return response(workspace, 200, { etag: workspaceEtag(workspace) }); + } + if (request.method === "PUT" || request.method === "PATCH") { + await requireCapability(identity, request, "kubernetes:write"); + const ifMatch = request.headers.get("if-match"); + if (!ifMatch) + throw new HttpError( + 428, + "Precondition required", + "IF_MATCH_REQUIRED", + "If-Match header is required", + ); + const updated = await requireWorkspaceStore().update( + id, + (await readJson(request)) as unknown as UpdateWorkspaceInput, + ifMatch, + ); + await audit( + identity, + request, + "workspace.update", + "success", + undefined, + id, + ); + return response(updated, 200, { etag: workspaceEtag(updated) }); + } + if (request.method === "DELETE") { + await requireCapability(identity, request, "kubernetes:write"); + const workspace = await getWorkspace(id); + return runOperation( + identity, + request, + workspace, + "workspace.delete", + { full: true }, + async () => { + const result = options.management + ? await options.management.down( + workspaceIdentity(workspace), + true, + ) + : undefined; + if (!requireWorkspaceStore().delete) + throw new Error("Workspace store does not support deletion"); + await requireWorkspaceStore().delete!(id); + return result ?? { deleted: true }; + }, + ); + } + } + + const workspace = await getWorkspace(id); + if (subpath === "adopt" && request.method === "POST") { + await requireCapability(identity, request, "kubernetes:write"); + const adopted = await requireAdoption().adopt( + workspace.metadata.name, + workspace.metadata.uid, + ); + await audit( + identity, + request, + "workspace.adopt", + "success", + undefined, + id, + ); + return response(adopted); + } + if (subpath === "status" && request.method === "GET") { + await requireCapability(identity, request, "kubernetes:read"); + return response( + await requireManagement().graphStatus(workspaceIdentity(workspace), { + includeIdle: url.searchParams.get("includeIdle") === "true", + }), + ); + } + if (subpath === "logs" && request.method === "GET") { + await requireCapability(identity, request, "kubernetes:read"); + if (!options.logs) + throw new HttpError( + 503, + "Service unavailable", + "LOGS_UNAVAILABLE", + "Log service is not configured", + ); + const service = url.searchParams.get("service")?.trim(); + const logOptions = { + namespace: workspace.metadata.name, + target: service + ? ({ kind: "service", name: service } as const) + : ({ kind: "managed-deployments" } as const), + tailLines: nonNegativeInteger( + url.searchParams.get("tailLines"), + "tailLines", + ), + sinceSeconds: nonNegativeInteger( + url.searchParams.get("sinceSeconds"), + "sinceSeconds", + ), + sinceTime: url.searchParams.get("sinceTime") ?? undefined, + timestamps: url.searchParams.get("timestamps") === "true", + }; + if (url.searchParams.get("follow") !== "true") + return ndjson(await options.logs.collect(logOptions, request.signal)); + return ndjson( + options.logs.follow({ ...logOptions, signal: request.signal }), + request.signal, + ); + } + if (subpath === "revisions" && request.method === "GET") { + await requireCapability(identity, request, "kubernetes:read"); + return response({ + items: await requireWorkspaceStore().listRevisions(id), + }); + } + const revisionMatch = /^revisions\/(\d+)$/.exec(subpath ?? ""); + if (revisionMatch && request.method === "GET") { + await requireCapability(identity, request, "kubernetes:read"); + const revision = await requireWorkspaceStore().getRevision( + id, + Number(revisionMatch[1]), + ); + if (!revision) + throw new HttpError( + 404, + "Not found", + "REVISION_NOT_FOUND", + "Workspace revision not found", + ); + return response(revision); + } + + if ( + ["lifecycle", "stop", "restart", "rollback", "down"].includes( + subpath ?? "", + ) && + request.method === "POST" + ) { + await requireCapability(identity, request, "kubernetes:write"); + const body = await readJson(request); + const lifecycleAction = + subpath === "lifecycle" && + (body.action === "stop" || body.action === "restart") + ? body.action + : subpath; + if (subpath === "lifecycle" && lifecycleAction === "lifecycle") + throw new HttpError( + 400, + "Invalid lifecycle action", + "LIFECYCLE_INVALID", + "action must be stop or restart", + ); + return runOperation( + identity, + request, + workspace, + `workspace.${lifecycleAction}`, + body, + async () => { + const management = requireManagement(); + const names = Array.isArray(body.services) + ? (body.services as string[]) + : undefined; + if (lifecycleAction === "stop") + return management.stop(workspaceIdentity(workspace), names); + if (lifecycleAction === "restart") + return management.restart(workspaceIdentity(workspace), names); + if (lifecycleAction === "rollback") + return management.rollback( + workspaceIdentity(workspace), + names, + typeof body.timeoutMs === "number" ? body.timeoutMs : undefined, + ); + return management.down( + workspaceIdentity(workspace), + body.full === true, + ); + }, + ); + } + + if (subpath === "resources/plan" && request.method === "POST") { + await requireCapability(identity, request, "kubernetes:read"); + const body = await readJson(request); + if (!Array.isArray(body.resources)) + throw new HttpError( + 400, + "Invalid resources", + "RESOURCES_INVALID", + "resources must be an array", + ); + return response( + await requireManagement().planResources( + workspaceIdentity(workspace), + body.resources as KubernetesObject[], + ), + ); + } + if ( + ["resources/apply", "resources/wait", "resources/delete"].includes( + subpath ?? "", + ) && + request.method === "POST" + ) { + await requireCapability(identity, request, "kubernetes:write"); + const body = await readJson(request); + return runOperation( + identity, + request, + workspace, + subpath!, + body, + async () => { + const management = requireManagement(); + if (subpath === "resources/apply") { + if (!Array.isArray(body.resources)) + throw new Error("resources must be an array"); + return management.applyResources( + workspaceIdentity(workspace), + body.resources as KubernetesObject[], + ); + } + if (subpath === "resources/wait") { + if (!Array.isArray(body.deployments)) + throw new Error("deployments must be an array"); + await management.waitForResources( + workspaceIdentity(workspace), + body.deployments as string[], + typeof body.timeoutMs === "number" ? body.timeoutMs : undefined, + ); + return { ready: true }; + } + if (!Array.isArray(body.resources)) + throw new Error("resources must be an array"); + await management.deleteResources( + workspaceIdentity(workspace), + body.resources as unknown as ResourceIdentity[], + ); + return { deleted: body.resources.length }; + }, + ); + } + + if (subpath === "databases" && request.method === "POST") { + await requireCapability(identity, request, "kubernetes:write"); + const body = await readJson(request); + const compose = (body.compose ?? body) as ComposeSpecification; + return runOperation( + identity, + request, + workspace, + "databases.reconcile", + body, + () => + requireManagement().reconcileDatabases( + workspaceIdentity(workspace), + compose, + ), + ); + } + const databaseCredentials = /^databases\/([^/]+)\/credentials$/.exec( + subpath ?? "", + ); + if (databaseCredentials && request.method === "GET") { + await requireCapability(identity, request, "kubernetes:write"); + return response( + await requireManagement().getDatabaseCredentials( + workspaceIdentity(workspace), + pathPart(databaseCredentials[1]!), + ), + ); + } + if (subpath === "databases/credentials" && request.method === "POST") { + await requireCapability(identity, request, "kubernetes:write"); + const body = await readJson(request); + const claim = isRecord(body.claim) ? body.claim : body; + if (typeof claim.username !== "string") + throw new HttpError( + 400, + "Invalid database claim", + "DATABASE_CLAIM_INVALID", + "claim.username is required", + ); + return response( + await requireManagement().getDatabaseCredentials( + workspaceIdentity(workspace), + claim.username, + ), + ); + } + if ( + subpath === "databases/credentials-metadata" && + request.method === "POST" + ) { + await requireCapability(identity, request, "kubernetes:read"); + const body = await readJson(request); + return response({ + items: requireManagement().databaseCredentialsMetadata( + (body.compose ?? body) as ComposeSpecification, + ), + }); + } + + if (subpath === "storage" && request.method === "POST") { + await requireCapability(identity, request, "kubernetes:write"); + const body = await readJson(request); + const compose = (body.compose ?? body) as ComposeSpecification; + return runOperation( + identity, + request, + workspace, + "storage.reconcile", + body, + () => + requireManagement().reconcileStorage( + workspaceIdentity(workspace), + compose, + ), + ); + } + if (subpath === "storage/credentials" && request.method === "POST") { + await requireCapability(identity, request, "kubernetes:write"); + const body = await readJson(request); + const claim = isRecord(body.claim) ? body.claim : body; + return response( + await requireManagement().getStorageCredentials( + workspaceIdentity(workspace), + claim as unknown as { + service: string; + key: string; + bucket: string; + }, + ), + ); + } + if ( + subpath === "storage/credentials-metadata" && + request.method === "POST" + ) { + await requireCapability(identity, request, "kubernetes:read"); + const body = await readJson(request); + return response({ + items: requireManagement().storageCredentialsMetadata( + (body.compose ?? body) as ComposeSpecification, + ), + }); + } + } + + if (path === `${API_PREFIX}/operations` && request.method === "GET") { + await requireCapability(identity, request, "kubernetes:read"); + if (!options.operationStore) + throw new HttpError( + 503, + "Service unavailable", + "OPERATION_STORE_UNAVAILABLE", + "Operation storage is not configured", + ); + return response({ + items: ( + await options.operationStore.list( + url.searchParams.get("workspaceId") ?? undefined, + ) + ).map(publicOperation), + }); + } + match = new RegExp(`^${API_PREFIX}/operations/([^/]+)$`).exec(path); + if (match && request.method === "GET") { + await requireCapability(identity, request, "kubernetes:read"); + const operation = await options.operationStore?.get(pathPart(match[1]!)); + if (!operation) + throw new HttpError( + 404, + "Not found", + "OPERATION_NOT_FOUND", + "Operation not found", + ); + return response(publicOperation(operation)); + } + + if (path === `${API_PREFIX}/audit` && request.method === "GET") { + await requireCapability(identity, request, "users:read"); + if (!options.auditStore) + throw new HttpError( + 503, + "Service unavailable", + "AUDIT_STORE_UNAVAILABLE", + "Audit storage is not configured", + ); + return response({ + items: await options.auditStore.list( + url.searchParams.get("workspaceId") ?? undefined, + ), + }); + } + + throw new HttpError( + 404, + "Not found", + "NOT_FOUND", + "The requested API endpoint does not exist", + ); + } + + function normalizeError(error: unknown): HttpError { + if (error instanceof HttpError) return error; + if (error instanceof WorkspaceNotFoundError) + return new HttpError(404, "Not found", error.code, error.message); + if (error instanceof OperationNotFoundError) + return new HttpError(404, "Not found", error.code, error.message); + if (error instanceof BuildNotFoundError) + return new HttpError(404, "Not found", error.code, error.message); + if ( + error instanceof WorkspaceConflictError || + error instanceof OperationConflictError || + error instanceof BuildConflictError || + error instanceof WorkspaceAdoptionError + ) + return new HttpError(409, "Conflict", error.code, error.message); + if ( + error instanceof WorkspaceValidationError || + error instanceof OperationValidationError || + error instanceof BuildValidationError + ) + return new HttpError(400, "Invalid request", error.code, error.message); + if (error instanceof KubernetesLogError) + return new HttpError( + 502, + "Kubernetes log error", + "LOGS_FAILED", + error.message, + ); + if (error instanceof Error && /already exists/i.test(error.message)) + return new HttpError(409, "Conflict", "ALREADY_EXISTS", error.message); + console.error(error); + return new HttpError( + 500, + "Internal server error", + "INTERNAL_ERROR", + "The request could not be completed", + ); + } + + return async (request) => { + const suppliedRequestId = request.headers.get("x-request-id"); + const requestId = + suppliedRequestId && /^[\x21-\x7e]{1,128}$/.test(suppliedRequestId) + ? suppliedRequestId + : makeRequestId(); + requestIds.set(request, requestId); + const url = new URL(request.url); + const origin = request.headers.get("origin"); + const originAllowed = + !origin || + (hasOriginConfiguration + ? allowedOrigins.has(origin) + : origin === url.origin); + let result: Response; + try { + if (!originAllowed) + throw new HttpError( + 403, + "Forbidden", + "ORIGIN_NOT_ALLOWED", + "Request origin is not allowed", + ); + if (request.method === "OPTIONS") { + if (!origin) + throw new HttpError( + 400, + "Bad request", + "ORIGIN_REQUIRED", + "Origin header is required for preflight", + ); + result = new Response(null, { + status: 204, + headers: { + "access-control-allow-methods": + "GET, POST, PUT, PATCH, DELETE, OPTIONS", + "access-control-allow-headers": + "Authorization, Content-Type, Idempotency-Key, If-Match, Upload-Offset, X-Request-Id", + "access-control-max-age": "600", + }, + }); + } else if ( + request.method === "GET" && + url.pathname === `${API_PREFIX}/health` + ) { + result = response({ status: "ok" }); + } else if ( + request.method === "POST" && + url.pathname === `${API_PREFIX}/login` + ) { + result = await handleLogin(request); + } else { + const identity = await authenticate(request); + if (!identity) + throw new HttpError( + 401, + "Unauthorized", + "UNAUTHORIZED", + "A valid kuber login is required", + { "www-authenticate": "Bearer" }, + ); + result = await handleAuthenticated(request, url, identity); + } + } catch (error) { + result = problem(normalizeError(error), requestId); + } + result.headers.set("x-request-id", requestId); + if (origin && originAllowed) { + result.headers.set("access-control-allow-origin", origin); + result.headers.set("vary", "Origin"); + } + return result; + }; +} + +const EXEC_UPGRADE_PATH = new RegExp( + `^${API_PREFIX}/workspaces/([^/]+)/exec$`, +); + +export function execUpgradeMatch(url: URL): string | undefined { + const match = EXEC_UPGRADE_PATH.exec(url.pathname); + return match?.[1]; +} + +export type ExecConnection = { + identity: Identity; + workspace: Workspace; +}; + +export type ExecAuthOptions = Pick< + AppOptions, + "store" | "workspaceStore" | "auditStore" | "now" +>; + +export async function authorizeExecConnection( + options: ExecAuthOptions, + request: Request, + workspaceId: string, +): Promise { + const now = options.now ?? Date.now; + const identity = await authenticateRequest(options, request, now); + if (!identity) + throw new HttpError( + 401, + "Unauthorized", + "UNAUTHORIZED", + "A valid kuber login is required", + { "www-authenticate": "Bearer" }, + ); + if (!hasCapability(identity.user.roles, "kubernetes:exec")) { + if (options.auditStore) { + try { + await options.auditStore.append({ + actor: { + username: identity.user.username, + roles: identity.user.roles, + ip: + request.headers.get("cf-connecting-ip") ?? + request.headers.get("x-forwarded-for")?.split(",")[0]?.trim(), + userAgent: request.headers.get("user-agent") ?? undefined, + }, + action: "authorization.kubernetes:exec", + outcome: "denied", + workspaceId, + }); + } catch (error) { + console.error("Failed to append denied exec audit event", error); + } + } + throw new HttpError( + 403, + "Forbidden", + "FORBIDDEN", + "Capability 'kubernetes:exec' is required", + ); + } + const workspaceStore = options.workspaceStore; + const workspace = workspaceStore + ? await workspaceStore.get(pathPart(workspaceId)) + : undefined; + if (!workspace) + throw new HttpError( + 404, + "Not found", + "WORKSPACE_NOT_FOUND", + `Workspace '${workspaceId}' not found`, + ); + return { identity, workspace }; +} + +export function execProblem( + error: unknown, + requestId = "", +): Response { + let httpError: HttpError; + if (error instanceof HttpError) httpError = error; + else { + console.error(error); + httpError = new HttpError( + 500, + "Internal server error", + "INTERNAL_ERROR", + "The request could not be completed", + ); + } + const result = Response.json( + { + type: `https://kuber.astrxl.dev/problems/${httpError.code.toLowerCase()}`, + title: httpError.title, + status: httpError.status, + detail: httpError.message, + code: httpError.code, + requestId, + ...(httpError.operationId && { operationId: httpError.operationId }), + }, + { + status: httpError.status, + headers: { + "cache-control": "no-store", + ...httpError.headers, + }, + }, + ); + result.headers.set("content-type", "application/problem+json"); + return result; +} + +export function execWireExit( + server: ExecServerFrame, +): ExecServerWireFrame | undefined { + if (server.type !== "exit" && server.type !== "error") return; + if (server.type === "error") + return { type: "error", code: server.code, message: server.message }; + return { + type: "exit", + exitCode: server.exitCode, + ...(server.reason !== undefined && { reason: server.reason }), + ...(server.message !== undefined && { message: server.message }), + }; +} + +function isStartFrame(value: ExecClientWireFrame): value is ExecStartFrame { + return (value as ExecStartFrame).type === "start"; +} + +function base64ToBytes(value: string): Uint8Array | undefined { + const bytes = Uint8Array.from(Buffer.from(value, "base64")); + return bytes; +} + +export type ExecWebSocketLink = { + sendText(data: string): void; + close(code?: number, reason?: string): void; +}; + +export type ExecLinkOptions = { + maxFrameBytes?: number; +}; + +const DEFAULT_EXEC_MAX_FRAME_BYTES = 64 * 1024; + +export class WireExecSession { + private readonly controller = new AbortController(); + private session?: Awaited< + ReturnType + >; + private readonly maxFrameBytes: number; + private started = false; + private done: Promise = Promise.resolve(); + private closed = false; + + constructor( + private readonly socket: ExecWebSocketLink, + private readonly execService: ExecService, + private readonly connection: ExecConnection, + options: ExecLinkOptions = {}, + ) { + this.maxFrameBytes = + options.maxFrameBytes ?? DEFAULT_EXEC_MAX_FRAME_BYTES; + } + + private send(frame: ExecServerWireFrame) { + if (this.closed) return; + this.socket.sendText(JSON.stringify(frame)); + } + + async receive(raw: unknown): Promise { + if (typeof raw !== "string") { + this.fail("EXEC_INVALID", "Exec frames must be JSON text", 1003); + return; + } + if (new TextEncoder().encode(raw).byteLength > this.maxFrameBytes) { + this.fail("EXEC_INVALID", "Exec frame exceeds the size limit", 1009); + return; + } + let frame: ExecClientWireFrame; + try { + frame = JSON.parse(raw) as ExecClientWireFrame; + } catch { + this.fail("EXEC_INVALID", "Exec frame is not valid JSON", 1003); + return; + } + if (!frame || typeof frame !== "object" || !("type" in frame)) { + this.fail("EXEC_INVALID", "Invalid exec frame", 1003); + return; + } + if (!this.started) { + if (!isStartFrame(frame)) { + this.fail("EXEC_INVALID", "First exec frame must be start", 1003); + return; + } + await this.start(frame); + return; + } + if (!this.session) return; + try { + await this.session.send(this.toClientFrame(frame)); + } catch (error) { + if (!this.controller.signal.aborted) { + this.send({ + type: "error", + code: "EXEC_INVALID", + message: + error instanceof Error ? error.message : "Invalid exec frame", + }); + } + } + } + + private toClientFrame(frame: ExecClientWireFrame): ExecClientFrame { + switch (frame.type) { + case "stdin": { + if ( + frame.encoding !== "base64" || + typeof frame.data !== "string" + ) + throw new Error("Invalid stdin frame"); + const data = base64ToBytes(frame.data); + if (!data) + throw new Error("Invalid stdin frame"); + return { + type: "stdin", + data, + ...(frame.eof ? { eof: true } : {}), + }; + } + case "resize": { + if ( + !Number.isSafeInteger(frame.columns) || + !Number.isSafeInteger(frame.rows) || + frame.columns < 1 || + frame.rows < 1 || + frame.columns > 65_535 || + frame.rows > 65_535 + ) + throw new Error("Terminal dimensions are invalid"); + return { type: "resize", columns: frame.columns, rows: frame.rows }; + } + case "close": + return { type: "close" }; + default: + throw new Error("Unknown exec frame"); + } + } + + private fail(code: string, message: string, closeCode: number) { + if (this.closed) return; + this.send({ type: "error", code, message }); + this.close(closeCode, message); + } + + close(code = 1000, reason = "closed") { + if (this.closed) return; + this.closed = true; + this.controller.abort(); + void this.closeSession(); + this.socket.close(code, reason); + } + + private async closeSession() { + try { + await this.session?.close(); + } catch { + // session already closed + } + this.session = undefined; + } + + private async start(frame: ExecStartFrame) { + if (frame.version !== 1) { + this.fail( + "EXEC_INVALID", + `Unsupported exec protocol version ${frame.version}`, + 1003, + ); + return; + } + if ( + typeof frame.deployment !== "string" || + !frame.deployment || + !Array.isArray(frame.command) || + frame.command.length === 0 + ) { + this.fail("EXEC_INVALID", "deployment and command are required", 1003); + return; + } + const workspace = this.connection.workspace; + try { + this.session = await this.execService.openInteractive({ + workspace: { + project: workspace.metadata.name, + uid: workspace.metadata.uid, + }, + deployment: frame.deployment, + container: frame.container, + command: frame.command, + tty: frame.tty ?? true, + signal: this.controller.signal, + }); + } catch (error) { + if (this.controller.signal.aborted) return; + this.send({ + type: "error", + code: + error instanceof Error && + "code" in error && + typeof (error as { code?: unknown }).code === "string" + ? ((error as { code: string }).code) + : "EXEC_FAILED", + message: + error instanceof Error ? error.message : "Exec failed to start", + }); + this.close(1011, "exec failed"); + return; + } + this.started = true; + this.done = this.pump(); + } + + private async pump() { + const session = this.session; + if (!session) return; + try { + for await (const server of session) { + if (this.closed || this.controller.signal.aborted) return; + if (server.type === "stdout" || server.type === "stderr") { + this.send({ + type: server.type, + data: Buffer.from(server.data).toString("base64"), + encoding: "base64", + }); + } else { + const wire = execWireExit(server); + if (wire) { + this.send(wire); + this.close(1000, "process finished"); + return; + } + } + } + } catch (error) { + if (!this.controller.signal.aborted && !this.closed) { + const code = + error instanceof Error && + "code" in error && + typeof (error as { code?: unknown }).code === "string" + ? ((error as { code: string }).code) + : "EXEC_FAILED"; + this.send({ + type: "error", + code, + message: + error instanceof Error ? error.message : "Exec session failed", + }); + this.close(1011, "exec failed"); + } + } + } +} + +export async function handleExecUpgrade( + options: AppOptions, + request: Request, + workspaceId: string, + upgrade: (request: Request, data: ExecConnection) => boolean, +): Promise { + let connection: ExecConnection; + try { + connection = await authorizeExecConnection(options, request, workspaceId); + } catch (error) { + return execProblem(error); + } + if (!upgrade(request, connection)) + return execProblem( + new HttpError( + 400, + "Bad request", + "UPGRADE_FAILED", + "WebSocket upgrade failed", + ), + ); + return; +} diff --git a/server/audit-store.ts b/server/audit-store.ts new file mode 100644 index 0000000..5784d90 --- /dev/null +++ b/server/audit-store.ts @@ -0,0 +1,143 @@ +import { randomUUID } from "node:crypto"; +import { KUBER_API_VERSION, type ObjectMeta } from "./workspace-store"; +import { redactString, REDACTED } from "./redact"; + +export const AUDIT_REDACTED = REDACTED; +export const MAX_AUDIT_EVENT_BYTES = 256 * 1024; + +const SENSITIVE_KEY = + /(?:authorization|cookie|credential|password|passwd|secret|token|api[-_]?key|private[-_]?key)/i; +const SENSITIVE_VALUE = /^(?:bearer|basic)\s+\S+/i; + +export interface AuditActor { + username: string; + roles?: string[]; + ip?: string; + userAgent?: string; +} + +export interface AuditEvent { + apiVersion: typeof KUBER_API_VERSION; + kind: "AuditEvent"; + metadata: ObjectMeta; + spec: { + actor: AuditActor; + action: string; + workspaceId?: string; + operationId?: string; + outcome: "success" | "failure" | "denied"; + details?: unknown; + }; +} + +export interface AppendAuditEventInput { + actor: AuditActor; + action: string; + workspaceId?: string; + operationId?: string; + outcome: AuditEvent["spec"]["outcome"]; + details?: unknown; +} + +/** Implementations expose append/list only: audit records are never updated or deleted. */ +export interface AuditPersistence { + append(event: AuditEvent): Promise; + list(workspaceId?: string): Promise; +} + +export interface AuditStore { + append(input: AppendAuditEventInput): Promise; + list(workspaceId?: string): Promise; +} + +export class AuditValidationError extends Error { + readonly code = "AUDIT_INVALID"; +} + +function clone(value: T): T { + return structuredClone(value); +} + +export function redactAuditValue(value: unknown): unknown { + if (typeof value === "string") { + return SENSITIVE_VALUE.test(value) ? AUDIT_REDACTED : redactString(value); + } + if (Array.isArray(value)) return value.map(redactAuditValue); + if (value && typeof value === "object") { + const redacted: Record = {}; + for (const [key, nested] of Object.entries(value)) { + redacted[key] = SENSITIVE_KEY.test(key) + ? AUDIT_REDACTED + : redactAuditValue(nested); + } + return redacted; + } + return value; +} + +export class RedactingAuditStore implements AuditStore { + constructor( + private readonly persistence: AuditPersistence, + private readonly now: () => Date = () => new Date(), + private readonly uid: () => string = randomUUID, + ) {} + + async append(input: AppendAuditEventInput): Promise { + if (!input.actor.username.trim() || !input.action.trim()) { + throw new AuditValidationError("Audit actor and action are required"); + } + const id = this.uid(); + const event: AuditEvent = { + apiVersion: KUBER_API_VERSION, + kind: "AuditEvent", + metadata: { + name: `audit-${id}`, + uid: id, + resourceVersion: "1", + creationTimestamp: this.now().toISOString(), + }, + spec: { + actor: redactAuditValue(input.actor) as AuditActor, + action: input.action, + ...(input.workspaceId && { workspaceId: input.workspaceId }), + ...(input.operationId && { operationId: input.operationId }), + outcome: input.outcome, + ...(input.details !== undefined && { + details: redactAuditValue(input.details), + }), + }, + }; + const serialized = JSON.stringify(event); + if (Buffer.byteLength(serialized) > MAX_AUDIT_EVENT_BYTES) { + throw new AuditValidationError("Audit event is too large"); + } + await this.persistence.append(event); + return clone(event); + } + + async list(workspaceId?: string) { + return clone(await this.persistence.list(workspaceId)); + } +} + +export class MemoryAuditPersistence implements AuditPersistence { + private readonly events: AuditEvent[] = []; + + async append(event: AuditEvent) { + this.events.push(clone(event)); + } + + async list(workspaceId?: string) { + return this.events + .filter((event) => + workspaceId ? event.spec.workspaceId === workspaceId : true, + ) + .map(clone); + } +} + +export class MemoryAuditStore extends RedactingAuditStore { + constructor(now?: () => Date, uid?: () => string) { + super(new MemoryAuditPersistence(), now, uid); + } +} diff --git a/server/auth.ts b/server/auth.ts new file mode 100644 index 0000000..6291302 --- /dev/null +++ b/server/auth.ts @@ -0,0 +1,204 @@ +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; +import { isRole, type Role } from "./authorization"; + +export type KuberUser = { + username: string; + passwordHash: string; + roles: Role[]; + authVersion: number; + disabled?: boolean; +}; + +export type SessionRecord = { + tokenHash: string; + username: string; + authVersion: number; + expiresAt: string; +}; + +export type SessionInput = + | SessionRecord + | { + tokenHash: string; + username: string; + roles: string[]; + expiresAt: string; + }; + +export type NewKuberUser = Omit & { + authVersion?: number; +}; + +export type UserUpdate = Partial< + Pick +>; + +export interface AuthStore { + getUser(username: string): Promise; + listUsers(): Promise; + putUser(user: NewKuberUser | KuberUser): Promise; + createUser(user: NewKuberUser): Promise; + updateUser( + username: string, + update: UserUpdate, + ): Promise; + deleteUser(username: string): Promise; + getSession(tokenHash: string): Promise; + putSession(session: SessionInput): Promise; + deleteSession(tokenHash: string): Promise; + revokeUserSessions(username: string): Promise; + listExpiredSessions(now?: number): Promise; + deleteExpiredSessions(now?: number): Promise; +} + +export function normalizeUser(user: NewKuberUser | KuberUser): KuberUser { + if (!user.username || user.username !== user.username.trim()) + throw new Error("Username must be a non-empty trimmed string"); + if (!user.passwordHash) throw new Error("Password hash is required"); + if ( + !Array.isArray(user.roles) || + user.roles.length === 0 || + new Set(user.roles).size !== user.roles.length || + !user.roles.every(isRole) + ) { + throw new Error("At least one unique valid role is required"); + } + const authVersion = user.authVersion ?? 1; + if (!Number.isSafeInteger(authVersion) || authVersion < 1) + throw new Error("Auth version must be a positive integer"); + return { ...user, roles: [...user.roles], authVersion }; +} + +export function normalizeSession(session: SessionRecord): SessionRecord { + if (!/^[a-f0-9]{64}$/.test(session.tokenHash)) + throw new Error("Session token hash must be a SHA-256 hex digest"); + if (!session.username) throw new Error("Session username is required"); + if (!Number.isSafeInteger(session.authVersion) || session.authVersion < 1) + throw new Error("Session auth version must be a positive integer"); + const expiresAt = new Date(session.expiresAt); + if ( + !Number.isFinite(expiresAt.getTime()) || + expiresAt.toISOString() !== session.expiresAt + ) { + throw new Error("Session expiration must be an ISO timestamp"); + } + return { ...session }; +} + +export function hashToken(token: string): string { + return createHash("sha256").update(token).digest("hex"); +} + +export function createToken(): string { + return randomBytes(32).toString("base64url"); +} + +export function tokenHashesEqual(left: string, right: string): boolean { + if (!/^[a-f0-9]{64}$/.test(left) || !/^[a-f0-9]{64}$/.test(right)) + return false; + const leftBuffer = Buffer.from(left, "hex"); + const rightBuffer = Buffer.from(right, "hex"); + return ( + leftBuffer.length === rightBuffer.length && + timingSafeEqual(leftBuffer, rightBuffer) + ); +} + +export class MemoryAuthStore implements AuthStore { + readonly users = new Map(); + readonly sessions = new Map(); + + async getUser(username: string): Promise { + return this.users.get(username); + } + + async listUsers(): Promise { + return [...this.users.values()].sort((a, b) => + a.username.localeCompare(b.username), + ); + } + + async putUser(user: NewKuberUser | KuberUser): Promise { + const normalized = normalizeUser(user); + this.users.set(normalized.username, normalized); + } + + async createUser(user: NewKuberUser): Promise { + if (this.users.has(user.username)) throw new Error("User already exists"); + const normalized = normalizeUser(user); + this.users.set(normalized.username, normalized); + return normalized; + } + + async updateUser( + username: string, + update: UserUpdate, + ): Promise { + const existing = this.users.get(username); + if (!existing) return; + const updated = normalizeUser({ + ...existing, + ...update, + username, + authVersion: existing.authVersion + 1, + }); + this.users.set(username, updated); + return updated; + } + + async deleteUser(username: string): Promise { + await this.revokeUserSessions(username); + return this.users.delete(username); + } + + async getSession(tokenHash: string): Promise { + const session = this.sessions.get(tokenHash); + if (!session) return; + const user = this.users.get(session.username); + if (!user || user.disabled || user.authVersion !== session.authVersion) + return; + return session; + } + + async putSession(session: SessionInput): Promise { + const user = this.users.get(session.username); + if (!user || user.disabled) throw new Error("Session user is not active"); + const authVersion = + "authVersion" in session ? session.authVersion : user.authVersion; + if (authVersion !== user.authVersion) + throw new Error("Session auth version is stale"); + const normalized = normalizeSession({ + tokenHash: session.tokenHash, + username: session.username, + authVersion, + expiresAt: session.expiresAt, + }); + this.sessions.set(normalized.tokenHash, normalized); + } + + async deleteSession(tokenHash: string): Promise { + this.sessions.delete(tokenHash); + } + + async revokeUserSessions(username: string): Promise { + let deleted = 0; + for (const [tokenHash, session] of this.sessions) { + if (session.username !== username) continue; + this.sessions.delete(tokenHash); + deleted += 1; + } + return deleted; + } + + async listExpiredSessions(now = Date.now()): Promise { + return [...this.sessions.values()].filter( + (session) => Date.parse(session.expiresAt) <= now, + ); + } + + async deleteExpiredSessions(now = Date.now()): Promise { + const expired = await this.listExpiredSessions(now); + for (const session of expired) this.sessions.delete(session.tokenHash); + return expired.length; + } +} diff --git a/server/authorization.ts b/server/authorization.ts new file mode 100644 index 0000000..96370e7 --- /dev/null +++ b/server/authorization.ts @@ -0,0 +1,43 @@ +export const ROLES = ["viewer", "operator", "admin"] as const; +export type Role = (typeof ROLES)[number]; + +export const CAPABILITIES = [ + "kubernetes:read", + "kubernetes:write", + "kubernetes:exec", + "users:read", + "users:write", + "sessions:revoke", +] as const; +export type Capability = (typeof CAPABILITIES)[number]; + +const ROLE_CAPABILITIES: Readonly> = { + viewer: ["kubernetes:read"], + operator: ["kubernetes:read", "kubernetes:write", "kubernetes:exec"], + admin: CAPABILITIES, +}; + +export function isRole(value: unknown): value is Role { + return ( + typeof value === "string" && (ROLES as readonly string[]).includes(value) + ); +} + +export function capabilitiesForRoles( + roles: readonly string[], +): ReadonlySet { + const capabilities = new Set(); + for (const role of roles) { + if (!isRole(role)) continue; + for (const capability of ROLE_CAPABILITIES[role]) + capabilities.add(capability); + } + return capabilities; +} + +export function hasCapability( + roles: readonly string[], + capability: Capability, +): boolean { + return capabilitiesForRoles(roles).has(capability); +} diff --git a/server/build-controller.ts b/server/build-controller.ts new file mode 100644 index 0000000..b93c7b6 --- /dev/null +++ b/server/build-controller.ts @@ -0,0 +1,661 @@ +import { createHash } from "node:crypto"; +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { + BUILD_PROTOCOL_VERSION, + assertSha256Digest, + type BuildEvent, + type BuildRequest, + type BuildStatus, + type Sha256Digest, +} from "../shared/build-protocol"; +import { createBuildJob, type KubernetesJob } from "./build-job"; +import { + BUILD_RECORD_API_VERSION, + BuildStoreConflictError, + type BuildRecord, + type BuildStore, + type UploadRecord, +} from "./build-store"; +import { + materializeWorkspace, + parseWorkspaceManifest, + type MaterializeCas, +} from "./materialize"; +import { parseImageReference, resolveRegistryDigest } from "./registry"; + +export const DEFAULT_MAX_BLOB_BYTES = 1024 * 1024 * 1024; +export const DEFAULT_MAX_UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024; +export const DEFAULT_MAX_LOG_BYTES = 1024 * 1024; + +export interface BuildCas extends MaterializeCas { + put(data: Uint8Array, expected?: Sha256Digest): Promise; +} + +export type JobPhase = "queued" | "running" | "succeeded" | "failed"; + +export interface BuildJobObservation { + phase: JobPhase; + startedAt?: string; + finishedAt?: string; + error?: string; +} + +export interface BuildKubernetesOperations { + createJob(job: KubernetesJob): Promise; + getJob( + namespace: string, + name: string, + ): Promise; + getJobLogs(namespace: string, name: string): Promise; + deleteJob(namespace: string, name: string): Promise; +} + +export interface BuildControllerOptions { + cas: BuildCas; + store: BuildStore; + kubernetes: BuildKubernetesOperations; + namespace: string; + workspaceRoot: string; + workspaceClaimName: string; + cacheImage: string | ((request: BuildRequest) => string); + imageName?: (request: BuildRequest) => string; + pushImage?: (request: BuildRequest) => string; + pushRegistryInsecure?: boolean; + cacheRegistryInsecure?: boolean; + buildkitImage?: string; + serviceAccountName?: string; + registrySecretName?: string; + nodeSelector?: Record; + tolerations?: Array>; + maxBlobBytes?: number; + maxUploadChunkBytes?: number; + maxLogBytes?: number; + now?: () => Date; + materialize?: typeof materializeWorkspace; + resolveDigest?: typeof resolveRegistryDigest; +} + +export class BuildValidationError extends Error { + readonly code = "BUILD_INVALID"; +} + +export class BuildNotFoundError extends Error { + readonly code = "BUILD_NOT_FOUND"; +} + +export class BuildConflictError extends Error { + readonly code = "BUILD_CONFLICT"; +} + +export type SnapshotNegotiation = { + workspace: Sha256Digest; + missing: Sha256Digest[]; + ready: boolean; +}; + +export type UploadProgress = { + digest: Sha256Digest; + size: number; + offset: number; + complete: boolean; +}; + +export function buildImageName( + registry: string, + project: string, + service: string, +): string { + const valid = (value: string) => + value.length <= 63 && /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/.test(value); + if (!valid(project) || !valid(service)) + throw new BuildValidationError( + "Build project and service must be valid Kubernetes names", + ); + const owner = registry.replace(/\/+$/, ""); + if (!owner) throw new BuildValidationError("Build registry is required"); + return `${owner}/kuber/${project}-${service}:latest`; +} + +function clone(value: T): T { + return structuredClone(value); +} + +function requestFingerprint(request: BuildRequest): string { + return createHash("sha256").update(JSON.stringify(request)).digest("hex"); +} + +function jobWorkspaceSubPath(workspaceRoot: string, subPath: string): string { + const segments = workspaceRoot.split("/").filter(Boolean); + const prefix = segments.at(-1); + if (!prefix || prefix === ".") return subPath; + return subPath ? `${prefix}/${subPath}` : prefix; +} + +function validateRequest(request: BuildRequest): void { + if (request.version !== BUILD_PROTOCOL_VERSION) + throw new BuildValidationError("Unsupported build protocol version"); + if (!request.id || !request.project || !request.service) + throw new BuildValidationError( + "Build ID, project, and service are required", + ); + if (Buffer.byteLength(request.id) > 256) + throw new BuildValidationError("Build ID is too long"); + if (!request.spec || !["amd64", "arm64"].includes(request.spec.architecture)) + throw new BuildValidationError("Invalid build architecture"); + if ( + !Array.isArray(request.spec.buildArgs) || + !request.spec.buildArgs.every((value) => typeof value === "string") + ) { + throw new BuildValidationError("Build arguments must be strings"); + } + assertSha256Digest(request.spec.workspace); + parseImageReference(request.spec.image); +} + +function recordStatus(record: BuildRecord): BuildStatus { + const { + version, + id, + state, + createdAt, + startedAt, + finishedAt, + digest, + error, + } = record.status; + return { + version, + id, + state, + createdAt, + ...(startedAt && { startedAt }), + ...(finishedAt && { finishedAt }), + ...(digest && { digest }), + ...(error && { error }), + }; +} + +export class BuildController { + private readonly now: () => Date; + private readonly maxBlobBytes: number; + private readonly maxUploadChunkBytes: number; + private readonly maxLogBytes: number; + private readonly materializer: typeof materializeWorkspace; + private readonly digestResolver: typeof resolveRegistryDigest; + + constructor(private readonly options: BuildControllerOptions) { + this.now = options.now ?? (() => new Date()); + this.maxBlobBytes = options.maxBlobBytes ?? DEFAULT_MAX_BLOB_BYTES; + this.maxUploadChunkBytes = + options.maxUploadChunkBytes ?? DEFAULT_MAX_UPLOAD_CHUNK_BYTES; + this.maxLogBytes = options.maxLogBytes ?? DEFAULT_MAX_LOG_BYTES; + this.materializer = options.materialize ?? materializeWorkspace; + this.digestResolver = options.resolveDigest ?? resolveRegistryDigest; + } + + async negotiateSnapshot( + workspace: Sha256Digest, + ): Promise { + assertSha256Digest(workspace); + if (!(await this.options.cas.has(workspace))) + return { workspace, missing: [workspace], ready: false }; + const manifest = parseWorkspaceManifest( + await this.options.cas.get(workspace), + ); + const missing: Sha256Digest[] = []; + const seen = new Set(); + for (const file of manifest.files) { + if (!seen.has(file.digest) && !(await this.options.cas.has(file.digest))) + missing.push(file.digest); + seen.add(file.digest); + } + return { workspace, missing, ready: missing.length === 0 }; + } + + async beginBlobUpload( + digest: Sha256Digest, + size: number, + ): Promise { + assertSha256Digest(digest); + if (!Number.isSafeInteger(size) || size < 0 || size > this.maxBlobBytes) + throw new BuildValidationError( + `Blob size must be between 0 and ${this.maxBlobBytes}`, + ); + if (await this.options.cas.has(digest)) { + const actualSize = (await this.options.cas.get(digest)).byteLength; + if (actualSize !== size) + throw new BuildConflictError( + "Blob size does not match content already in CAS", + ); + return { digest, size, offset: size, complete: true }; + } + const current = await this.options.store.getUpload(digest); + if (current) { + if (current.spec.size !== size) + throw new BuildConflictError( + "Upload size does not match the existing upload", + ); + return { digest, size, offset: current.status.offset, complete: false }; + } + const timestamp = this.now().toISOString(); + const upload: UploadRecord = { + apiVersion: BUILD_RECORD_API_VERSION, + kind: "BuildUpload", + metadata: { + name: digest.replace(":", "-"), + resourceVersion: "1", + creationTimestamp: timestamp, + }, + spec: { digest, size }, + status: { offset: 0, data: new Uint8Array() }, + }; + const stored = await this.options.store.createUpload(upload); + if (stored.spec.size !== size) + throw new BuildConflictError( + "Upload size does not match the existing upload", + ); + return { digest, size, offset: stored.status.offset, complete: false }; + } + + async uploadBlobChunk( + digest: Sha256Digest, + offset: number, + chunk: Uint8Array, + ): Promise { + assertSha256Digest(digest); + if ( + !(chunk instanceof Uint8Array) || + chunk.byteLength > this.maxUploadChunkBytes + ) + throw new BuildValidationError( + `Upload chunks may not exceed ${this.maxUploadChunkBytes} bytes`, + ); + const current = await this.options.store.getUpload(digest); + if (!current) { + if (await this.options.cas.has(digest)) { + const size = (await this.options.cas.get(digest)).byteLength; + return { digest, size, offset: size, complete: true }; + } + throw new BuildNotFoundError("Blob upload was not initialized"); + } + if (offset !== current.status.offset) + throw new BuildConflictError( + `Upload offset mismatch; expected ${current.status.offset}`, + ); + const nextOffset = offset + chunk.byteLength; + if (nextOffset > current.spec.size) + throw new BuildValidationError("Upload exceeds the declared blob size"); + const data = new Uint8Array(nextOffset); + data.set(current.status.data); + data.set(chunk, offset); + const next = clone(current); + next.metadata.resourceVersion = String( + Number(current.metadata.resourceVersion) + 1, + ); + next.status = { offset: nextOffset, data }; + await this.options.store.replaceUpload( + next, + current.metadata.resourceVersion, + ); + return { + digest, + size: current.spec.size, + offset: nextOffset, + complete: false, + }; + } + + async completeBlobUpload(digest: Sha256Digest): Promise { + assertSha256Digest(digest); + const current = await this.options.store.getUpload(digest); + if (!current) { + if (await this.options.cas.has(digest)) { + const size = (await this.options.cas.get(digest)).byteLength; + return { digest, size, offset: size, complete: true }; + } + throw new BuildNotFoundError("Blob upload was not initialized"); + } + if ( + current.status.offset !== current.spec.size || + current.status.data.byteLength !== current.spec.size + ) { + throw new BuildConflictError( + `Blob upload is incomplete at offset ${current.status.offset}`, + ); + } + try { + await this.options.cas.put(current.status.data, digest); + } catch (error) { + throw new BuildValidationError( + error instanceof Error ? error.message : String(error), + ); + } + await this.options.store.deleteUpload(digest); + return { + digest, + size: current.spec.size, + offset: current.spec.size, + complete: true, + }; + } + + async submitBuild(request: BuildRequest): Promise { + request = clone(request); + if (this.options.imageName) + request.spec.image = this.options.imageName(request); + validateRequest(request); + const existing = await this.options.store.getBuild(request.id); + if (existing) { + if ( + requestFingerprint(existing.spec.request) !== + requestFingerprint(request) + ) + throw new BuildConflictError( + "Build ID was already used for a different request", + ); + return recordStatus(existing); + } + const snapshot = await this.negotiateSnapshot(request.spec.workspace); + if (!snapshot.ready) + throw new BuildConflictError( + `Workspace snapshot is incomplete: ${snapshot.missing.join(", ")}`, + ); + const createdAt = this.now().toISOString(); + const hash = createHash("sha256") + .update(request.id) + .digest("hex") + .slice(0, 24); + const jobName = `kuber-build-${hash}`; + const imageKey = `${request.project}\0${request.service}\0${request.spec.image}`; + const initial: BuildStatus = { + version: BUILD_PROTOCOL_VERSION, + id: request.id, + state: "queued", + createdAt, + }; + const record: BuildRecord = { + apiVersion: BUILD_RECORD_API_VERSION, + kind: "BuildRecord", + metadata: { + name: request.id, + resourceVersion: "1", + creationTimestamp: createdAt, + labels: { project: request.project, service: request.service }, + }, + spec: { + request: clone(request), + imageKey, + jobName, + workspaceSubPath: jobName, + }, + status: { + ...initial, + logBytes: 0, + logOffset: 0, + nextSequence: 1, + events: [{ type: "status", status: initial }], + }, + }; + let stored: BuildRecord; + try { + const result = await this.options.store.createBuild(record); + stored = result.record; + if (!result.created) { + if ( + requestFingerprint(stored.spec.request) !== + requestFingerprint(request) + ) + throw new BuildConflictError( + "Build ID was already used for a different request", + ); + return recordStatus(stored); + } + } catch (error) { + if (error instanceof BuildStoreConflictError) + throw new BuildConflictError(error.message); + throw error; + } + try { + await this.materializer( + this.options.cas, + request.spec.workspace, + join(this.options.workspaceRoot, stored.spec.workspaceSubPath), + ); + const cacheImage = + typeof this.options.cacheImage === "function" + ? this.options.cacheImage(request) + : this.options.cacheImage; + const pushImage = this.options.pushImage + ? this.options.pushImage(request) + : request.spec.image; + const job = createBuildJob({ + name: jobName, + namespace: this.options.namespace, + spec: request.spec, + workspaceClaimName: this.options.workspaceClaimName, + workspaceSubPath: jobWorkspaceSubPath( + this.options.workspaceRoot, + stored.spec.workspaceSubPath, + ), + cacheImage, + pushImage, + pushRegistryInsecure: this.options.pushRegistryInsecure, + cacheRegistryInsecure: + this.options.cacheRegistryInsecure ?? + this.options.pushRegistryInsecure, + buildkitImage: this.options.buildkitImage, + serviceAccountName: this.options.serviceAccountName, + registrySecretName: this.options.registrySecretName, + nodeSelector: this.options.nodeSelector, + tolerations: this.options.tolerations, + }); + await this.options.kubernetes.createJob(job); + await this.updateBuild(stored, (next) => { + next.status.jobCreated = true; + }); + return initial; + } catch (error) { + await this.failBuild( + stored.metadata.name, + error instanceof Error ? error.message : String(error), + ); + throw error; + } + } + + async getBuildStatus(id: string): Promise { + const record = await this.requireBuild(id); + return recordStatus(record); + } + + async getBuildEvents(id: string, afterSequence = 0): Promise { + if (!Number.isSafeInteger(afterSequence) || afterSequence < 0) + throw new BuildValidationError( + "Event sequence must be a non-negative integer", + ); + const record = await this.requireBuild(id); + return clone( + record.status.events.filter( + (event) => event.type === "status" || event.sequence > afterSequence, + ), + ); + } + + async reconcileBuild(id: string): Promise { + let record = await this.requireBuild(id); + if (record.status.state === "succeeded" || record.status.state === "failed") + return recordStatus(record); + if (record.status.jobCreated) record = await this.captureLogs(record); + const observation = await this.options.kubernetes.getJob( + this.options.namespace, + record.spec.jobName, + ); + if (!observation) return recordStatus(record); + if (observation.phase === "running" && record.status.state === "queued") { + record = await this.setState(record, "running", { + startedAt: observation.startedAt ?? this.now().toISOString(), + }); + } else if (observation.phase === "failed") { + record = await this.setState(record, "failed", { + startedAt: record.status.startedAt ?? observation.startedAt, + finishedAt: observation.finishedAt ?? this.now().toISOString(), + error: observation.error ?? "BuildKit Job failed", + }); + } else if (observation.phase === "succeeded") { + try { + const digest = await this.digestResolver( + record.spec.request.spec.image, + ); + assertSha256Digest(digest); + record = await this.setState(record, "succeeded", { + startedAt: record.status.startedAt ?? observation.startedAt, + finishedAt: observation.finishedAt ?? this.now().toISOString(), + digest, + }); + } catch (error) { + record = await this.setState(record, "failed", { + finishedAt: this.now().toISOString(), + error: `Unable to resolve pushed image digest: ${error instanceof Error ? error.message : String(error)}`, + }); + } + } + return recordStatus(record); + } + + async cancelBuild(id: string): Promise { + const record = await this.requireBuild(id); + if (record.status.state === "succeeded" || record.status.state === "failed") + return recordStatus(record); + if (record.status.jobCreated) + await this.options.kubernetes.deleteJob( + this.options.namespace, + record.spec.jobName, + ); + const next = await this.setState(record, "failed", { + finishedAt: this.now().toISOString(), + error: "Build cancelled", + cancelled: true, + }); + return recordStatus(next); + } + + async cleanupBuild(id: string): Promise { + const record = await this.requireBuild(id); + if (record.status.state !== "succeeded" && record.status.state !== "failed") + throw new BuildConflictError("An active build cannot be cleaned up"); + if (record.status.jobCreated) + await this.options.kubernetes.deleteJob( + this.options.namespace, + record.spec.jobName, + ); + await rm(join(this.options.workspaceRoot, record.spec.workspaceSubPath), { + recursive: true, + force: true, + }); + } + + async getBuildResult( + id: string, + ): Promise<{ image: string; digest: Sha256Digest; reference: string }> { + const record = await this.requireBuild(id); + if (record.status.state !== "succeeded" || !record.status.digest) + throw new BuildConflictError("Build has no immutable image result"); + const parsed = parseImageReference(record.spec.request.spec.image); + const image = `${parsed.registry}/${parsed.repository}`; + return { + image, + digest: record.status.digest, + reference: `${image}@${record.status.digest}`, + }; + } + + private async requireBuild(id: string): Promise { + const record = await this.options.store.getBuild(id); + if (!record) throw new BuildNotFoundError(`Build '${id}' not found`); + return record; + } + + private async updateBuild( + record: BuildRecord, + change: (next: BuildRecord) => void, + ): Promise { + const next = clone(record); + next.metadata.resourceVersion = String( + Number(record.metadata.resourceVersion) + 1, + ); + change(next); + await this.options.store.replaceBuild( + next, + record.metadata.resourceVersion, + ); + return this.requireBuild(record.metadata.name); + } + + private async setState( + record: BuildRecord, + state: BuildStatus["state"], + values: Partial, + ): Promise { + if (record.status.state === state && state === "running") return record; + return this.updateBuild(record, (next) => { + Object.assign(next.status, values, { state }); + next.status.events.push({ type: "status", status: recordStatus(next) }); + }); + } + + private async failBuild(id: string, message: string): Promise { + const current = await this.requireBuild(id); + if ( + current.status.state === "succeeded" || + current.status.state === "failed" + ) + return; + await this.setState(current, "failed", { + finishedAt: this.now().toISOString(), + error: message, + }); + } + + private async captureLogs(record: BuildRecord): Promise { + let raw: string | Uint8Array; + try { + raw = await this.options.kubernetes.getJobLogs( + this.options.namespace, + record.spec.jobName, + ); + } catch { + return record; + } + const bytes = typeof raw === "string" ? Buffer.from(raw) : Buffer.from(raw); + const offset = + bytes.byteLength < record.status.logOffset ? 0 : record.status.logOffset; + if (bytes.byteLength === offset) return record; + const delta = bytes.subarray(offset); + return this.updateBuild(record, (next) => { + next.status.logOffset = bytes.byteLength; + for (const message of delta + .toString("utf8") + .split(/(?<=\n)/) + .filter(Boolean)) { + const event: BuildEvent = { + type: "log", + id: record.metadata.name, + sequence: next.status.nextSequence++, + message, + }; + next.status.events.push(event); + next.status.logBytes += Buffer.byteLength(message); + } + while (next.status.logBytes > this.maxLogBytes) { + const index = next.status.events.findIndex( + (event) => event.type === "log", + ); + if (index === -1) break; + const [removed] = next.status.events.splice(index, 1); + if (removed?.type === "log") + next.status.logBytes -= Buffer.byteLength(removed.message); + } + }); + } +} diff --git a/server/build-job.ts b/server/build-job.ts new file mode 100644 index 0000000..4ca0ce2 --- /dev/null +++ b/server/build-job.ts @@ -0,0 +1,218 @@ +import type { BuildArchitecture, BuildSpec } from "../shared/build-protocol"; + +export type BuildJobOptions = { + name: string; + namespace: string; + spec: BuildSpec; + workspaceClaimName: string; + workspaceSubPath?: string; + cacheImage: string; + pushImage?: string; + pushRegistryInsecure?: boolean; + cacheRegistryInsecure?: boolean; + buildkitImage?: string; + serviceAccountName?: string; + registrySecretName?: string; + labels?: Record; + nodeSelector?: Record; + tolerations?: Array>; + ttlSecondsAfterFinished?: number; + backoffLimit?: number; +}; + +export type KubernetesJob = { + apiVersion: "batch/v1"; + kind: "Job"; + metadata: { + name: string; + namespace: string; + labels: Record; + annotations: Record; + }; + spec: Record; +}; + +function relativeBuildPath(path: string, name: string): string { + const normalized = path === "." ? "" : path.replace(/^\.\//, ""); + if ( + !path || + path.startsWith("/") || + path.includes("\\") || + path.includes("\0") || + (normalized !== "" && + normalized + .split("/") + .some((part) => !part || part === ".." || part === ".")) + ) + throw new Error(`${name} must be a safe workspace-relative path`); + return normalized; +} + +function platform(architecture: BuildArchitecture): string { + return `linux/${architecture}`; +} + +export function createBuildJob(options: BuildJobOptions): KubernetesJob { + const contextPath = relativeBuildPath(options.spec.context, "Build context"); + const dockerfilePath = options.spec.dockerfile + ? relativeBuildPath(options.spec.dockerfile, "Dockerfile") + : undefined; + const workspaceSubPath = options.workspaceSubPath + ? relativeBuildPath(options.workspaceSubPath, "Workspace subPath") + : undefined; + if ( + !/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(options.name) || + options.name.length > 63 + ) + throw new Error("Job name must be a valid DNS label"); + + const workspace = "/workspace"; + const context = contextPath ? `${workspace}/${contextPath}` : workspace; + const dockerfile = dockerfilePath + ? `${workspace}/${dockerfilePath}` + : `${context}/Dockerfile`; + const outputImage = options.pushImage ?? options.spec.image; + const importCacheInsecure = options.cacheRegistryInsecure + ? ",registry.insecure=true" + : ""; + const exportCacheInsecure = options.cacheRegistryInsecure + ? ",registry.insecure=true" + : ""; + const outputInsecure = options.pushRegistryInsecure + ? ",registry.insecure=true" + : ""; + const args = [ + "build", + "--frontend=dockerfile.v0", + `--local=context=${context}`, + `--local=dockerfile=${dockerfile.slice(0, dockerfile.lastIndexOf("/"))}`, + `--opt=filename=${dockerfile.slice(dockerfile.lastIndexOf("/") + 1)}`, + `--opt=platform=${platform(options.spec.architecture)}`, + ...(options.spec.target ? [`--opt=target=${options.spec.target}`] : []), + ...options.spec.buildArgs.map((arg) => `--opt=build-arg:${arg}`), + `--import-cache=type=registry,ref=${options.cacheImage}${importCacheInsecure}`, + `--export-cache=type=registry,ref=${options.cacheImage},mode=max${exportCacheInsecure}`, + `--output=type=image,name=${outputImage},push=true${outputInsecure}`, + ]; + const labels = { + "app.kubernetes.io/name": "kuber-buildkit", + "app.kubernetes.io/managed-by": "kuber", + "kuber.astrxl.dev/build": options.name, + ...options.labels, + }; + + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: options.name, + namespace: options.namespace, + labels, + annotations: { + "container.apparmor.security.beta.kubernetes.io/buildkit": "unconfined", + "kuber.astrxl.dev/workspace": options.spec.workspace, + }, + }, + spec: { + backoffLimit: options.backoffLimit ?? 0, + ttlSecondsAfterFinished: options.ttlSecondsAfterFinished ?? 3600, + template: { + metadata: { + labels, + annotations: { + "container.apparmor.security.beta.kubernetes.io/buildkit": + "unconfined", + }, + }, + spec: { + restartPolicy: "Never", + ...(options.serviceAccountName + ? { serviceAccountName: options.serviceAccountName } + : {}), + automountServiceAccountToken: false, + nodeSelector: { + ...options.nodeSelector, + "kubernetes.io/arch": options.spec.architecture, + }, + ...(options.tolerations ? { tolerations: options.tolerations } : {}), + securityContext: { + runAsNonRoot: true, + runAsUser: 1000, + runAsGroup: 1000, + fsGroup: 1000, + seccompProfile: { type: "Unconfined" }, + }, + ...(options.registrySecretName + ? { imagePullSecrets: [{ name: options.registrySecretName }] } + : {}), + containers: [ + { + name: "buildkit", + image: options.buildkitImage ?? "moby/buildkit:rootless", + imagePullPolicy: "IfNotPresent", + command: ["buildctl-daemonless.sh"], + args, + env: [ + { + name: "BUILDKITD_FLAGS", + value: "--oci-worker-no-process-sandbox", + }, + ...(options.registrySecretName + ? [{ name: "DOCKER_CONFIG", value: "/docker-config" }] + : []), + ], + securityContext: { + runAsNonRoot: true, + runAsUser: 1000, + allowPrivilegeEscalation: true, + seccompProfile: { type: "Unconfined" }, + appArmorProfile: { type: "Unconfined" }, + }, + volumeMounts: [ + { + name: "workspace", + mountPath: workspace, + readOnly: true, + ...(workspaceSubPath ? { subPath: workspaceSubPath } : {}), + }, + { + name: "buildkit-state", + mountPath: "/home/user/.local/share/buildkit", + }, + ...(options.registrySecretName + ? [ + { + name: "registry-auth", + mountPath: "/docker-config", + readOnly: true, + }, + ] + : []), + ], + }, + ], + volumes: [ + { + name: "workspace", + persistentVolumeClaim: { claimName: options.workspaceClaimName }, + }, + { name: "buildkit-state", emptyDir: {} }, + ...(options.registrySecretName + ? [ + { + name: "registry-auth", + secret: { + secretName: options.registrySecretName, + items: [ + { key: ".dockerconfigjson", path: "config.json" }, + ], + }, + }, + ] + : []), + ], + }, + }, + }, + }; +} diff --git a/server/build-kubernetes.ts b/server/build-kubernetes.ts new file mode 100644 index 0000000..bce9942 --- /dev/null +++ b/server/build-kubernetes.ts @@ -0,0 +1,429 @@ +import { + BatchV1Api, + CoreV1Api, + KubernetesObjectApi, + type V1Job, +} from "@kubernetes/client-node"; +import { createHash } from "node:crypto"; +import { mkdir, open, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { Sha256Digest } from "../shared/build-protocol"; +import type { + BuildJobObservation, + BuildKubernetesOperations, +} from "./build-controller"; +import type { KubernetesJob } from "./build-job"; +import { + BuildStoreConflictError, + type BuildRecord, + type BuildStore, + type CreateBuildResult, + type UploadRecord, +} from "./build-store"; + +const TYPE_LABEL = "kuber.astrxl.dev/type"; + +type ConfigMap = { + apiVersion: "v1"; + kind: "ConfigMap"; + metadata: { + name: string; + namespace: string; + resourceVersion?: string; + labels?: Record; + }; + data?: Record; +}; + +export interface BuildObjectApi { + create(value: ConfigMap): Promise; + read(value: ConfigMap): Promise; + replace(value: ConfigMap): Promise; + delete(value: ConfigMap): Promise; + list( + apiVersion: string, + kind: string, + namespace?: string, + pretty?: string, + exact?: boolean, + exportValue?: boolean, + fieldSelector?: string, + labelSelector?: string, + ): Promise<{ items: unknown[] }>; +} + +function hashName(prefix: string, value: string): string { + return `${prefix}-${createHash("sha256").update(value).digest("hex").slice(0, 48)}`; +} + +function statusCode(error: unknown): number | undefined { + if (!error || typeof error !== "object") return; + if ("code" in error && typeof error.code === "number") return error.code; + if ("statusCode" in error && typeof error.statusCode === "number") + return error.statusCode; +} + +function payload(value: unknown): T | undefined { + const object = value as ConfigMap; + const raw = object.data?.payload; + if (!raw) return; + try { + const parsed = JSON.parse(raw) as T & { + metadata?: { resourceVersion?: string }; + }; + if (parsed.metadata && object.metadata.resourceVersion) + parsed.metadata.resourceVersion = object.metadata.resourceVersion; + return parsed; + } catch { + return; + } +} + +function map( + namespace: string, + name: string, + type: "build" | "build-upload" | "build-lock", + value?: unknown, + resourceVersion?: string, +): ConfigMap { + return { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name, + namespace, + ...(resourceVersion && { resourceVersion }), + labels: { [TYPE_LABEL]: type }, + }, + ...(value !== undefined && { data: { payload: JSON.stringify(value) } }), + }; +} + +function terminal(record: BuildRecord): boolean { + return record.status.state === "succeeded" || record.status.state === "failed"; +} + +function sameSpec(left: BuildRecord, right: BuildRecord): boolean { + return JSON.stringify(left.spec) === JSON.stringify(right.spec); +} + +/** Build metadata lives in ConfigMaps; resumable upload bytes live only on the RWX volume. */ +export class KubernetesBuildStore implements BuildStore { + constructor( + private readonly objects: BuildObjectApi, + private readonly namespace: string, + private readonly uploadRoot: string, + ) {} + + private buildName(id: string): string { + return hashName("build", id); + } + + private uploadName(digest: Sha256Digest): string { + return hashName("upload", digest); + } + + private uploadPath(digest: Sha256Digest): string { + return join(this.uploadRoot, digest.slice("sha256:".length)); + } + + private lockName(imageKey: string): string { + return hashName("build-lock", imageKey); + } + + private async read(value: ConfigMap): Promise { + try { + return payload(await this.objects.read(value)); + } catch (error) { + if (statusCode(error) === 404) return; + throw error; + } + } + + private async delete(value: ConfigMap): Promise { + try { + await this.objects.delete(value); + } catch (error) { + if (statusCode(error) !== 404) throw error; + } + } + + async createBuild(record: BuildRecord): Promise { + const existing = await this.getBuild(record.metadata.name); + if (existing) { + if (!sameSpec(existing, record)) + throw new BuildStoreConflictError( + "Build ID was already used for a different request", + ); + return { record: existing, created: false }; + } + + const lockName = this.lockName(record.spec.imageKey); + try { + await this.objects.create( + map(this.namespace, lockName, "build-lock", { + buildId: record.metadata.name, + }), + ); + } catch (error) { + if (statusCode(error) !== 409) throw error; + const lock = await this.read<{ buildId: string }>( + map(this.namespace, lockName, "build-lock"), + ); + const active = lock && (await this.getBuild(lock.buildId)); + if (!active || terminal(active)) { + await this.delete(map(this.namespace, lockName, "build-lock")); + return this.createBuild(record); + } + throw new BuildStoreConflictError( + `Build '${active.metadata.name}' is already active for ${record.spec.imageKey}`, + ); + } + + try { + const created = (await this.objects.create( + map(this.namespace, this.buildName(record.metadata.name), "build", record), + )) as ConfigMap; + return { record: payload(created) ?? record, created: true }; + } catch (error) { + await this.delete(map(this.namespace, lockName, "build-lock")); + if (statusCode(error) === 409) { + const concurrent = await this.getBuild(record.metadata.name); + if (concurrent && sameSpec(concurrent, record)) + return { record: concurrent, created: false }; + throw new BuildStoreConflictError( + "Build ID was already used for a different request", + ); + } + throw error; + } + } + + async getBuild(id: string): Promise { + const record = await this.read( + map(this.namespace, this.buildName(id), "build"), + ); + return record?.metadata.name === id ? record : undefined; + } + + async listBuilds(): Promise { + const result = await this.objects.list( + "v1", + "ConfigMap", + this.namespace, + undefined, + undefined, + undefined, + undefined, + `${TYPE_LABEL}=build`, + ); + return result.items + .map((item) => payload(item)) + .filter((item): item is BuildRecord => item?.kind === "BuildRecord") + .sort((a, b) => + a.metadata.creationTimestamp.localeCompare(b.metadata.creationTimestamp), + ); + } + + async replaceBuild( + record: BuildRecord, + expectedResourceVersion: string, + ): Promise { + const current = await this.getBuild(record.metadata.name); + if (!current || current.metadata.resourceVersion !== expectedResourceVersion) + throw new BuildStoreConflictError("Build record was concurrently modified"); + if (!sameSpec(current, record)) + throw new BuildStoreConflictError("Build specification is immutable"); + if ( + current.status.state === "succeeded" && + (record.status.state !== "succeeded" || + current.status.digest !== record.status.digest) + ) + throw new BuildStoreConflictError( + "A successful image digest is immutable", + ); + + try { + await this.objects.replace( + map( + this.namespace, + this.buildName(record.metadata.name), + "build", + record, + expectedResourceVersion, + ), + ); + } catch (error) { + if (statusCode(error) === 409) + throw new BuildStoreConflictError("Build record was concurrently modified"); + throw error; + } + if (terminal(record)) + await this.delete( + map(this.namespace, this.lockName(record.spec.imageKey), "build-lock"), + ); + } + + async getUpload(digest: Sha256Digest): Promise { + const record = await this.read( + map(this.namespace, this.uploadName(digest), "build-upload"), + ); + if (!record || record.spec.digest !== digest) return; + try { + record.status.data = new Uint8Array(await readFile(this.uploadPath(digest))); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + record.status.data = new Uint8Array(); + } + if (record.status.data.byteLength !== record.status.offset) + throw new Error(`Upload file for ${digest} does not match its record`); + return record; + } + + async createUpload(record: UploadRecord): Promise { + const existing = await this.getUpload(record.spec.digest); + if (existing) return existing; + await mkdir(this.uploadRoot, { recursive: true, mode: 0o700 }); + const path = this.uploadPath(record.spec.digest); + const handle = await open(path, "wx", 0o600).catch((error) => { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return; + throw error; + }); + await handle?.close(); + const metadata = structuredClone(record); + metadata.status.data = new Uint8Array(); + try { + const created = (await this.objects.create( + map( + this.namespace, + this.uploadName(record.spec.digest), + "build-upload", + metadata, + ), + )) as ConfigMap; + const result = payload(created) ?? metadata; + result.status.data = new Uint8Array(); + return result; + } catch (error) { + if (statusCode(error) === 409) return (await this.getUpload(record.spec.digest))!; + await rm(path, { force: true }); + throw error; + } + } + + async replaceUpload( + record: UploadRecord, + expectedResourceVersion: string, + ): Promise { + const metadata = structuredClone(record); + metadata.status.data = new Uint8Array(); + try { + await this.objects.replace( + map( + this.namespace, + this.uploadName(record.spec.digest), + "build-upload", + metadata, + expectedResourceVersion, + ), + ); + await writeFile(this.uploadPath(record.spec.digest), record.status.data, { + mode: 0o600, + }); + } catch (error) { + if (statusCode(error) === 409) + throw new BuildStoreConflictError("Upload record was concurrently modified"); + throw error; + } + } + + async deleteUpload(digest: Sha256Digest): Promise { + await this.delete( + map(this.namespace, this.uploadName(digest), "build-upload"), + ); + await rm(this.uploadPath(digest), { force: true }); + } +} + +export class KubernetesBuildOperations implements BuildKubernetesOperations { + constructor( + private readonly batch: BatchV1Api, + private readonly core: CoreV1Api, + ) {} + + async createJob(job: KubernetesJob): Promise { + await this.batch.createNamespacedJob({ + namespace: job.metadata.namespace, + body: job as unknown as V1Job, + fieldManager: "kuber-server", + fieldValidation: "Strict", + }); + } + + async getJob( + namespace: string, + name: string, + ): Promise { + let job: V1Job; + try { + job = await this.batch.readNamespacedJob({ namespace, name }); + } catch (error) { + if (statusCode(error) === 404) return; + throw error; + } + const failed = job.status?.conditions?.find( + (condition) => condition.type === "Failed" && condition.status === "True", + ); + const complete = job.status?.conditions?.find( + (condition) => condition.type === "Complete" && condition.status === "True", + ); + const phase = failed + ? "failed" + : complete + ? "succeeded" + : (job.status?.active ?? 0) > 0 + ? "running" + : "queued"; + return { + phase, + startedAt: job.status?.startTime?.toISOString(), + finishedAt: job.status?.completionTime?.toISOString(), + ...(failed?.message && { error: failed.message }), + }; + } + + async getJobLogs(namespace: string, name: string): Promise { + const pods = await this.core.listNamespacedPod({ + namespace, + labelSelector: `job-name=${name}`, + }); + const pod = pods.items + .sort((a, b) => + (a.metadata?.creationTimestamp?.getTime() ?? 0) - + (b.metadata?.creationTimestamp?.getTime() ?? 0), + ) + .at(-1); + if (!pod?.metadata?.name) return ""; + return this.core.readNamespacedPodLog({ + namespace, + name: pod.metadata.name, + container: "buildkit", + }); + } + + async deleteJob(namespace: string, name: string): Promise { + try { + await this.batch.deleteNamespacedJob({ + namespace, + name, + propagationPolicy: "Background", + }); + } catch (error) { + if (statusCode(error) !== 404) throw error; + } + } +} + +export function buildObjectApi(objects: KubernetesObjectApi): BuildObjectApi { + return objects as unknown as BuildObjectApi; +} diff --git a/server/build-store.ts b/server/build-store.ts new file mode 100644 index 0000000..27c3c80 --- /dev/null +++ b/server/build-store.ts @@ -0,0 +1,188 @@ +import type { + BuildEvent, + BuildRequest, + BuildStatus, + Sha256Digest, +} from "../shared/build-protocol"; + +export const BUILD_RECORD_API_VERSION = "kuber.astrxl.dev/v1" as const; + +export interface BuildRecord { + apiVersion: typeof BUILD_RECORD_API_VERSION; + kind: "BuildRecord"; + metadata: { + name: string; + resourceVersion: string; + creationTimestamp: string; + labels: Record; + }; + spec: { + request: BuildRequest; + imageKey: string; + jobName: string; + workspaceSubPath: string; + }; + status: BuildStatus & { + cancelled?: boolean; + jobCreated?: boolean; + logBytes: number; + logOffset: number; + nextSequence: number; + events: BuildEvent[]; + }; +} + +export interface UploadRecord { + apiVersion: typeof BUILD_RECORD_API_VERSION; + kind: "BuildUpload"; + metadata: { + name: string; + resourceVersion: string; + creationTimestamp: string; + }; + spec: { digest: Sha256Digest; size: number }; + status: { offset: number; data: Uint8Array }; +} + +export type CreateBuildResult = { record: BuildRecord; created: boolean }; + +/** Implementations must make createBuild and replace operations atomic. */ +export interface BuildStore { + createBuild(record: BuildRecord): Promise; + getBuild(id: string): Promise; + listBuilds(): Promise; + replaceBuild( + record: BuildRecord, + expectedResourceVersion: string, + ): Promise; + getUpload(digest: Sha256Digest): Promise; + createUpload(record: UploadRecord): Promise; + replaceUpload( + record: UploadRecord, + expectedResourceVersion: string, + ): Promise; + deleteUpload(digest: Sha256Digest): Promise; +} + +export class BuildStoreConflictError extends Error { + readonly code = "BUILD_STORE_CONFLICT"; +} + +function clone(value: T): T { + return structuredClone(value); +} + +function terminal(record: BuildRecord): boolean { + return ( + record.status.state === "succeeded" || record.status.state === "failed" + ); +} + +function sameSpec(left: BuildRecord, right: BuildRecord): boolean { + return JSON.stringify(left.spec) === JSON.stringify(right.spec); +} + +export class MemoryBuildStore implements BuildStore { + private readonly builds = new Map(); + private readonly uploads = new Map(); + private readonly active = new Map(); + + async createBuild(record: BuildRecord): Promise { + const existing = this.builds.get(record.metadata.name); + if (existing) { + if (!sameSpec(existing, record)) { + throw new BuildStoreConflictError( + "Build ID was already used for a different request", + ); + } + return { record: clone(existing), created: false }; + } + const activeId = this.active.get(record.spec.imageKey); + if (activeId) { + const active = this.builds.get(activeId); + if (active && !terminal(active)) { + throw new BuildStoreConflictError( + `Build '${activeId}' is already active for ${record.spec.imageKey}`, + ); + } + } + this.builds.set(record.metadata.name, clone(record)); + this.active.set(record.spec.imageKey, record.metadata.name); + return { record: clone(record), created: true }; + } + + async getBuild(id: string) { + const value = this.builds.get(id); + return value && clone(value); + } + + async listBuilds() { + return [...this.builds.values()] + .sort((a, b) => + a.metadata.creationTimestamp.localeCompare( + b.metadata.creationTimestamp, + ), + ) + .map(clone); + } + + async replaceBuild(record: BuildRecord, expectedResourceVersion: string) { + const current = this.builds.get(record.metadata.name); + if ( + !current || + current.metadata.resourceVersion !== expectedResourceVersion + ) { + throw new BuildStoreConflictError( + "Build record was concurrently modified", + ); + } + if (!sameSpec(current, record)) { + throw new BuildStoreConflictError("Build specification is immutable"); + } + if ( + current.status.state === "succeeded" && + (record.status.state !== "succeeded" || + record.status.digest !== current.status.digest) + ) { + throw new BuildStoreConflictError( + "A successful image digest is immutable", + ); + } + this.builds.set(record.metadata.name, clone(record)); + if ( + terminal(record) && + this.active.get(record.spec.imageKey) === record.metadata.name + ) { + this.active.delete(record.spec.imageKey); + } + } + + async getUpload(digest: Sha256Digest) { + const value = this.uploads.get(digest); + return value && clone(value); + } + + async createUpload(record: UploadRecord) { + const current = this.uploads.get(record.spec.digest); + if (current) return clone(current); + this.uploads.set(record.spec.digest, clone(record)); + return clone(record); + } + + async replaceUpload(record: UploadRecord, expectedResourceVersion: string) { + const current = this.uploads.get(record.spec.digest); + if ( + !current || + current.metadata.resourceVersion !== expectedResourceVersion + ) { + throw new BuildStoreConflictError( + "Upload record was concurrently modified", + ); + } + this.uploads.set(record.spec.digest, clone(record)); + } + + async deleteUpload(digest: Sha256Digest) { + this.uploads.delete(digest); + } +} diff --git a/server/cas.ts b/server/cas.ts new file mode 100644 index 0000000..88ce82c --- /dev/null +++ b/server/cas.ts @@ -0,0 +1,99 @@ +import { createHash, randomBytes } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, rename, rm } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { + assertSha256Digest, + type Sha256Digest, +} from "../shared/build-protocol"; + +function digest(data: Uint8Array): Sha256Digest { + return `sha256:${createHash("sha256").update(data).digest("hex")}`; +} + +export class FilesystemCas { + readonly root: string; + + constructor(root: string) { + this.root = root; + } + + private path(blobDigest: Sha256Digest): string { + assertSha256Digest(blobDigest); + return join( + this.root, + "sha256", + blobDigest.slice(7, 9), + blobDigest.slice(9), + ); + } + + async has(blobDigest: Sha256Digest): Promise { + try { + return (await lstat(this.path(blobDigest))).isFile(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } + } + + async put(data: Uint8Array, expected?: Sha256Digest): Promise { + const actual = digest(data); + if (expected !== undefined) { + assertSha256Digest(expected); + if (actual !== expected) + throw new Error( + `CAS digest mismatch: expected ${expected}, got ${actual}`, + ); + } + + const target = this.path(actual); + const directory = dirname(target); + await mkdir(directory, { recursive: true, mode: 0o755 }); + if (await this.has(actual)) { + await this.get(actual); + return actual; + } + + const temporary = `${target}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`; + const handle = await open( + temporary, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, + 0o644, + ); + try { + await handle.writeFile(data); + await handle.sync(); + } finally { + await handle.close(); + } + try { + await rename(temporary, target); + } finally { + await rm(temporary, { force: true }); + } + return actual; + } + + async get(blobDigest: Sha256Digest): Promise { + const handle = await open( + this.path(blobDigest), + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + let data: Uint8Array; + try { + const blobStat = await handle.stat(); + if (!blobStat.isFile()) + throw new Error(`CAS blob is not a regular file: ${blobDigest}`); + data = await handle.readFile(); + } finally { + await handle.close(); + } + const actual = digest(data); + if (actual !== blobDigest) + throw new Error( + `Corrupt CAS blob: expected ${blobDigest}, got ${actual}`, + ); + return data; + } +} diff --git a/server/exec-service.ts b/server/exec-service.ts new file mode 100644 index 0000000..988b791 --- /dev/null +++ b/server/exec-service.ts @@ -0,0 +1,729 @@ +export const EXEC_MANAGED_BY_LABEL = "app.kubernetes.io/managed-by"; +export const EXEC_MANAGED_BY_VALUE = "kuber"; +export const EXEC_WORKSPACE_UID_LABEL = "kuber.dev/workspace-uid"; + +export const DEFAULT_MAX_COMMAND_ARGUMENTS = 64; +export const DEFAULT_MAX_COMMAND_BYTES = 32 * 1024; +export const DEFAULT_MAX_ARGUMENT_BYTES = 8 * 1024; +export const DEFAULT_OUTPUT_CAP_BYTES = 1024 * 1024; +export const DEFAULT_INTERACTIVE_QUEUE_CAPACITY = 128; +export const DEFAULT_MAX_STDIN_FRAME_BYTES = 64 * 1024; + +const DNS_LABEL = /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/; +const DNS_SUBDOMAIN = + /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*$/; +const textEncoder = new TextEncoder(); + +export type ExecWorkspace = { + project: string; + uid: string; +}; + +export type ExecDeployment = { + name: string; + uid: string; + labels: Readonly>; + selector: Readonly>; + containers: readonly string[]; +}; + +export type ExecPodContainer = { + name: string; + running: boolean; + ready?: boolean; +}; + +export type ExecPod = { + name: string; + uid: string; + /** The controlling Deployment UID, resolved through the ReplicaSet owner. */ + deploymentUid: string; + phase?: string; + deletionTimestamp?: string; + containers: readonly ExecPodContainer[]; +}; + +export type ResolvedExecTarget = { + namespace: string; + deployment: string; + deploymentUid: string; + pod: string; + podUid: string; + container: string; +}; + +export type KubernetesExecRequest = ResolvedExecTarget & { + command: readonly string[]; + tty: boolean; +}; + +export type ExecChunk = string | Uint8Array; + +export type KubernetesExecExit = { + exitCode: number; + reason?: string; + message?: string; +}; + +export interface KubernetesExecProcess { + stdout: AsyncIterable; + stderr: AsyncIterable; + writeStdin(data: Uint8Array): void | Promise; + closeStdin(): void | Promise; + resize(columns: number, rows: number): void | Promise; + wait(): Promise; + close(): void | Promise; +} + +/** All cluster-specific behavior, including @kubernetes/client-node, lives here. */ +export interface KubernetesExecBackend { + getDeployment( + namespace: string, + name: string, + signal?: AbortSignal, + ): Promise; + listPods( + namespace: string, + selector: Readonly>, + signal?: AbortSignal, + ): Promise; + exec( + request: KubernetesExecRequest, + signal: AbortSignal, + ): Promise; +} + +export type ExecErrorCode = + | "EXEC_INVALID" + | "EXEC_TARGET_NOT_FOUND" + | "EXEC_TARGET_FORBIDDEN" + | "EXEC_TARGET_NOT_READY" + | "EXEC_OUTPUT_LIMIT" + | "EXEC_ABORTED" + | "EXEC_FAILED"; + +export class ExecServiceError extends Error { + constructor( + readonly code: ExecErrorCode, + message: string, + ) { + super(message); + this.name = "ExecServiceError"; + } +} + +export class ExecOutputLimitError extends ExecServiceError { + constructor( + readonly stream: "stdout" | "stderr", + readonly limitBytes: number, + ) { + super( + "EXEC_OUTPUT_LIMIT", + `${stream} exceeded the ${limitBytes} byte output limit`, + ); + this.name = "ExecOutputLimitError"; + } +} + +export type ExecInput = { + workspace: ExecWorkspace; + deployment: string; + container?: string; + command: readonly string[]; + signal?: AbortSignal; +}; + +export type NonTtyExecInput = ExecInput & { + stdoutCapBytes?: number; + stderrCapBytes?: number; +}; + +export type NonTtyExecResult = ResolvedExecTarget & + KubernetesExecExit & { + stdout: string; + stderr: string; + }; + +export type ExecClientFrame = + | { type: "stdin"; data: string | Uint8Array; eof?: boolean } + | { type: "resize"; columns: number; rows: number } + | { type: "close" }; + +export type ExecServerFrame = + | { type: "stdout"; data: Uint8Array } + | { type: "stderr"; data: Uint8Array } + | ({ type: "exit" } & KubernetesExecExit) + | { type: "error"; code: ExecErrorCode; message: string }; + +export interface InteractiveExecSession extends AsyncIterable { + readonly target: ResolvedExecTarget; + send(frame: ExecClientFrame): Promise; + close(): Promise; +} + +export type InteractiveExecInput = ExecInput & { + signal: AbortSignal; + tty?: boolean; + queueCapacity?: number; +}; + +export type ExecServiceOptions = { + maxCommandArguments?: number; + maxCommandBytes?: number; + maxArgumentBytes?: number; + maxOutputCapBytes?: number; + defaultOutputCapBytes?: number; + interactiveQueueCapacity?: number; + maxStdinFrameBytes?: number; +}; + +type NormalizedLimits = Required; + +function positiveInteger(value: number, name: string): number { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} + +function limitsFrom(options: ExecServiceOptions): NormalizedLimits { + const limits = { + maxCommandArguments: positiveInteger( + options.maxCommandArguments ?? DEFAULT_MAX_COMMAND_ARGUMENTS, + "maxCommandArguments", + ), + maxCommandBytes: positiveInteger( + options.maxCommandBytes ?? DEFAULT_MAX_COMMAND_BYTES, + "maxCommandBytes", + ), + maxArgumentBytes: positiveInteger( + options.maxArgumentBytes ?? DEFAULT_MAX_ARGUMENT_BYTES, + "maxArgumentBytes", + ), + maxOutputCapBytes: positiveInteger( + options.maxOutputCapBytes ?? DEFAULT_OUTPUT_CAP_BYTES, + "maxOutputCapBytes", + ), + defaultOutputCapBytes: positiveInteger( + options.defaultOutputCapBytes ?? DEFAULT_OUTPUT_CAP_BYTES, + "defaultOutputCapBytes", + ), + interactiveQueueCapacity: positiveInteger( + options.interactiveQueueCapacity ?? DEFAULT_INTERACTIVE_QUEUE_CAPACITY, + "interactiveQueueCapacity", + ), + maxStdinFrameBytes: positiveInteger( + options.maxStdinFrameBytes ?? DEFAULT_MAX_STDIN_FRAME_BYTES, + "maxStdinFrameBytes", + ), + }; + if (limits.defaultOutputCapBytes > limits.maxOutputCapBytes) { + throw new Error("defaultOutputCapBytes cannot exceed maxOutputCapBytes"); + } + return limits; +} + +function invalid(message: string): never { + throw new ExecServiceError("EXEC_INVALID", message); +} + +function validateName( + value: string, + kind: string, + maxLength = 63, + pattern = DNS_LABEL, +): void { + if (!value || value.length > maxLength || !pattern.test(value)) { + invalid( + `${kind} must be a valid lowercase DNS name of at most ${maxLength} characters`, + ); + } +} + +function validateInput(input: ExecInput, limits: NormalizedLimits): void { + validateName(input.workspace.project, "Workspace project"); + if (!input.workspace.uid?.trim() || input.workspace.uid.length > 256) { + invalid("Workspace UID must be between 1 and 256 characters"); + } + validateName(input.deployment, "Deployment name", 253, DNS_SUBDOMAIN); + if (input.container !== undefined) + validateName(input.container, "Container name"); + if (!Array.isArray(input.command) || input.command.length === 0) { + invalid("Command is required"); + } + if (input.command.length > limits.maxCommandArguments) { + invalid( + `Command cannot contain more than ${limits.maxCommandArguments} arguments`, + ); + } + let commandBytes = 0; + for (const argument of input.command) { + if (typeof argument !== "string" || argument.includes("\0")) { + invalid("Command arguments must be strings without null bytes"); + } + const bytes = textEncoder.encode(argument).byteLength; + if (bytes > limits.maxArgumentBytes) { + invalid(`Command argument exceeds ${limits.maxArgumentBytes} bytes`); + } + commandBytes += bytes; + } + if (commandBytes > limits.maxCommandBytes) { + invalid(`Command exceeds ${limits.maxCommandBytes} bytes`); + } +} + +function outputCap( + value: number | undefined, + stream: "stdout" | "stderr", + limits: NormalizedLimits, +): number { + const cap = value ?? limits.defaultOutputCapBytes; + if ( + !Number.isSafeInteger(cap) || + cap <= 0 || + cap > limits.maxOutputCapBytes + ) { + invalid( + `${stream}CapBytes must be between 1 and ${limits.maxOutputCapBytes}`, + ); + } + return cap; +} + +function aborted(): ExecServiceError { + return new ExecServiceError("EXEC_ABORTED", "Exec request was aborted"); +} + +function asBytes(chunk: ExecChunk): Uint8Array { + return typeof chunk === "string" ? textEncoder.encode(chunk) : chunk; +} + +async function collect( + chunks: AsyncIterable, + stream: "stdout" | "stderr", + cap: number, +): Promise { + const values: Uint8Array[] = []; + let size = 0; + for await (const chunk of chunks) { + const bytes = asBytes(chunk); + size += bytes.byteLength; + if (size > cap) throw new ExecOutputLimitError(stream, cap); + values.push(bytes); + } + const output = new Uint8Array(size); + let offset = 0; + for (const value of values) { + output.set(value, offset); + offset += value.byteLength; + } + return new TextDecoder().decode(output); +} + +type QueueReader = (value: IteratorResult) => void; + +class BoundedQueue implements AsyncIterable { + private readonly values: T[] = []; + private readonly readers: QueueReader[] = []; + private readonly writers: Array<() => void> = []; + private closed = false; + + constructor(private readonly capacity: number) {} + + async push(value: T): Promise { + while (!this.closed) { + const reader = this.readers.shift(); + if (reader) { + reader({ value, done: false }); + return true; + } + if (this.values.length < this.capacity) { + this.values.push(value); + return true; + } + await new Promise((resolve) => this.writers.push(resolve)); + } + return false; + } + + close(discard = false): void { + if (this.closed) return; + this.closed = true; + if (discard) this.values.length = 0; + if (this.values.length === 0) { + for (const reader of this.readers.splice(0)) + reader({ value: undefined, done: true }); + } + for (const writer of this.writers.splice(0)) writer(); + } + + [Symbol.asyncIterator](): AsyncIterator { + return { + next: () => { + const value = this.values.shift(); + if (value !== undefined) { + this.writers.shift()?.(); + return Promise.resolve({ value, done: false }); + } + if (this.closed) + return Promise.resolve({ value: undefined, done: true }); + return new Promise>((resolve) => + this.readers.push(resolve), + ); + }, + }; + } +} + +function safeError(error: unknown): { code: ExecErrorCode; message: string } { + if (error instanceof ExecServiceError) { + return { code: error.code, message: error.message }; + } + return { + code: "EXEC_FAILED", + message: error instanceof Error ? error.message : "Kubernetes exec failed", + }; +} + +async function closeQuietly(process: KubernetesExecProcess): Promise { + try { + await process.close(); + } catch { + // Closing an already-ended Kubernetes transport is harmless. + } +} + +class DuplexSession implements InteractiveExecSession { + private readonly queue: BoundedQueue; + private readonly done: Promise; + private closedByClient = false; + private readonly onExternalAbort: () => void; + + constructor( + readonly target: ResolvedExecTarget, + private readonly process: KubernetesExecProcess, + private readonly externalSignal: AbortSignal, + private readonly controller: AbortController, + private readonly maxStdinFrameBytes: number, + queueCapacity: number, + ) { + this.queue = new BoundedQueue(queueCapacity); + this.onExternalAbort = () => this.stop(true); + if (externalSignal.aborted) this.onExternalAbort(); + else + externalSignal.addEventListener("abort", this.onExternalAbort, { + once: true, + }); + this.done = this.run(); + } + + [Symbol.asyncIterator](): AsyncIterator { + return this.queue[Symbol.asyncIterator](); + } + + async send(frame: ExecClientFrame): Promise { + if (this.controller.signal.aborted) throw aborted(); + switch (frame.type) { + case "stdin": { + if ( + typeof frame.data !== "string" && + !(frame.data instanceof Uint8Array) + ) { + invalid("stdin data must be a string or Uint8Array"); + } + const data = asBytes(frame.data); + if (data.byteLength > this.maxStdinFrameBytes) { + invalid(`stdin frame exceeds ${this.maxStdinFrameBytes} bytes`); + } + if (data.byteLength) await this.process.writeStdin(data); + if (frame.eof) await this.process.closeStdin(); + return; + } + case "resize": + if ( + !Number.isSafeInteger(frame.columns) || + !Number.isSafeInteger(frame.rows) || + frame.columns < 1 || + frame.rows < 1 || + frame.columns > 65_535 || + frame.rows > 65_535 + ) { + invalid("Terminal dimensions must be integers between 1 and 65535"); + } + await this.process.resize(frame.columns, frame.rows); + return; + case "close": + await this.close(); + return; + default: + invalid("Unknown interactive exec frame"); + } + } + + async close(): Promise { + this.closedByClient = true; + this.stop(true); + await this.done; + } + + private stop(discard: boolean): void { + this.controller.abort(); + this.queue.close(discard); + void closeQuietly(this.process); + } + + private async pump( + stream: "stdout" | "stderr", + chunks: AsyncIterable, + ): Promise { + for await (const chunk of chunks) { + if (this.controller.signal.aborted) return; + await this.queue.push({ type: stream, data: asBytes(chunk) }); + } + } + + private async run(): Promise { + try { + const [, , status] = await Promise.all([ + this.pump("stdout", this.process.stdout), + this.pump("stderr", this.process.stderr), + this.process.wait(), + ]); + if (!this.controller.signal.aborted) { + await this.queue.push({ type: "exit", ...status }); + } + } catch (error) { + if (!this.externalSignal.aborted && !this.closedByClient) { + await this.queue.push({ type: "error", ...safeError(error) }); + } + } finally { + this.externalSignal.removeEventListener("abort", this.onExternalAbort); + this.controller.abort(); + await closeQuietly(this.process); + this.queue.close(); + } + } +} + +export class ExecService { + private readonly limits: NormalizedLimits; + + constructor( + private readonly backend: KubernetesExecBackend, + options: ExecServiceOptions = {}, + ) { + this.limits = limitsFrom(options); + } + + async resolveTarget(input: ExecInput): Promise { + validateInput(input, this.limits); + if (input.signal?.aborted) throw aborted(); + const namespace = input.workspace.project; + const deployment = await this.backend.getDeployment( + namespace, + input.deployment, + input.signal, + ); + if (!deployment) { + throw new ExecServiceError( + "EXEC_TARGET_NOT_FOUND", + `Deployment ${input.deployment} was not found`, + ); + } + if (deployment.name !== input.deployment) { + throw new ExecServiceError( + "EXEC_TARGET_NOT_FOUND", + `Deployment ${input.deployment} was not found`, + ); + } + if ( + deployment.labels[EXEC_MANAGED_BY_LABEL] !== EXEC_MANAGED_BY_VALUE || + deployment.labels[EXEC_WORKSPACE_UID_LABEL] !== input.workspace.uid + ) { + throw new ExecServiceError( + "EXEC_TARGET_FORBIDDEN", + `Deployment ${input.deployment} is not owned by this workspace`, + ); + } + if (!deployment.uid || Object.keys(deployment.selector).length === 0) { + throw new ExecServiceError( + "EXEC_TARGET_NOT_READY", + `Deployment ${input.deployment} has no usable pod selector`, + ); + } + + const requestedContainer = input.container ?? deployment.containers[0]; + if (!requestedContainer) { + throw new ExecServiceError( + "EXEC_TARGET_NOT_READY", + `Deployment ${input.deployment} has no container`, + ); + } + + const pods = await this.backend.listPods( + namespace, + deployment.selector, + input.signal, + ); + const running = pods + .filter( + (pod) => + pod.phase === "Running" && + !pod.deletionTimestamp && + pod.deploymentUid === deployment.uid && + Boolean(pod.name && pod.uid) && + pod.containers.some( + (container) => + container.name === requestedContainer && container.running, + ), + ) + .sort((left, right) => { + const leftReady = left.containers.find( + (container) => container.name === requestedContainer, + )?.ready + ? 1 + : 0; + const rightReady = right.containers.find( + (container) => container.name === requestedContainer, + )?.ready + ? 1 + : 0; + return rightReady - leftReady || left.name.localeCompare(right.name); + }); + const pod = running[0]; + if (!pod) { + const hasOwnedRunningPod = pods.some( + (candidate) => + candidate.phase === "Running" && + !candidate.deletionTimestamp && + candidate.deploymentUid === deployment.uid, + ); + throw new ExecServiceError( + input.container && hasOwnedRunningPod + ? "EXEC_TARGET_NOT_FOUND" + : "EXEC_TARGET_NOT_READY", + input.container && hasOwnedRunningPod + ? `Container ${requestedContainer} was not found or running in a deployment pod` + : `Deployment ${input.deployment} has no running owned pod/container`, + ); + } + return { + namespace, + deployment: deployment.name, + deploymentUid: deployment.uid, + pod: pod.name, + podUid: pod.uid, + container: requestedContainer, + }; + } + + async execute(input: NonTtyExecInput): Promise { + const stdoutCap = outputCap(input.stdoutCapBytes, "stdout", this.limits); + const stderrCap = outputCap(input.stderrCapBytes, "stderr", this.limits); + const target = await this.resolveTarget(input); + const controller = new AbortController(); + let process: KubernetesExecProcess | undefined; + const onAbort = () => { + controller.abort(); + if (process) void closeQuietly(process); + }; + if (input.signal?.aborted) onAbort(); + else input.signal?.addEventListener("abort", onAbort, { once: true }); + try { + if (controller.signal.aborted) throw aborted(); + process = await this.backend.exec( + { ...target, command: [...input.command], tty: false }, + controller.signal, + ); + const stopOnFailure = (promise: Promise): Promise => + promise.catch((error) => { + controller.abort(); + if (process) void closeQuietly(process); + throw error; + }); + const stdout = stopOnFailure( + collect(process.stdout, "stdout", stdoutCap), + ); + const stderr = stopOnFailure( + collect(process.stderr, "stderr", stderrCap), + ); + const results = await Promise.allSettled([ + stdout, + stderr, + process.wait(), + ]); + const failure = results.find( + (result): result is PromiseRejectedResult => + result.status === "rejected", + ); + if (failure) { + controller.abort(); + await closeQuietly(process); + if (input.signal?.aborted) throw aborted(); + if (failure.reason instanceof ExecServiceError) throw failure.reason; + throw new ExecServiceError( + "EXEC_FAILED", + safeError(failure.reason).message, + ); + } + const [stdoutResult, stderrResult, statusResult] = results as [ + PromiseFulfilledResult, + PromiseFulfilledResult, + PromiseFulfilledResult, + ]; + if (input.signal?.aborted) throw aborted(); + return { + ...target, + ...statusResult.value, + stdout: stdoutResult.value, + stderr: stderrResult.value, + }; + } finally { + input.signal?.removeEventListener("abort", onAbort); + controller.abort(); + if (process) await closeQuietly(process); + } + } + + async openInteractive( + input: InteractiveExecInput, + ): Promise { + const queueCapacity = positiveInteger( + input.queueCapacity ?? this.limits.interactiveQueueCapacity, + "queueCapacity", + ); + validateInput(input, this.limits); + if (input.signal.aborted) throw aborted(); + const target = await this.resolveTarget(input); + if (input.signal.aborted) throw aborted(); + const controller = new AbortController(); + const onAbort = () => controller.abort(); + input.signal.addEventListener("abort", onAbort, { once: true }); + try { + const process = await this.backend.exec( + { ...target, command: [...input.command], tty: input.tty ?? true }, + controller.signal, + ); + input.signal.removeEventListener("abort", onAbort); + return new DuplexSession( + target, + process, + input.signal, + controller, + this.limits.maxStdinFrameBytes, + queueCapacity, + ); + } catch (error) { + input.signal.removeEventListener("abort", onAbort); + if (input.signal.aborted) throw aborted(); + throw error; + } + } +} + +export function createExecService( + backend: KubernetesExecBackend, + options?: ExecServiceOptions, +): ExecService { + return new ExecService(backend, options); +} diff --git a/server/index.ts b/server/index.ts new file mode 100644 index 0000000..db21711 --- /dev/null +++ b/server/index.ts @@ -0,0 +1,257 @@ +import { createApp, cleanupExpiredSessions } from "./app"; +import { RedactingAuditStore } from "./audit-store"; +import { readFile } from "node:fs/promises"; +import { IMAGE_REGISTRY } from "../const"; +import { + BuildController, + buildImageName, +} from "./build-controller"; +import { + KubernetesBuildOperations, + KubernetesBuildStore, + buildObjectApi, +} from "./build-kubernetes"; +import { FilesystemCas } from "./cas"; +import { KubernetesAuthStore } from "./kubernetes-store"; +import { + createKubernetesClients, + createKubernetesLeaseObjects, + createKubernetesManagementDependencies, + KubernetesAuditPersistence, + KubernetesOperationPersistence, + KubernetesWorkspaceLeaseProvider, + KubernetesWorkspacePersistence, + KubernetesWorkspaceStore, +} from "./kubernetes-state"; +import { + execUpgradeMatch, + handleExecUpgrade, + WireExecSession, + type ExecConnection, +} from "./app"; +import { KubernetesExec } from "./kubernetes-exec"; +import { createExecService } from "./exec-service"; +import { createManagementService } from "./management"; +import { + PersistentOperationStore, + recoverStaleOperations, +} from "./operation-store"; +import { KubernetesLogs } from "./kubernetes-logs"; +import { createLogService } from "./log-service"; +import { resolveRegistryDigest, type RegistryCredentials } from "./registry"; + +const clients = createKubernetesClients(); +const store = new KubernetesAuthStore(clients.objects); +const workspaceStore = new KubernetesWorkspaceStore( + new KubernetesWorkspacePersistence(clients.objects), +); +const operationStore = new PersistentOperationStore( + new KubernetesOperationPersistence(clients.objects), +); +const auditStore = new RedactingAuditStore( + new KubernetesAuditPersistence(clients.objects), +); +const management = createManagementService( + createKubernetesManagementDependencies(clients), +); +const namespace = process.env.KUBER_SYSTEM_NAMESPACE?.trim() || "kuber-system"; +const dataRoot = process.env.KUBER_DATA_ROOT?.trim() || "/data"; +const registry = (process.env.KUBER_BUILD_REGISTRY?.trim() || IMAGE_REGISTRY).replace( + /\/+$/, + "", +); +const registryConfigPath = + process.env.KUBER_REGISTRY_CONFIG?.trim() || "/etc/kuber/registry/config.json"; +const registryResolveOrigin = process.env.KUBER_REGISTRY_RESOLVE_ORIGIN?.trim(); +const internalRegistryHost = process.env.KUBER_INTERNAL_REGISTRY_HOST?.trim(); +const internalRegistryInsecure = + process.env.KUBER_INTERNAL_REGISTRY_INSECURE?.trim() === "true"; +const pushImagePrefix = + process.env.KUBER_PUSH_IMAGE_PREFIX?.trim() || "kuber/"; +if (!process.env.KUBER_REGISTRY_SECRET?.trim()) { + console.warn( + "KUBER_REGISTRY_SECRET is unset; BuildKit will use anonymous registry access", + ); +} + +async function registryCredentials(): Promise { + let config: { auths?: Record }; + try { + config = JSON.parse(await readFile(registryConfigPath, "utf8")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + const host = registry.split("/")[0]!; + const entry = + config.auths?.[host] ?? + config.auths?.[`https://${host}`] ?? + config.auths?.[`https://${host}/v1/`]; + if (!entry?.auth) return; + const separator = Buffer.from(entry.auth, "base64").toString("utf8").indexOf(":"); + if (separator < 0) return; + const value = Buffer.from(entry.auth, "base64").toString("utf8"); + return { username: value.slice(0, separator), password: value.slice(separator + 1) }; +} + +const buildStore = new KubernetesBuildStore( + buildObjectApi(clients.objects), + namespace, + `${dataRoot}/uploads`, +); +const builds = new BuildController({ + cas: new FilesystemCas(`${dataRoot}/cas`), + store: buildStore, + kubernetes: new KubernetesBuildOperations(clients.batch, clients.core), + namespace, + workspaceRoot: `${dataRoot}/workspaces`, + workspaceClaimName: process.env.KUBER_BUILD_DATA_CLAIM?.trim() || "kuber-build-data", + cacheImage: (request) => + `${(internalRegistryHost ?? registry)}/kuber/cache-${request.project}-${request.service}`, + imageName: (request) => buildImageName(registry, request.project, request.service), + pushImage: internalRegistryHost + ? (request) => + `${internalRegistryHost}/${pushImagePrefix}${request.project}-${request.service}:latest` + : undefined, + pushRegistryInsecure: internalRegistryHost + ? internalRegistryInsecure + : undefined, + buildkitImage: process.env.KUBER_BUILDKIT_IMAGE?.trim() || undefined, + registrySecretName: process.env.KUBER_REGISTRY_SECRET?.trim() || undefined, + maxLogBytes: Number(process.env.KUBER_BUILD_LOG_BYTES ?? 512 * 1024), + resolveDigest: async (image) => + resolveRegistryDigest(image, { + credentials: await registryCredentials(), + origin: registryResolveOrigin, + insecure: registryResolveOrigin?.startsWith("http://"), + }), +}); +const logs = createLogService( + new KubernetesLogs(clients.config, clients.apps, clients.core), +); +const execService = createExecService(new KubernetesExec(clients.config)); +const bootstrapUsername = process.env.KUBER_BOOTSTRAP_USERNAME?.trim(); +const bootstrapPassword = process.env.KUBER_BOOTSTRAP_PASSWORD; + +if (bootstrapUsername && bootstrapPassword) { + const existing = await store.getUser(bootstrapUsername); + if (!existing) { + await store.putUser({ + username: bootstrapUsername, + passwordHash: await Bun.password.hash(bootstrapPassword, { + algorithm: "argon2id", + }), + roles: ["admin"], + }); + console.log(`Created bootstrap user ${bootstrapUsername}`); + } +} + +const leases = new KubernetesWorkspaceLeaseProvider( + createKubernetesLeaseObjects(clients.coordination), +); + +try { + const recovered = await recoverStaleOperations(operationStore); + if (recovered > 0) + console.log(`Marked ${recovered} stale operation(s) as failed`); +} catch (error) { + console.error("Startup operation recovery failed", error); +} + +const sessionCleanupIntervalMs = Number( + process.env.KUBER_SESSION_CLEANUP_MS ?? 10 * 60 * 1000, +); +const sessionCleanupTimer = setInterval(async () => { + try { + await cleanupExpiredSessions(store); + } catch (error) { + console.error("Expired session cleanup failed", error); + } +}, Number.isFinite(sessionCleanupIntervalMs) && sessionCleanupIntervalMs > 0 + ? sessionCleanupIntervalMs + : 10 * 60 * 1000); +sessionCleanupTimer.unref?.(); + +const app = createApp({ + store, + workspaceStore, + operationStore, + auditStore, + management, + builds, + logs, + execService, + leases, + resolveImage: async (project, service) => { + const image = buildImageName(registry, project, service); + const digest = await resolveRegistryDigest(image, { + credentials: await registryCredentials(), + origin: registryResolveOrigin, + insecure: registryResolveOrigin?.startsWith("http://"), + }); + return { image: image.replace(/:latest$/, ""), digest, reference: `${image.replace(/:latest$/, "")}@${digest}` }; + }, + allowedOrigins: ( + process.env.KUBER_ALLOWED_ORIGINS ?? "https://kuber.astrxl.dev" + ) + .split(",") + .map((origin) => origin.trim()) + .filter(Boolean), +}); + +type ExecConnectionData = { + connection: ExecConnection; +}; + +const server = Bun.serve({ + port: Number(process.env.PORT ?? 3000), + fetch(request, server) { + const url = new URL(request.url); + const workspaceId = execUpgradeMatch(url); + if (workspaceId && request.method === "GET") { + return handleExecUpgrade( + { + store, + workspaceStore, + auditStore, + }, + request, + workspaceId, + (upgradeRequest, connection) => + server.upgrade(upgradeRequest, { data: { connection } }), + ); + } + return app(request); + }, + websocket: { + open(ws) { + const { connection } = ws.data; + const session = new WireExecSession( + { + sendText: (data) => ws.sendText(data), + close: (code, reason) => ws.close(code, reason), + }, + execService, + connection, + ); + ws.data = { connection, session } as unknown as ExecConnectionData; + }, + message(ws, message) { + const state = ws.data as unknown as { + session?: WireExecSession; + }; + state.session?.receive( + typeof message === "string" ? message : new TextDecoder().decode(message), + ); + }, + close(ws) { + const state = ws.data as unknown as { + session?: WireExecSession; + }; + state.session?.close(); + }, + }, +}); + +console.log(`kuber server listening on ${server.url}`); diff --git a/server/kubernetes-exec.ts b/server/kubernetes-exec.ts new file mode 100644 index 0000000..1d6268c --- /dev/null +++ b/server/kubernetes-exec.ts @@ -0,0 +1,265 @@ +import { + AppsV1Api, + CoreV1Api, + Exec, + KubeConfig, + type V1Deployment, + type V1Status, +} from "@kubernetes/client-node"; +import { PassThrough } from "node:stream"; +import { + type ExecDeployment, + type ExecPod, + type ExecPodContainer, + type KubernetesExecBackend, + type KubernetesExecExit, + type KubernetesExecProcess, + type KubernetesExecRequest, +} from "./exec-service"; + +function selectorString( + labels: Readonly>, +): string { + return Object.entries(labels) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => `${key}=${value}`) + .join(","); +} + +function matchLabels( + deployment: V1Deployment, +): Record { + const matchExpressions = deployment.spec?.selector?.matchExpressions; + if (matchExpressions?.length) return {}; + return deployment.spec?.selector?.matchLabels ?? {}; +} + +async function ownerDeploymentUid( + apps: AppsV1Api, + namespace: string, + ownerReferences: Array<{ + kind?: string; + name?: string; + uid?: string; + }> | undefined, +): Promise { + if (!ownerReferences?.length) return ""; + for (const ref of ownerReferences) { + if (ref.kind === "ReplicaSet" && ref.name && ref.uid) { + try { + const rs = await apps.readNamespacedReplicaSet({ + namespace, + name: ref.name, + }); + const rsOwners = rs.metadata?.ownerReferences; + if (rsOwners?.length) { + for (const rsRef of rsOwners) { + if (rsRef.kind === "Deployment" && rsRef.uid) return rsRef.uid; + } + } + } catch { + // ReplicaSet not found or inaccessible; fall through to empty + } + } + } + return ""; +} + +async function* passthrough( + stream: PassThrough, + signal: AbortSignal, +): AsyncGenerator { + try { + for await (const chunk of stream) { + if (signal.aborted) return; + yield chunk instanceof Buffer + ? new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength) + : new Uint8Array(chunk); + } + } finally { + stream.destroy(); + } +} + +function exitCodeFromStatus(status: V1Status): number | undefined { + const messages: string[] = []; + if (status.message) messages.push(status.message); + for (const cause of status.details?.causes ?? []) { + if (cause.reason) messages.push(cause.reason); + if (cause.message) messages.push(cause.message); + } + for (const message of messages) { + const match = /exit code\s*(\d+)/i.exec(message); + if (match) { + const code = Number(match[1]); + if (Number.isSafeInteger(code) && code >= 0) return code; + } + } + return undefined; +} + +export class KubernetesExec implements KubernetesExecBackend { + private readonly apps: AppsV1Api; + private readonly core: CoreV1Api; + private readonly execClient: Exec; + + constructor(config: KubeConfig) { + this.apps = config.makeApiClient(AppsV1Api); + this.core = config.makeApiClient(CoreV1Api); + this.execClient = new Exec(config); + } + + async getDeployment( + namespace: string, + name: string, + _signal?: AbortSignal, + ): Promise { + try { + const deployment = await this.apps.readNamespacedDeployment({ + namespace, + name, + }); + const uid = deployment.metadata?.uid; + const labels = deployment.metadata?.labels ?? {}; + const selector = matchLabels(deployment); + const containers = + deployment.spec?.template?.spec?.containers + ?.map((container) => container.name ?? "") + .filter(Boolean) ?? []; + if (!uid || Object.keys(selector).length === 0) return undefined; + return { name, uid, labels, selector, containers }; + } catch { + return undefined; + } + } + + async listPods( + namespace: string, + selector: Readonly>, + _signal?: AbortSignal, + ): Promise { + const result = await this.core.listNamespacedPod({ + namespace, + labelSelector: selectorString(selector), + }); + const pods: ExecPod[] = []; + for (const pod of result.items) { + const name = pod.metadata?.name; + const uid = pod.metadata?.uid; + if (!name || !uid) continue; + const deploymentUid = await ownerDeploymentUid( + this.apps, + namespace, + pod.metadata?.ownerReferences, + ); + const containerStatuses = pod.status?.containerStatuses ?? []; + const containers: ExecPodContainer[] = ( + pod.spec?.containers ?? [] + ).map((container) => { + const status = containerStatuses.find( + (item) => item.name === container.name, + ); + return { + name: container.name ?? "", + running: status?.state?.running !== undefined, + ready: status?.ready, + }; + }); + pods.push({ + name, + uid, + deploymentUid, + phase: pod.status?.phase, + deletionTimestamp: pod.metadata?.deletionTimestamp + ? new Date( + pod.metadata.deletionTimestamp, + ).toISOString() + : undefined, + containers, + }); + } + return pods; + } + + async exec( + request: KubernetesExecRequest, + signal: AbortSignal, + ): Promise { + const stdout = new PassThrough(); + const stderr = new PassThrough(); + const stdin = new PassThrough(); + + let completed: (exit: KubernetesExecExit) => void = () => {}; + let failed: (error: unknown) => void = () => {}; + const finished = new Promise((resolve, reject) => { + completed = resolve; + failed = reject; + }); + + let close: () => void = () => {}; + const abort = () => { + close(); + stdout.destroy(); + stderr.destroy(); + stdin.destroy(); + }; + signal.addEventListener("abort", abort, { once: true }); + + try { + const ws = await this.execClient.exec( + request.namespace, + request.pod, + request.container, + [...request.command], + stdout, + stderr, + stdin, + request.tty, + (status) => { + completed({ + exitCode: exitCodeFromStatus(status) ?? 1, + reason: status.reason, + message: status.message, + }); + }, + ); + close = () => { + try { + ws.close(); + } catch { + // already closed + } + }; + if (signal.aborted) abort(); + } catch (error) { + signal.removeEventListener("abort", abort); + stdout.destroy(); + stderr.destroy(); + stdin.destroy(); + failed(error); + throw error; + } + + return { + stdout: passthrough(stdout, signal), + stderr: passthrough(stderr, signal), + writeStdin(data: Uint8Array) { + if (!stdin.destroyed) stdin.write(data); + }, + closeStdin() { + if (!stdin.destroyed) stdin.end(); + }, + resize(_columns: number, _rows: number) { + // TTY resize is delivered through the exec attach resize stream. + // @kubernetes/client-node does not expose a live resize API during an + // ongoing attach; the initial size is passed at exec start. + }, + wait(): Promise { + return finished; + }, + close() { + abort(); + }, + }; + } +} diff --git a/server/kubernetes-logs.ts b/server/kubernetes-logs.ts new file mode 100644 index 0000000..fbed4fe --- /dev/null +++ b/server/kubernetes-logs.ts @@ -0,0 +1,152 @@ +import { + AppsV1Api, + CoreV1Api, + KubeConfig, + Log, + type V1LabelSelector, +} from "@kubernetes/client-node"; +import { PassThrough } from "node:stream"; +import { + KubernetesLogError, + type ContainerLogRequest, + type KubernetesLogsBackend, +} from "./log-service"; + +function selector(labels: Readonly>): string { + return Object.entries(labels) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => `${key}=${value}`) + .join(","); +} + +function matchLabels(value: V1LabelSelector | undefined): Record { + if (value?.matchExpressions?.length) return {}; + return value?.matchLabels ?? {}; +} + +function statusCode(error: unknown): number | undefined { + if (!error || typeof error !== "object") return; + if ("code" in error && typeof error.code === "number") return error.code; + if ("statusCode" in error && typeof error.statusCode === "number") + return error.statusCode; +} + +function logError(error: unknown): KubernetesLogError { + const status = statusCode(error); + return new KubernetesLogError( + error instanceof Error ? error.message : "Kubernetes log request failed", + status === undefined || status === 408 || status === 429 || status >= 500, + ); +} + +export class KubernetesLogs implements KubernetesLogsBackend { + private readonly logger: Log; + + constructor( + config: KubeConfig, + private readonly apps = config.makeApiClient(AppsV1Api), + private readonly core = config.makeApiClient(CoreV1Api), + ) { + this.logger = new Log(config); + } + + async listDeployments( + namespace: string, + labels: Readonly>, + ) { + try { + const deployments = await this.apps.listNamespacedDeployment({ + namespace, + labelSelector: selector(labels), + }); + return deployments.items.flatMap((deployment) => { + const name = deployment.metadata?.name; + const labels = matchLabels(deployment.spec?.selector); + return name && Object.keys(labels).length ? [{ name, selector: labels }] : []; + }); + } catch (error) { + throw logError(error); + } + } + + async getService(namespace: string, name: string) { + try { + const service = await this.core.readNamespacedService({ namespace, name }); + return { name, selector: service.spec?.selector ?? {} }; + } catch (error) { + if (statusCode(error) === 404) return; + throw logError(error); + } + } + + async listPods( + namespace: string, + labels: Readonly>, + ) { + try { + const pods = await this.core.listNamespacedPod({ + namespace, + labelSelector: selector(labels), + }); + return pods.items.map((pod) => ({ + name: pod.metadata?.name ?? "", + uid: pod.metadata?.uid ?? "", + phase: pod.status?.phase, + containers: (pod.spec?.containers ?? []).map((container) => container.name), + })); + } catch (error) { + throw logError(error); + } + } + + async readContainerLogs(request: ContainerLogRequest): Promise { + try { + return await this.core.readNamespacedPodLog({ + namespace: request.namespace, + name: request.pod, + container: request.container, + tailLines: request.tailLines, + sinceSeconds: request.sinceSeconds, + timestamps: request.timestamps, + }); + } catch (error) { + throw logError(error); + } + } + + async *streamContainerLogs( + request: ContainerLogRequest, + signal: AbortSignal, + ): AsyncIterable { + const output = new PassThrough(); + let upstream: AbortController | undefined; + const abort = () => { + upstream?.abort(); + output.destroy(); + }; + signal.addEventListener("abort", abort, { once: true }); + try { + upstream = await this.logger.log( + request.namespace, + request.pod, + request.container, + output, + { + follow: true, + tailLines: request.tailLines, + sinceSeconds: request.sinceSeconds, + sinceTime: request.sinceTime, + timestamps: request.timestamps, + }, + ); + if (signal.aborted) abort(); + for await (const chunk of output) yield new Uint8Array(chunk); + } catch (error) { + if (!signal.aborted) throw logError(error); + } finally { + signal.removeEventListener("abort", abort); + upstream?.abort(); + output.destroy(); + } + } +} diff --git a/server/kubernetes-state.ts b/server/kubernetes-state.ts new file mode 100644 index 0000000..ec3c682 --- /dev/null +++ b/server/kubernetes-state.ts @@ -0,0 +1,1129 @@ +import { + AppsV1Api, + BatchV1Api, + CoordinationV1Api, + CoreV1Api, + KubeConfig, + KubernetesObjectApi, + PatchStrategy, + type KubernetesObject, + type V1Lease, + type V1LeaseSpec, + type V1PodTemplateSpec, + type V1ReplicaSet, +} from "@kubernetes/client-node"; +import { createHash } from "node:crypto"; +import { createKubernetesHttpLibrary } from "../lib/k8s-http"; +import { type AuditEvent, type AuditPersistence } from "./audit-store"; +import { + managementDependencies, + type ManagementDependencies, + type ResourceIdentity, +} from "./management"; +import type { RollbackCandidate } from "../lib/rollback"; +import { LABELS } from "../const"; +import type { WorkspaceAdoptionResult, WorkspaceAdoptionService } from "./app"; +import { WorkspaceAdoptionError } from "./app"; +import { + RESERVED_NAMESPACES, + WORKSPACE_PROJECT_LABEL, + WORKSPACE_UID_LABEL, +} from "./management"; +import { + OperationConflictError, + OperationNotFoundError, + OperationValidationError, + type Operation, + type OperationPersistence, + type WorkspaceLease, + type WorkspaceLeaseProvider, +} from "./operation-store"; +import { + PersistentWorkspaceStore, + WorkspaceConflictError, + WorkspaceNotFoundError, + type Workspace, + type WorkspacePersistence, + type WorkspaceRevision, + type WorkspaceStoreOptions, +} from "./workspace-store"; + +export const KUBER_STATE_NAMESPACE = "kuber-system"; +const FIELD_MANAGER = "kuber-server"; +const TYPE_LABEL = "kuber.astrxl.dev/type"; +const WORKSPACE_LABEL = "kuber.astrxl.dev/workspace"; +const MANAGED_SELECTOR = "app.kubernetes.io/managed-by=kuber"; +const ADOPTABLE_RESOURCES = [ + { apiVersion: "v1", kind: "PersistentVolumeClaim" }, + { apiVersion: "v1", kind: "Secret" }, + { apiVersion: "v1", kind: "ConfigMap" }, + { apiVersion: "v1", kind: "Service" }, + { apiVersion: "apps/v1", kind: "Deployment" }, + { apiVersion: "networking.k8s.io/v1", kind: "Ingress" }, + { apiVersion: "traefik.io/v1alpha1", kind: "IngressRoute" }, +] as const; + +type DataObject = KubernetesObject & { + data?: Record; + stringData?: Record; + type?: string; +}; + +function isNotFound(error: unknown): boolean { + return Boolean( + error && + typeof error === "object" && + (("code" in error && error.code === 404) || + ("statusCode" in error && error.statusCode === 404)), + ); +} + +function isConflict(error: unknown): boolean { + return Boolean( + error && + typeof error === "object" && + (("code" in error && error.code === 409) || + ("statusCode" in error && error.statusCode === 409)), + ); +} + +function digestName(prefix: string, value: string): string { + return `${prefix}-${createHash("sha256").update(value).digest("hex").slice(0, 48)}`; +} + +function encode(value: unknown): string { + return JSON.stringify(value); +} + +function decodeSecretData(value: string | undefined): string | undefined { + if (!value) return; + try { + return Buffer.from(value, "base64").toString("utf8"); + } catch { + return; + } +} + +function parsePayload(object: DataObject): T | undefined { + const payload = + object.kind === "Secret" + ? decodeSecretData(object.data?.payload) + : object.data?.payload; + if (!payload) return; + try { + return JSON.parse(payload) as T; + } catch { + return; + } +} + +export function createKubernetesConfig(): KubeConfig { + const config = new KubeConfig(); + if (process.env.KUBERNETES_SERVICE_HOST) config.loadFromCluster(); + else config.loadFromDefault(); + const makeApiClient = config.makeApiClient.bind(config); + const httpLibrary = createKubernetesHttpLibrary(); + config.makeApiClient = ((apiClientType) => { + const client = makeApiClient(apiClientType) as unknown as { + api?: { configuration?: { httpApi?: typeof httpLibrary } }; + configuration?: { httpApi?: typeof httpLibrary }; + }; + if (client.api?.configuration) + client.api.configuration.httpApi = httpLibrary; + if (client.configuration) client.configuration.httpApi = httpLibrary; + return client; + }) as typeof config.makeApiClient; + return config; +} + +export function createKubernetesClients(): { + config: KubeConfig; + objects: KubernetesObjectApi; + apps: AppsV1Api; + batch: BatchV1Api; + core: CoreV1Api; + coordination: CoordinationV1Api; +} { + const config = createKubernetesConfig(); + return { + config, + objects: KubernetesObjectApi.makeApiClient(config), + apps: config.makeApiClient(AppsV1Api), + batch: config.makeApiClient(BatchV1Api), + core: config.makeApiClient(CoreV1Api), + coordination: config.makeApiClient(CoordinationV1Api), + }; +} + +/** + * Minimal adapter over the coordination.k8s.io/v1 Lease API used by the lease + * provider. Kept small and fakeable so the provider can be tested without a + * cluster. + */ +export interface LeaseObjects { + create(value: V1Lease): Promise; + read(name: string, namespace: string): Promise; + replace(value: V1Lease): Promise; + delete(name: string, namespace: string): Promise; +} + +/** Wraps a CoordinationV1Api client in the LeaseObjects adapter. */ +export function createKubernetesLeaseObjects( + coordination: CoordinationV1Api, +): LeaseObjects { + return { + async create(value) { + return coordination.createNamespacedLease({ + namespace: value.metadata?.namespace ?? KUBER_STATE_NAMESPACE, + body: value, + }); + }, + async read(name, namespace) { + try { + return await coordination.readNamespacedLease({ name, namespace }); + } catch (error) { + if (isNotFound(error)) return; + throw error; + } + }, + replace(value) { + return coordination.replaceNamespacedLease({ + name: value.metadata?.name ?? "", + namespace: value.metadata?.namespace ?? KUBER_STATE_NAMESPACE, + body: value, + }); + }, + async delete(name, namespace) { + try { + await coordination.deleteNamespacedLease({ name, namespace }); + } catch (error) { + if (isNotFound(error)) return; + throw error; + } + }, + }; +} + +const KUBER_LEASE_API_VERSION = "coordination.k8s.io/v1"; +const DEFAULT_LEASE_TTL_MS = 30_000; +const MAX_LEASE_ACQUIRE_RETRIES = 5; + +/** + * Kubernetes coordinates.k8s.io/v1 Lease acquireTime/renewTime are + * metav1.MicroTime, which expect six fractional-second digits (for example + * "2026-09-03T00:23:00.205000Z"). JavaScript Date#toISOString only emits three + * digits (milliseconds), and kubernetes-client-node does not re-format + * V1MicroTime when serializing, so the API server rejects the unpadded value. + */ +function microTimeString(ms: number): string { + const iso = new Date(ms).toISOString(); + const match = + /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d+))?Z$/.exec(iso); + if (!match) return iso; + const fraction = (match[2] ?? "").padEnd(6, "0"); + return `${match[1]}.${fraction}Z`; +} + +function leaseExpired(lease: V1Lease, nowMs: number): boolean { + const renewTime = lease.spec?.renewTime + ? Date.parse(String(lease.spec.renewTime)) + : Number.NaN; + const durationMs = (lease.spec?.leaseDurationSeconds ?? 0) * 1000; + if (!Number.isFinite(renewTime)) return true; + return renewTime + durationMs <= nowMs; +} + +/** + * Production WorkspaceLeaseProvider backed by coordination.k8s.io/v1 Lease + * objects. Acquisition, renewal, and release are all optimistic: every mutation + * carries the expected resourceVersion so the API server rejects lost-update + * races. An expired lease may be taken over by any holder (expiry takeover). + */ +export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider { + private readonly now: () => number; + + constructor( + private readonly objects: LeaseObjects, + private readonly namespace = KUBER_STATE_NAMESPACE, + private readonly clock: () => number = Date.now, + ) { + this.now = clock; + } + + private leaseName(workspaceId: string): string { + return digestName("lease", workspaceId); + } + + private leaseSpec(workspaceId: string, holder: string, ttlMs: number): V1Lease { + return { + apiVersion: KUBER_LEASE_API_VERSION, + kind: "Lease", + metadata: { + name: this.leaseName(workspaceId), + namespace: this.namespace, + }, + spec: { + holderIdentity: holder, + leaseDurationSeconds: Math.max(1, Math.round(ttlMs / 1000)), + acquireTime: microTimeString( + this.now(), + ) as unknown as V1LeaseSpec["acquireTime"], + renewTime: microTimeString(this.now()) as unknown as V1LeaseSpec["renewTime"], + leaseTransitions: 0, + }, + }; + } + + async acquire( + workspaceId: string, + holder: string, + ttlMs = DEFAULT_LEASE_TTL_MS, + ): Promise { + if (!workspaceId || !holder || !Number.isFinite(ttlMs) || ttlMs <= 0) + throw new OperationValidationError("Invalid workspace lease request"); + const name = this.leaseName(workspaceId); + + for (let attempt = 0; attempt <= MAX_LEASE_ACQUIRE_RETRIES; attempt++) { + const nowMs = this.now(); + + const lease = await this.objects.read(name, this.namespace); + + if (lease && !leaseExpired(lease, nowMs)) return; + + try { + if (!lease) { + const created = await this.objects.create( + this.leaseSpec(workspaceId, holder, ttlMs), + ); + return this.wrap(created, workspaceId, holder); + } + const next: V1Lease = { + ...this.leaseSpec(workspaceId, holder, ttlMs), + metadata: { + ...this.leaseSpec(workspaceId, holder, ttlMs).metadata, + resourceVersion: lease.metadata?.resourceVersion, + }, + spec: { + ...this.leaseSpec(workspaceId, holder, ttlMs).spec, + leaseTransitions: (lease.spec?.leaseTransitions ?? 0) + 1, + }, + }; + const replaced = await this.objects.replace(next); + return this.wrap(replaced, workspaceId, holder); + } catch (error) { + if (isConflict(error)) { + const raced = await this.objects.read(name, this.namespace); + if (raced && !leaseExpired(raced, this.now())) return; + continue; + } + throw error; + } + } + + return undefined; + } + + private wrap( + lease: V1Lease, + workspaceId: string, + holder: string, + ): WorkspaceLease { + const leaseDurationMs = + (lease.spec?.leaseDurationSeconds ?? 30) * 1000; + const expiresAt = ( + Date.parse(String(lease.spec?.renewTime)) + leaseDurationMs + ).toString(); + + const renew = async (ttlMs = 30_000): Promise => { + if (!Number.isFinite(ttlMs) || ttlMs <= 0) return false; + const name = lease.metadata?.name; + if (!name) return false; + const current = await this.objects.read(name, this.namespace); + if (!current || leaseExpired(current, this.now())) return false; + if (current.spec?.holderIdentity !== holder) return false; + const next: V1Lease = { + ...current, + metadata: { + ...current.metadata, + resourceVersion: current.metadata?.resourceVersion, + }, + spec: { + ...current.spec, + holderIdentity: holder, + leaseDurationSeconds: Math.max(1, Math.round(ttlMs / 1000)), + renewTime: microTimeString( + this.now(), + ) as unknown as V1LeaseSpec["renewTime"], + }, + }; + try { + await this.objects.replace(next); + return true; + } catch (error) { + if (isConflict(error)) return false; + throw error; + } + }; + + const release = async (): Promise => { + const name = lease.metadata?.name; + if (!name) return; + const current = await this.objects.read(name, this.namespace); + if (current?.spec?.holderIdentity !== holder) return; + await this.objects.delete(name, this.namespace); + }; + + return { workspaceId, holder, expiresAt, renew, release }; + } +} + +async function read( + objects: KubernetesObjectApi, + kind: "Secret" | "ConfigMap", + name: string, +): Promise { + try { + return (await objects.read({ + apiVersion: "v1", + kind, + metadata: { name, namespace: KUBER_STATE_NAMESPACE }, + })) as DataObject; + } catch (error) { + if (isNotFound(error)) return; + throw error; + } +} + +async function list( + objects: KubernetesObjectApi, + kind: "Secret" | "ConfigMap", + type: string, + workspaceId?: string, +): Promise { + const selector = [ + `${TYPE_LABEL}=${type}`, + workspaceId && `${WORKSPACE_LABEL}=${workspaceId}`, + ] + .filter(Boolean) + .join(","); + const response = await objects.list( + "v1", + kind, + KUBER_STATE_NAMESPACE, + undefined, + undefined, + undefined, + undefined, + selector, + ); + return response.items.map((item) => ({ + apiVersion: "v1", + kind, + ...item, + })) as DataObject[]; +} + +function stateObject( + kind: "Secret" | "ConfigMap", + name: string, + type: string, + value: unknown, + workspaceId?: string, + resourceVersion?: string, +): DataObject { + const metadata = { + name, + namespace: KUBER_STATE_NAMESPACE, + labels: { + [TYPE_LABEL]: type, + ...(workspaceId && { [WORKSPACE_LABEL]: workspaceId }), + }, + ...(resourceVersion && { resourceVersion }), + }; + return kind === "Secret" + ? { + apiVersion: "v1", + kind, + metadata, + type: "Opaque", + stringData: { payload: encode(value) }, + } + : { apiVersion: "v1", kind, metadata, data: { payload: encode(value) } }; +} + +async function createObject( + objects: KubernetesObjectApi, + value: DataObject, +): Promise { + await objects.create(value); +} + +async function deleteObject( + objects: KubernetesObjectApi, + kind: "Secret" | "ConfigMap", + name: string, +): Promise { + try { + await objects.delete({ + apiVersion: "v1", + kind, + metadata: { name, namespace: KUBER_STATE_NAMESPACE }, + }); + return true; + } catch (error) { + if (isNotFound(error)) return false; + throw error; + } +} + +export class KubernetesWorkspacePersistence implements WorkspacePersistence { + constructor(private readonly objects = createKubernetesClients().objects) {} + + private workspaceName(id: string): string { + return digestName("workspace", id); + } + + private revisionName(id: string, revision: number): string { + return digestName("revision", `${id}\0${revision}`); + } + + async get(id: string): Promise { + const object = await read(this.objects, "Secret", this.workspaceName(id)); + const workspace = object && parsePayload(object); + return workspace?.metadata.name === id ? workspace : undefined; + } + + async list(): Promise { + return (await list(this.objects, "Secret", "workspace")) + .map((item) => parsePayload(item)) + .filter((item): item is Workspace => item?.kind === "Workspace") + .sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)); + } + + async create( + workspace: Workspace, + revision: WorkspaceRevision, + ): Promise { + const revisionName = this.revisionName( + workspace.metadata.name, + revision.spec.revision, + ); + let revisionCreated = false; + try { + await createObject( + this.objects, + stateObject( + "Secret", + revisionName, + "workspace-revision", + revision, + workspace.metadata.name, + ), + ); + revisionCreated = true; + await createObject( + this.objects, + stateObject( + "Secret", + this.workspaceName(workspace.metadata.name), + "workspace", + workspace, + workspace.metadata.name, + ), + ); + } catch (error) { + if (revisionCreated) + await deleteObject(this.objects, "Secret", revisionName).catch( + () => false, + ); + if (isConflict(error)) + throw new WorkspaceConflictError("Workspace already exists"); + throw error; + } + } + + async replace( + workspace: Workspace, + revision: WorkspaceRevision, + expectedResourceVersion: string, + ): Promise { + const name = this.workspaceName(workspace.metadata.name); + const currentObject = await read(this.objects, "Secret", name); + const current = currentObject && parsePayload(currentObject); + if (!current || !currentObject?.metadata?.resourceVersion) + throw new WorkspaceNotFoundError("Workspace not found"); + if (current.metadata.resourceVersion !== expectedResourceVersion) + throw new WorkspaceConflictError("Workspace was concurrently modified"); + + const revisionName = this.revisionName( + workspace.metadata.name, + revision.spec.revision, + ); + let revisionCreated = false; + try { + try { + await createObject( + this.objects, + stateObject( + "Secret", + revisionName, + "workspace-revision", + revision, + workspace.metadata.name, + ), + ); + revisionCreated = true; + } catch (error) { + if (!isConflict(error)) throw error; + const existingObject = await read(this.objects, "Secret", revisionName); + const existing = + existingObject && parsePayload(existingObject); + const matches = + existing?.kind === "WorkspaceRevision" && + existing.metadata.name === revision.metadata.name && + existing.metadata.workspaceUid === revision.metadata.workspaceUid && + existing.metadata.resourceVersion === + revision.metadata.resourceVersion && + JSON.stringify(existing.spec) === JSON.stringify(revision.spec); + if (!matches) + throw new WorkspaceConflictError("Workspace revision already exists"); + } + await this.objects.replace( + stateObject( + "Secret", + name, + "workspace", + workspace, + workspace.metadata.name, + currentObject.metadata.resourceVersion, + ), + ); + } catch (error) { + if (revisionCreated) + await deleteObject(this.objects, "Secret", revisionName).catch( + () => false, + ); + if (isConflict(error)) + throw new WorkspaceConflictError("Workspace was concurrently modified"); + throw error; + } + } + + async getRevision( + id: string, + revision: number, + ): Promise { + const object = await read( + this.objects, + "Secret", + this.revisionName(id, revision), + ); + const value = object && parsePayload(object); + return value?.spec.workspaceId === id && value.spec.revision === revision + ? value + : undefined; + } + + async listRevisions(id: string): Promise { + return (await list(this.objects, "Secret", "workspace-revision", id)) + .map((item) => parsePayload(item)) + .filter( + (item): item is WorkspaceRevision => + item?.kind === "WorkspaceRevision" && item.spec.workspaceId === id, + ) + .sort((a, b) => a.spec.revision - b.spec.revision); + } + + async delete(id: string): Promise { + const deleted = await deleteObject( + this.objects, + "Secret", + this.workspaceName(id), + ); + for (const revision of await list( + this.objects, + "Secret", + "workspace-revision", + id, + )) { + if (revision.metadata?.name) + await deleteObject(this.objects, "Secret", revision.metadata.name); + } + return deleted; + } + + adopt(workspaceId: string, workspaceUid: string) { + return new KubernetesWorkspaceAdoptionService(this.objects).adopt( + workspaceId, + workspaceUid, + ); + } + + adoptPlatform(workspaceUid: string) { + return new KubernetesWorkspaceAdoptionService(this.objects).adoptPlatform( + workspaceUid, + ); + } +} + +export class KubernetesWorkspaceStore extends PersistentWorkspaceStore { + constructor( + private readonly kubernetesPersistence = new KubernetesWorkspacePersistence(), + options: WorkspaceStoreOptions = {}, + ) { + super(kubernetesPersistence, options); + } + + delete(id: string): Promise { + return this.kubernetesPersistence.delete(id); + } + + adopt(workspaceId: string, workspaceUid: string) { + return this.kubernetesPersistence.adopt(workspaceId, workspaceUid); + } + + adoptPlatform(workspaceUid: string) { + return this.kubernetesPersistence.adoptPlatform(workspaceUid); + } +} + +export class KubernetesWorkspaceAdoptionService implements WorkspaceAdoptionService { + constructor(private readonly objects = createKubernetesClients().objects) {} + + private async adoptNamespace( + workspaceId: string, + workspaceUid: string, + platform: boolean, + ): Promise { + if (!workspaceUid.trim()) + throw new WorkspaceAdoptionError("Workspace UID is required"); + if ( + platform + ? workspaceId !== KUBER_STATE_NAMESPACE + : RESERVED_NAMESPACES.has(workspaceId) || + workspaceId.startsWith("kube-") + ) { + throw new WorkspaceAdoptionError(`Namespace ${workspaceId} is reserved`); + } + + let namespace: KubernetesObject; + try { + namespace = await this.objects.read({ + apiVersion: "v1", + kind: "Namespace", + metadata: { name: workspaceId }, + }); + } catch (error) { + if (isNotFound(error)) { + return { workspaceId, workspaceUid, resourcesAdopted: 0 }; + } + throw error; + } + if ( + namespace.metadata?.labels?.["app.kubernetes.io/managed-by"] !== + LABELS["app.kubernetes.io/managed-by"] + ) { + throw new WorkspaceAdoptionError( + `Namespace ${workspaceId} is not managed by kuber; refusing adoption`, + ); + } + const namespaceOwner = namespace.metadata.labels?.[WORKSPACE_UID_LABEL]; + if (namespaceOwner && namespaceOwner !== workspaceUid) { + throw new WorkspaceAdoptionError( + `Namespace ${workspaceId} belongs to another workspace`, + ); + } + + const resources: Array<{ + apiVersion: string; + kind: string; + item: KubernetesObject; + }> = []; + for (const { apiVersion, kind } of ADOPTABLE_RESOURCES) { + try { + const result = await this.objects.list( + apiVersion, + kind, + workspaceId, + undefined, + undefined, + undefined, + undefined, + MANAGED_SELECTOR, + ); + resources.push( + ...result.items.map((item) => ({ apiVersion, kind, item })), + ); + } catch (error) { + if (!isNotFound(error)) throw error; + } + } + for (const { kind, item } of resources) { + const owner = item.metadata?.labels?.[WORKSPACE_UID_LABEL]; + if (owner && owner !== workspaceUid) { + throw new WorkspaceAdoptionError( + `${kind}/${item.metadata?.name} belongs to another workspace`, + ); + } + } + + const labels = { + ...LABELS, + [WORKSPACE_PROJECT_LABEL]: workspaceId, + [WORKSPACE_UID_LABEL]: workspaceUid, + }; + await this.objects.patch( + { + apiVersion: "v1", + kind: "Namespace", + metadata: { name: workspaceId, labels }, + }, + undefined, + undefined, + FIELD_MANAGER, + false, + PatchStrategy.ServerSideApply, + ); + for (const { apiVersion, kind, item } of resources) { + await this.objects.patch( + { + apiVersion, + kind, + metadata: { + name: item.metadata?.name, + namespace: workspaceId, + labels, + }, + }, + undefined, + undefined, + FIELD_MANAGER, + false, + PatchStrategy.ServerSideApply, + ); + } + return { + workspaceId, + workspaceUid, + resourcesAdopted: resources.length, + }; + } + + adopt(workspaceId: string, workspaceUid: string) { + return this.adoptNamespace(workspaceId, workspaceUid, false); + } + + adoptPlatform(workspaceUid: string) { + return this.adoptNamespace(KUBER_STATE_NAMESPACE, workspaceUid, true); + } +} + +export class KubernetesOperationPersistence implements OperationPersistence { + constructor(private readonly objects = createKubernetesClients().objects) {} + + async createIdempotent(operation: Operation): Promise { + const operationName = digestName( + "operation", + `${operation.spec.workspaceId}\0${operation.spec.idempotencyKey}`, + ); + const deterministic: Operation = { + ...operation, + metadata: { ...operation.metadata, name: operationName }, + }; + try { + await createObject( + this.objects, + stateObject( + "Secret", + operationName, + "operation", + deterministic, + deterministic.spec.workspaceId, + ), + ); + return deterministic; + } catch (error) { + if (!isConflict(error)) throw error; + const existing = await this.get(operationName); + if (!existing) + throw new OperationConflictError( + "Idempotent operation could not be recovered", + ); + return existing; + } + } + + async get(id: string): Promise { + const object = await read(this.objects, "Secret", id); + const operation = object && parsePayload(object); + return operation?.kind === "Operation" ? operation : undefined; + } + + async list(workspaceId?: string): Promise { + return (await list(this.objects, "Secret", "operation", workspaceId)) + .map((item) => parsePayload(item)) + .filter((item): item is Operation => item?.kind === "Operation") + .sort((a, b) => + a.metadata.creationTimestamp.localeCompare( + b.metadata.creationTimestamp, + ), + ); + } + + async replace( + operation: Operation, + expectedResourceVersion: string, + ): Promise { + const currentObject = await read( + this.objects, + "Secret", + operation.metadata.name, + ); + const current = currentObject && parsePayload(currentObject); + if (!current || !currentObject?.metadata?.resourceVersion) + throw new OperationNotFoundError("Operation not found"); + if (current.metadata.resourceVersion !== expectedResourceVersion) + throw new OperationConflictError("Operation was concurrently modified"); + try { + await this.objects.replace( + stateObject( + "Secret", + operation.metadata.name, + "operation", + operation, + operation.spec.workspaceId, + currentObject.metadata.resourceVersion, + ), + ); + } catch (error) { + if (isConflict(error)) + throw new OperationConflictError("Operation was concurrently modified"); + throw error; + } + } +} + +export class KubernetesAuditPersistence implements AuditPersistence { + constructor(private readonly objects = createKubernetesClients().objects) {} + + async append(event: AuditEvent): Promise { + await createObject( + this.objects, + stateObject( + "ConfigMap", + event.metadata.name, + "audit", + event, + event.spec.workspaceId, + ), + ); + } + + async list(workspaceId?: string): Promise { + return (await list(this.objects, "ConfigMap", "audit", workspaceId)) + .map((item) => parsePayload(item)) + .filter((item): item is AuditEvent => item?.kind === "AuditEvent") + .sort((a, b) => + a.metadata.creationTimestamp.localeCompare( + b.metadata.creationTimestamp, + ), + ); + } +} + +function resource(identity: ResourceIdentity): KubernetesObject { + return { + apiVersion: identity.apiVersion, + kind: identity.kind, + metadata: { + name: identity.name, + namespace: identity.namespace, + uid: identity.uid, + }, + }; +} + +export function createKubernetesManagementDependencies( + clients = createKubernetesClients(), +): ManagementDependencies { + const { objects, apps } = clients; + const revision = (replicaSet: V1ReplicaSet): number | undefined => { + const value = Number( + replicaSet.metadata?.annotations?.["deployment.kubernetes.io/revision"], + ); + return Number.isSafeInteger(value) && value > 0 ? value : undefined; + }; + const stripHash = (template: V1PodTemplateSpec): V1PodTemplateSpec => { + const labels = { ...template.metadata?.labels }; + delete labels["pod-template-hash"]; + return { + ...template, + metadata: { + ...template.metadata, + labels: Object.keys(labels).length ? labels : undefined, + }, + }; + }; + const replicaSets = async (project: string, deploymentUid?: string) => + ( + // Deployment-created ReplicaSets inherit only the pod-template labels + // (e.g. app, pod-template-hash), never the Deployment's metadata + // managed-by label, so a managed selector here excludes every revision + // and rollback reports "no previous release". List namespace-wide and + // restrict by ownerReference instead. + await apps.listNamespacedReplicaSet({ namespace: project }) + ).items.filter((item) => + item.metadata?.ownerReferences?.some( + (owner) => owner.kind === "Deployment" && owner.uid === deploymentUid, + ), + ); + const overrides: Partial = { + listDeployments: async (project) => + ( + await apps.listNamespacedDeployment({ + namespace: project, + labelSelector: MANAGED_SELECTOR, + }) + ).items, + scaleDeployment: async (project, name, replicas) => + objects.patch({ + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name, namespace: project }, + spec: { replicas }, + }), + restartDeployment: async (project, name) => + objects.patch({ + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name, namespace: project }, + spec: { + template: { + metadata: { + annotations: { + "kubectl.kubernetes.io/restartedAt": new Date().toISOString(), + }, + }, + }, + }, + }), + waitForDeployment: async (project, name, timeoutMs = 300_000) => { + const started = Date.now(); + while (Date.now() - started < timeoutMs) { + const deployment = await apps.readNamespacedDeployment({ + namespace: project, + name, + }); + const desired = deployment.spec?.replicas ?? 1; + if ( + (deployment.status?.observedGeneration ?? 0) >= + (deployment.metadata?.generation ?? 0) && + (deployment.status?.updatedReplicas ?? 0) === desired && + (deployment.status?.availableReplicas ?? 0) === desired && + (deployment.status?.unavailableReplicas ?? 0) === 0 + ) + return; + await Bun.sleep(2_000); + } + throw new Error(`Timed out waiting for deployment ${name} rollout`); + }, + planRollback: async (project, names) => { + const deployments = await overrides.listDeployments!(project); + const selected = names + ? deployments.filter((item) => + names.includes(item.metadata?.name ?? ""), + ) + : deployments; + if (names) { + const found = new Set(selected.map((item) => item.metadata?.name)); + for (const name of names) { + if (!found.has(name)) + throw new Error( + `No managed deployment named ${name} in ${project}`, + ); + } + } + const candidates: RollbackCandidate[] = []; + for (const deployment of selected) { + const name = deployment.metadata?.name; + if (!name) continue; + const revisions = (await replicaSets(project, deployment.metadata?.uid)) + .map((item) => ({ item, revision: revision(item) })) + .filter( + (entry): entry is { item: V1ReplicaSet; revision: number } => + entry.revision !== undefined, + ) + .sort((left, right) => right.revision - left.revision); + const current = revisions[0]; + const previous = + revisions + .slice(1) + .find( + (entry) => + JSON.stringify(stripHash(entry.item.spec?.template ?? {})) !== + JSON.stringify(stripHash(deployment.spec?.template ?? {})), + ) ?? revisions[1]; + const image = + previous?.item.spec?.template?.spec?.containers?.[0]?.image; + if (current && previous && image) { + candidates.push({ + name, + currentRevision: current.revision, + previousRevision: previous.revision, + image, + }); + } + } + return candidates; + }, + rollbackDeployment: async (project, candidate) => { + const deployment = await apps.readNamespacedDeployment({ + namespace: project, + name: candidate.name, + }); + const previous = ( + await replicaSets(project, deployment.metadata?.uid) + ).find((item) => revision(item) === candidate.previousRevision); + if (!previous?.spec?.template) + throw new Error( + `Deployment ${candidate.name} has no ReplicaSet for revision ${candidate.previousRevision}`, + ); + return apps.patchNamespacedDeployment({ + namespace: project, + name: candidate.name, + body: [ + { + op: "replace", + path: "/spec/template", + value: stripHash(previous.spec.template), + }, + ], + }); + }, + }; + return managementDependencies( + { + readNamespace: async (project) => { + try { + const namespace = await objects.read({ + apiVersion: "v1", + kind: "Namespace", + metadata: { name: project }, + }); + return { + uid: namespace.metadata?.uid, + labels: namespace.metadata?.labels, + }; + } catch (error) { + if (isNotFound(error)) return; + throw error; + } + }, + deleteResource: async (identity) => { + // KubernetesObjectApi turns metadata.uid into a delete precondition. + await objects.delete(resource(identity)); + }, + }, + overrides, + ); +} diff --git a/server/kubernetes-store.ts b/server/kubernetes-store.ts new file mode 100644 index 0000000..e6f38a0 --- /dev/null +++ b/server/kubernetes-store.ts @@ -0,0 +1,381 @@ +import { + KubeConfig, + KubernetesObjectApi, + PatchStrategy, + type KubernetesObject, +} from "@kubernetes/client-node"; +import { createHash } from "node:crypto"; +import { createKubernetesHttpLibrary } from "../lib/k8s-http"; +import { + normalizeSession, + normalizeUser, + type AuthStore, + type KuberUser, + type NewKuberUser, + type SessionInput, + type SessionRecord, + type UserUpdate, +} from "./auth"; +import { isRole } from "./authorization"; + +const FIELD_MANAGER = "kuber-server"; +export const KUBER_SYSTEM_NAMESPACE = "kuber-system"; + +type SecretObject = KubernetesObject & { + data?: Record; + type?: string; +}; + +function objectName(prefix: string, value: string): string { + const digest = createHash("sha256").update(value).digest("hex").slice(0, 48); + return `${prefix}-${digest}`; +} + +function decode(value: unknown): string | undefined { + if ( + typeof value !== "string" || + value.length === 0 || + value.length % 4 !== 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test( + value, + ) + ) { + return; + } + const decoded = Buffer.from(value, "base64"); + if (decoded.toString("base64") !== value) return; + try { + return new TextDecoder("utf-8", { fatal: true }).decode(decoded); + } catch { + return; + } +} + +function hasOnlyKeys( + data: Record, + keys: readonly string[], +): boolean { + const actual = Object.keys(data).sort(); + const expected = [...keys].sort(); + return ( + actual.length === expected.length && + actual.every((key, index) => key === expected[index]) + ); +} + +function parseRoles(value: unknown): KuberUser["roles"] | undefined { + const decoded = decode(value); + if (!decoded) return; + try { + const roles: unknown = JSON.parse(decoded); + if ( + !Array.isArray(roles) || + roles.length === 0 || + new Set(roles).size !== roles.length || + !roles.every(isRole) + ) + return; + return roles; + } catch { + return; + } +} + +function isSecret(secret: SecretObject, type: "user" | "session"): boolean { + return ( + secret.apiVersion === "v1" && + secret.kind === "Secret" && + secret.type === "Opaque" && + secret.metadata?.namespace === KUBER_SYSTEM_NAMESPACE && + secret.metadata.labels?.["kuber.astrxl.dev/type"] === type && + Boolean(secret.data) && + typeof secret.data === "object" && + !Array.isArray(secret.data) + ); +} + +function parseUser( + secret: SecretObject, + expectedUsername?: string, +): KuberUser | undefined { + if (!isSecret(secret, "user")) return; + const userKeys = + secret.data?.authVersion === undefined + ? ["username", "passwordHash", "roles", "disabled"] + : ["username", "passwordHash", "roles", "authVersion", "disabled"]; + if (!secret.data || !hasOnlyKeys(secret.data, userKeys)) return; + const username = decode(secret.data?.username); + const passwordHash = decode(secret.data?.passwordHash); + const roles = parseRoles(secret.data?.roles); + const disabled = decode(secret.data?.disabled); + const encodedAuthVersion = secret.data?.authVersion; + const authVersion = + encodedAuthVersion === undefined ? 1 : Number(decode(encodedAuthVersion)); + if ( + !username || + username !== username.trim() || + (expectedUsername !== undefined && username !== expectedUsername) || + secret.metadata?.name !== objectName("user", username) || + !passwordHash || + !roles || + (disabled !== "true" && disabled !== "false") || + !Number.isSafeInteger(authVersion) || + authVersion < 1 + ) + return; + return { + username, + passwordHash, + roles, + disabled: disabled === "true", + authVersion, + }; +} + +function parseSession(secret: SecretObject): SessionRecord | undefined { + if (!isSecret(secret, "session")) return; + const sessionKeys = + secret.data?.authVersion === undefined + ? ["tokenHash", "username", "roles", "expiresAt"] + : ["tokenHash", "username", "authVersion", "expiresAt"]; + if (!secret.data || !hasOnlyKeys(secret.data, sessionKeys)) return; + const tokenHash = decode(secret.data?.tokenHash); + const username = decode(secret.data?.username); + const expiresAt = decode(secret.data?.expiresAt); + const encodedAuthVersion = secret.data?.authVersion; + const authVersion = + encodedAuthVersion === undefined ? 1 : Number(decode(encodedAuthVersion)); + if ( + !tokenHash || + !username || + !expiresAt || + secret.metadata?.name !== objectName("session", tokenHash) || + (encodedAuthVersion === undefined && !parseRoles(secret.data?.roles)) + ) + return; + try { + return normalizeSession({ tokenHash, username, authVersion, expiresAt }); + } catch { + return; + } +} + +function isNotFound(error: unknown): boolean { + return Boolean( + error && typeof error === "object" && "code" in error && error.code === 404, + ); +} + +function createObjectApi(): KubernetesObjectApi { + const config = new KubeConfig(); + if (process.env.KUBERNETES_SERVICE_HOST) config.loadFromCluster(); + else config.loadFromDefault(); + + const makeApiClient = config.makeApiClient.bind(config); + const httpLibrary = createKubernetesHttpLibrary(); + config.makeApiClient = ((apiClientType) => { + const client = makeApiClient(apiClientType) as unknown as { + api?: { configuration?: { httpApi?: typeof httpLibrary } }; + configuration?: { httpApi?: typeof httpLibrary }; + }; + if (client.api?.configuration) + client.api.configuration.httpApi = httpLibrary; + if (client.configuration) client.configuration.httpApi = httpLibrary; + return client; + }) as typeof config.makeApiClient; + + return KubernetesObjectApi.makeApiClient(config); +} + +export class KubernetesAuthStore implements AuthStore { + constructor(private readonly objects = createObjectApi()) {} + + private async readSecret(name: string): Promise { + try { + return (await this.objects.read({ + apiVersion: "v1", + kind: "Secret", + metadata: { name, namespace: KUBER_SYSTEM_NAMESPACE }, + })) as SecretObject; + } catch (error) { + if (isNotFound(error)) return; + throw error; + } + } + + private async applySecret( + name: string, + type: "user" | "session", + stringData: Record, + ): Promise { + await this.objects.patch( + { + apiVersion: "v1", + kind: "Secret", + metadata: { + name, + namespace: KUBER_SYSTEM_NAMESPACE, + labels: { "kuber.astrxl.dev/type": type }, + }, + type: "Opaque", + stringData, + } as KubernetesObject, + undefined, + undefined, + FIELD_MANAGER, + true, + PatchStrategy.ServerSideApply, + ); + } + + private async listSecrets(type: "user" | "session"): Promise { + const result = await this.objects.list( + "v1", + "Secret", + KUBER_SYSTEM_NAMESPACE, + undefined, + undefined, + undefined, + undefined, + `kuber.astrxl.dev/type=${type}`, + ); + return result.items.map((item) => ({ + ...item, + apiVersion: item.apiVersion ?? "v1", + kind: item.kind ?? "Secret", + })) as SecretObject[]; + } + + private async deleteSecret(name: string): Promise { + try { + await this.objects.delete({ + apiVersion: "v1", + kind: "Secret", + metadata: { name, namespace: KUBER_SYSTEM_NAMESPACE }, + }); + return true; + } catch (error) { + if (isNotFound(error)) return false; + throw error; + } + } + + async getUser(username: string): Promise { + const secret = await this.readSecret(objectName("user", username)); + return secret ? parseUser(secret, username) : undefined; + } + + async listUsers(): Promise { + return (await this.listSecrets("user")) + .map((secret) => parseUser(secret)) + .filter((user): user is KuberUser => Boolean(user)) + .sort((a, b) => a.username.localeCompare(b.username)); + } + + async putUser(user: NewKuberUser | KuberUser): Promise { + const normalized = normalizeUser(user); + await this.applySecret(objectName("user", normalized.username), "user", { + username: normalized.username, + passwordHash: normalized.passwordHash, + roles: JSON.stringify(normalized.roles), + authVersion: String(normalized.authVersion), + disabled: String(Boolean(normalized.disabled)), + }); + } + + async createUser(user: NewKuberUser): Promise { + if (await this.getUser(user.username)) + throw new Error("User already exists"); + const normalized = normalizeUser(user); + await this.putUser(normalized); + return normalized; + } + + async updateUser( + username: string, + update: UserUpdate, + ): Promise { + const existing = await this.getUser(username); + if (!existing) return; + const updated = normalizeUser({ + ...existing, + ...update, + username, + authVersion: existing.authVersion + 1, + }); + await this.putUser(updated); + return updated; + } + + async deleteUser(username: string): Promise { + await this.revokeUserSessions(username); + return this.deleteSecret(objectName("user", username)); + } + + async getSession(tokenHash: string): Promise { + const secret = await this.readSecret(objectName("session", tokenHash)); + if (!secret) return; + const session = parseSession(secret); + if (!session || session.tokenHash !== tokenHash) return; + const user = await this.getUser(session.username); + if (!user || user.disabled || user.authVersion !== session.authVersion) + return; + return session; + } + + async putSession(session: SessionInput): Promise { + const user = await this.getUser(session.username); + if (!user || user.disabled) throw new Error("Session user is not active"); + const authVersion = + "authVersion" in session ? session.authVersion : user.authVersion; + if (authVersion !== user.authVersion) + throw new Error("Session auth version is stale"); + const normalized = normalizeSession({ + tokenHash: session.tokenHash, + username: session.username, + authVersion, + expiresAt: session.expiresAt, + }); + await this.applySecret( + objectName("session", normalized.tokenHash), + "session", + { + tokenHash: normalized.tokenHash, + username: normalized.username, + authVersion: String(normalized.authVersion), + expiresAt: normalized.expiresAt, + }, + ); + } + + async deleteSession(tokenHash: string): Promise { + await this.deleteSecret(objectName("session", tokenHash)); + } + + async revokeUserSessions(username: string): Promise { + const sessions = (await this.listSecrets("session")) + .map((secret) => parseSession(secret)) + .filter( + (session): session is SessionRecord => session?.username === username, + ); + for (const session of sessions) await this.deleteSession(session.tokenHash); + return sessions.length; + } + + async listExpiredSessions(now = Date.now()): Promise { + return (await this.listSecrets("session")) + .map((secret) => parseSession(secret)) + .filter( + (session): session is SessionRecord => + session !== undefined && Date.parse(session.expiresAt) <= now, + ); + } + + async deleteExpiredSessions(now = Date.now()): Promise { + const expired = await this.listExpiredSessions(now); + for (const session of expired) { + await this.deleteSession(session.tokenHash); + } + return expired.length; + } +} diff --git a/server/log-service.ts b/server/log-service.ts new file mode 100644 index 0000000..9c27748 --- /dev/null +++ b/server/log-service.ts @@ -0,0 +1,575 @@ +export const MANAGED_BY_SELECTOR = { + "app.kubernetes.io/managed-by": "kuber", +} as const; + +export type LogTarget = + | { kind: "managed-deployments" } + | { kind: "service"; name: string }; + +export type KubernetesDeployment = { + name: string; + selector: Readonly>; +}; + +export type KubernetesService = { + name: string; + selector: Readonly>; +}; + +export type KubernetesPod = { + name: string; + uid: string; + phase?: string; + containers: readonly string[]; +}; + +export type ContainerLogRequest = { + namespace: string; + pod: string; + container: string; + tailLines?: number; + sinceSeconds?: number; + sinceTime?: string; + timestamps: boolean; +}; + +/** The only cluster operations the log service can perform. */ +export interface KubernetesLogsBackend { + listDeployments( + namespace: string, + labels: Readonly>, + signal?: AbortSignal, + ): Promise; + getService( + namespace: string, + name: string, + signal?: AbortSignal, + ): Promise; + listPods( + namespace: string, + selector: Readonly>, + signal?: AbortSignal, + ): Promise; + readContainerLogs( + request: ContainerLogRequest, + signal?: AbortSignal, + ): Promise; + streamContainerLogs( + request: ContainerLogRequest, + signal: AbortSignal, + ): AsyncIterable; +} + +export type LogOptions = { + namespace: string; + target: LogTarget; + tailLines?: number; + sinceSeconds?: number; + sinceTime?: string | Date; + timestamps?: boolean; +}; + +export type FollowLogOptions = LogOptions & { + signal: AbortSignal; + queueCapacity?: number; + discoveryIntervalMs?: number; + heartbeatIntervalMs?: number; + retryIntervalMs?: number; +}; + +export type LogContainer = { + namespace: string; + targetKind: "deployment" | "service"; + targetName: string; + pod: string; + podUid: string; + container: string; +}; + +export type LogLineEvent = LogContainer & { + type: "log"; + timestamp: string; + logTimestamp?: string; + message: string; +}; + +export type LogHeartbeatEvent = { + type: "heartbeat"; + timestamp: string; +}; + +export type LogErrorEvent = { + type: "error"; + timestamp: string; + message: string; + retryable: boolean; + retryAfterMs?: number; + namespace: string; + pod?: string; + container?: string; +}; + +export type LogEvent = LogLineEvent | LogHeartbeatEvent | LogErrorEvent; + +export class KubernetesLogError extends Error { + constructor( + message: string, + readonly retryable: boolean, + ) { + super(message); + this.name = "KubernetesLogError"; + } +} + +type NormalizedOptions = Omit & { + sinceTime?: string; + timestamps: boolean; +}; + +type QueueWaiter = (value: T | undefined) => void; + +class BoundedAsyncQueue { + private readonly items: T[] = []; + private readonly readers: QueueWaiter[] = []; + private readonly writers: Array<() => void> = []; + private closed = false; + + constructor(private readonly capacity: number) {} + + async push(value: T): Promise { + while (!this.closed) { + const reader = this.readers.shift(); + if (reader) { + reader(value); + return true; + } + if (this.items.length < this.capacity) { + this.items.push(value); + return true; + } + await new Promise((resolve) => this.writers.push(resolve)); + } + return false; + } + + async shift(): Promise { + const item = this.items.shift(); + if (item !== undefined) { + this.writers.shift()?.(); + return item; + } + if (this.closed) return; + return new Promise((resolve) => this.readers.push(resolve)); + } + + close(discard = false): void { + if (this.closed) return; + this.closed = true; + if (discard) this.items.length = 0; + for (const reader of this.readers.splice(0)) reader(undefined); + for (const writer of this.writers.splice(0)) writer(); + } +} + +function positiveInteger(value: number | undefined, fallback: number): number { + const result = value ?? fallback; + if (!Number.isSafeInteger(result) || result <= 0) + throw new Error( + "Log service intervals and queue capacity must be positive integers", + ); + return result; +} + +function normalizeOptions(options: LogOptions): NormalizedOptions { + if (!options.namespace.trim()) throw new Error("Namespace is required"); + if (options.target.kind === "service" && !options.target.name.trim()) + throw new Error("Service name is required"); + if ( + options.tailLines !== undefined && + (!Number.isSafeInteger(options.tailLines) || options.tailLines < 0) + ) + throw new Error("tailLines must be a non-negative integer"); + if ( + options.sinceSeconds !== undefined && + (!Number.isSafeInteger(options.sinceSeconds) || options.sinceSeconds < 0) + ) + throw new Error("sinceSeconds must be a non-negative integer"); + if (options.sinceSeconds !== undefined && options.sinceTime !== undefined) + throw new Error("sinceSeconds and sinceTime are mutually exclusive"); + + let sinceTime: string | undefined; + if (options.sinceTime !== undefined) { + const date = + options.sinceTime instanceof Date + ? options.sinceTime + : new Date(options.sinceTime); + if (Number.isNaN(date.getTime())) + throw new Error("sinceTime must be valid"); + sinceTime = date.toISOString(); + } + + return { ...options, sinceTime, timestamps: options.timestamps ?? false }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error + ? error.message + : "Kubernetes log request failed"; +} + +function isAbort(signal: AbortSignal): boolean { + return signal.aborted; +} + +function isRetryable(error: unknown): boolean { + return !(error instanceof KubernetesLogError) || error.retryable; +} + +function requestFor( + source: LogContainer, + options: NormalizedOptions, +): ContainerLogRequest { + return { + namespace: source.namespace, + pod: source.pod, + container: source.container, + tailLines: options.tailLines, + sinceSeconds: options.sinceSeconds, + sinceTime: options.sinceTime, + timestamps: options.timestamps, + }; +} + +function parseLine( + line: string, + source: LogContainer, + timestamps: boolean, + now: () => Date, +): LogLineEvent { + let message = line.replace(/\r$/, ""); + let logTimestamp: string | undefined; + if (timestamps) { + const separator = message.indexOf(" "); + if (separator > 0) { + const candidate = message.slice(0, separator); + if (!Number.isNaN(Date.parse(candidate))) { + logTimestamp = candidate; + message = message.slice(separator + 1); + } + } + } + return { + type: "log", + timestamp: now().toISOString(), + ...(logTimestamp ? { logTimestamp } : {}), + ...source, + message, + }; +} + +async function emitText( + text: string, + source: LogContainer, + timestamps: boolean, + now: () => Date, + emit: (event: LogLineEvent) => void | Promise, +): Promise { + if (!text) return; + const lines = text.split(/\n/); + if (text.endsWith("\n")) lines.pop(); + for (const line of lines) + await emit(parseLine(line, source, timestamps, now)); +} + +async function emitChunks( + chunks: AsyncIterable, + source: LogContainer, + timestamps: boolean, + now: () => Date, + signal: AbortSignal, + emit: (event: LogLineEvent) => Promise, +): Promise { + const decoder = new TextDecoder(); + let buffer = ""; + for await (const chunk of chunks) { + buffer += + typeof chunk === "string" + ? chunk + : decoder.decode(chunk, { stream: true }); + let newline = buffer.indexOf("\n"); + while (newline >= 0) { + await emit(parseLine(buffer.slice(0, newline), source, timestamps, now)); + buffer = buffer.slice(newline + 1); + newline = buffer.indexOf("\n"); + } + } + buffer += decoder.decode(); + if (buffer && !signal.aborted) + await emit(parseLine(buffer, source, timestamps, now)); +} + +function wait(ms: number, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.resolve(); + return new Promise((resolve) => { + const timer = setTimeout(done, ms); + function done() { + clearTimeout(timer); + signal.removeEventListener("abort", done); + resolve(); + } + signal.addEventListener("abort", done, { once: true }); + }); +} + +export function encodeLogEvent(event: LogEvent): string { + return JSON.stringify(event); +} + +export class LogService { + constructor( + private readonly backend: KubernetesLogsBackend, + private readonly now: () => Date = () => new Date(), + ) {} + + async listContainers( + input: LogOptions, + signal?: AbortSignal, + ): Promise { + const options = normalizeOptions(input); + const targets: Array<{ + kind: "deployment" | "service"; + name: string; + selector: Readonly>; + }> = []; + if (options.target.kind === "managed-deployments") { + const deployments = await this.backend.listDeployments( + options.namespace, + MANAGED_BY_SELECTOR, + signal, + ); + for (const deployment of deployments) { + if (Object.keys(deployment.selector).length > 0) + targets.push({ kind: "deployment", ...deployment }); + } + } else { + const service = await this.backend.getService( + options.namespace, + options.target.name, + signal, + ); + if (!service) + throw new KubernetesLogError( + `Service ${options.target.name} was not found`, + false, + ); + if (Object.keys(service.selector).length === 0) + throw new KubernetesLogError( + `Service ${options.target.name} has no pod selector`, + false, + ); + targets.push({ kind: "service", ...service }); + } + + const result = new Map(); + for (const target of targets) { + const pods = await this.backend.listPods( + options.namespace, + target.selector, + signal, + ); + for (const pod of pods) { + if (!pod.name || !pod.uid) continue; + for (const container of pod.containers) { + if (!container) continue; + const source: LogContainer = { + namespace: options.namespace, + targetKind: target.kind, + targetName: target.name, + pod: pod.name, + podUid: pod.uid, + container, + }; + result.set(`${pod.uid}\0${container}`, source); + } + } + } + return [...result.values()]; + } + + async collect(input: LogOptions, signal?: AbortSignal): Promise { + const options = normalizeOptions(input); + const sources = await this.listContainers(options, signal); + const events: LogEvent[] = []; + for (const source of sources) { + try { + const text = await this.backend.readContainerLogs( + requestFor(source, options), + signal, + ); + await emitText(text, source, options.timestamps, this.now, (event) => { + events.push(event); + }); + } catch (error) { + if (signal && isAbort(signal)) throw error; + events.push({ + type: "error", + timestamp: this.now().toISOString(), + namespace: options.namespace, + pod: source.pod, + container: source.container, + message: errorMessage(error), + retryable: isRetryable(error), + }); + } + } + return events; + } + + async *follow(input: FollowLogOptions): AsyncGenerator { + const options = normalizeOptions(input); + const capacity = positiveInteger(input.queueCapacity, 128); + const discoveryInterval = positiveInteger(input.discoveryIntervalMs, 2_000); + const heartbeatInterval = positiveInteger( + input.heartbeatIntervalMs, + 15_000, + ); + const retryInterval = positiveInteger(input.retryIntervalMs, 1_000); + const queue = new BoundedAsyncQueue(capacity); + const controller = new AbortController(); + const stop = () => { + queue.close(true); + controller.abort(); + }; + if (input.signal.aborted) controller.abort(); + else input.signal.addEventListener("abort", stop, { once: true }); + + const active = new Map(); + const completed = new Set(); + const retryNotBefore = new Map(); + const streamTasks = new Set>(); + const pushError = async ( + error: unknown, + source?: LogContainer, + ): Promise => { + await queue.push({ + type: "error", + timestamp: this.now().toISOString(), + namespace: options.namespace, + pod: source?.pod, + container: source?.container, + message: errorMessage(error), + retryable: isRetryable(error), + ...(isRetryable(error) ? { retryAfterMs: retryInterval } : {}), + }); + }; + + const startStream = (source: LogContainer): void => { + const key = `${source.podUid}\0${source.container}`; + const local = new AbortController(); + active.set(key, local); + const signal = AbortSignal.any([controller.signal, local.signal]); + let task!: Promise; + task = (async () => { + try { + await emitChunks( + this.backend.streamContainerLogs( + requestFor(source, options), + signal, + ), + source, + options.timestamps, + this.now, + signal, + (event) => queue.push(event), + ); + if (!signal.aborted) completed.add(key); + } catch (error) { + if (!isAbort(signal)) { + await pushError(error, source); + if (isRetryable(error)) + retryNotBefore.set(key, Date.now() + retryInterval); + else completed.add(key); + } + } finally { + active.delete(key); + streamTasks.delete(task); + } + })(); + streamTasks.add(task); + }; + + const discover = async (): Promise => { + while (!controller.signal.aborted) { + try { + const sources = await this.listContainers(options, controller.signal); + const desired = new Set( + sources.map((source) => `${source.podUid}\0${source.container}`), + ); + for (const key of completed) { + if (!desired.has(key)) completed.delete(key); + } + for (const key of retryNotBefore.keys()) { + if (!desired.has(key)) retryNotBefore.delete(key); + } + for (const [key, stream] of active) { + if (!desired.has(key)) stream.abort(); + } + for (const source of sources) { + const key = `${source.podUid}\0${source.container}`; + if ( + !active.has(key) && + !completed.has(key) && + (retryNotBefore.get(key) ?? 0) <= Date.now() + ) { + retryNotBefore.delete(key); + startStream(source); + } + } + } catch (error) { + if (!isAbort(controller.signal)) { + await pushError(error); + if (!isRetryable(error)) controller.abort(); + } + } + await wait(discoveryInterval, controller.signal); + } + }; + + const heartbeat = async (): Promise => { + while (!controller.signal.aborted) { + await wait(heartbeatInterval, controller.signal); + if (!controller.signal.aborted) + await queue.push({ + type: "heartbeat", + timestamp: this.now().toISOString(), + }); + } + }; + + const runner = Promise.all([discover(), heartbeat()]).finally(async () => { + for (const stream of active.values()) stream.abort(); + await Promise.allSettled([...streamTasks]); + queue.close(); + }); + + try { + for (;;) { + const event = await queue.shift(); + if (event === undefined) return; + yield event; + } + } finally { + input.signal.removeEventListener("abort", stop); + controller.abort(); + queue.close(true); + await runner; + } + } +} + +export function createLogService( + backend: KubernetesLogsBackend, + now?: () => Date, +): LogService { + return new LogService(backend, now); +} diff --git a/server/management.ts b/server/management.ts new file mode 100644 index 0000000..333af9c --- /dev/null +++ b/server/management.ts @@ -0,0 +1,677 @@ +import type { KubernetesObject, V1Deployment } from "@kubernetes/client-node"; +import type { ComposeSpecification } from "../schema/docker.d"; +import { LABELS } from "../const"; +import { + applyResource, + getStaleResources, + listManagedResources, + sortResources, +} from "../lib/apply"; +import { + DATABASE_NAMESPACE, + getComposePostgresClaims, + getRoleCredentials, + listManagedDatabaseResources, + reconcilePostgresClaims, + type PostgresClaim, + type RoleCredentials, +} from "../lib/database"; +import { + buildNamespaceGraphs, + fetchNamespaceObjects, + type NamespaceGraph, +} from "../lib/graph"; +import { + planRollback, + rollbackDeployment, + type RollbackCandidate, +} from "../lib/rollback"; +import { + listManagedDeployments, + restartDeployment, + scaleDeployment, + waitForDeploymentRollout, +} from "../lib/shared"; +import { + getComposeS3Claims, + getS3Credentials, + listManagedStorageResources, + reconcileS3Claims, + STORAGE_NAMESPACE, + type S3Claim, +} from "../lib/storage"; + +export const WORKSPACE_UID_LABEL = "kuber.dev/workspace-uid"; +export const WORKSPACE_PROJECT_LABEL = "kuber.dev/project"; + +export const RESERVED_NAMESPACES = new Set([ + "default", + "kube-system", + "kube-public", + "kube-node-lease", + "kuber-system", + DATABASE_NAMESPACE, + STORAGE_NAMESPACE, +]); + +export type Workspace = { + project: string; + uid: string; +}; + +export type NamespaceRecord = { + uid?: string; + labels?: Record; +}; + +export type NamespaceSafety = { + project: string; + status: "missing" | "owned" | "external" | "different-workspace"; + namespaceUid?: string; +}; + +export type ResourceIdentity = { + apiVersion: string; + kind: string; + name: string; + namespace?: string; + uid: string; + workspaceUid: string; +}; + +export type ResourcePlan = { + desired: KubernetesObject[]; + stale: ResourceIdentity[]; +}; + +export type DownPlan = { + full: boolean; + retained: ResourceIdentity[]; + delete: ResourceIdentity[]; +}; + +export type CredentialMetadata = { + service: string; + provider: "postgres" | "s3"; + principal: string; + resource: string; + secretNamespace: string; + secretName?: string; +}; + +export type ManagementDependencies = { + readNamespace(project: string): Promise; + listDeployments(project: string): Promise; + scaleDeployment( + project: string, + name: string, + replicas: number, + ): Promise; + restartDeployment(project: string, name: string): Promise; + waitForDeployment( + project: string, + name: string, + timeoutMs?: number, + ): Promise; + fetchGraphObjects(project: string): Promise; + planRollback(project: string, names?: string[]): Promise; + rollbackDeployment( + project: string, + candidate: RollbackCandidate, + ): Promise; + listProjectResources(project: string): Promise; + listDatabaseResources(project: string): Promise; + listStorageResources(project: string): Promise; + findStaleResources( + project: string, + desired: KubernetesObject[], + ): Promise; + applyResource(resource: KubernetesObject): Promise; + deleteResource(identity: ResourceIdentity): Promise; + reconcileDatabases( + project: string, + compose: ComposeSpecification, + ): Promise>>; + getDatabaseCredentials(username: string): Promise; + reconcileStorage( + project: string, + compose: ComposeSpecification, + ): Promise>>; + getStorageCredentials(claim: S3Claim): Promise>; +}; + +export type ManagementService = ReturnType; + +const defaultOperations: Omit< + ManagementDependencies, + "readNamespace" | "deleteResource" +> = { + listDeployments: async () => listManagedDeployments(), + scaleDeployment: async (_project, name, replicas) => + scaleDeployment(name, replicas), + restartDeployment: async (_project, name) => restartDeployment(name), + waitForDeployment: async (_project, name, timeoutMs) => + waitForDeploymentRollout(name, timeoutMs), + fetchGraphObjects: fetchNamespaceObjects, + planRollback: async (_project, names) => planRollback(names), + rollbackDeployment: async (_project, candidate) => + rollbackDeployment(candidate), + listProjectResources: listManagedResources, + listDatabaseResources: listManagedDatabaseResources, + listStorageResources: listManagedStorageResources, + findStaleResources: getStaleResources, + applyResource, + reconcileDatabases: reconcilePostgresClaims, + getDatabaseCredentials: getRoleCredentials, + reconcileStorage: reconcileS3Claims, + getStorageCredentials: getS3Credentials, +}; + +export function managementDependencies( + infrastructure: Pick< + ManagementDependencies, + "readNamespace" | "deleteResource" + >, + overrides: Partial = {}, +): ManagementDependencies { + return { ...defaultOperations, ...infrastructure, ...overrides }; +} + +function validateWorkspace(workspace: Workspace): void { + if (!workspace.uid.trim()) throw new Error("Workspace UID is required"); + if ( + !workspace.project || + workspace.project.length > 63 || + !/^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/.test(workspace.project) + ) { + throw new Error(`Invalid project namespace ${workspace.project}`); + } + if (RESERVED_NAMESPACES.has(workspace.project)) { + throw new Error(`Namespace ${workspace.project} is reserved`); + } +} + +function resourceName(resource: KubernetesObject): string { + const name = resource.metadata?.name; + if (!resource.apiVersion || !resource.kind || !name) { + throw new Error("Resources require apiVersion, kind, and metadata.name"); + } + return name; +} + +function assertResourceOwnership( + workspace: Workspace, + resource: KubernetesObject, +): void { + const owner = resource.metadata?.labels?.[WORKSPACE_UID_LABEL]; + if (owner !== workspace.uid) { + throw new Error( + `${resource.kind}/${resourceName(resource)} is not owned by workspace ${workspace.uid}`, + ); + } +} + +function identity( + workspace: Workspace, + resource: KubernetesObject, +): ResourceIdentity { + assertResourceOwnership(workspace, resource); + const uid = resource.metadata?.uid; + if (!uid) { + throw new Error( + `${resource.kind}/${resourceName(resource)} has no UID deletion precondition`, + ); + } + return { + apiVersion: resource.apiVersion!, + kind: resource.kind!, + name: resourceName(resource), + namespace: resource.metadata?.namespace, + uid, + workspaceUid: workspace.uid, + }; +} + +function labelDesired( + workspace: Workspace, + resource: KubernetesObject, +): KubernetesObject { + const name = resourceName(resource); + const namespace = + resource.kind === "Namespace" + ? undefined + : (resource.metadata?.namespace ?? workspace.project); + if (resource.kind === "Namespace" && name !== workspace.project) { + throw new Error( + `Cannot manage namespace ${name} from project ${workspace.project}`, + ); + } + if (namespace && namespace !== workspace.project) { + throw new Error( + `Cannot manage ${resource.kind}/${name} in namespace ${namespace}`, + ); + } + const existingOwner = resource.metadata?.labels?.[WORKSPACE_UID_LABEL]; + if (existingOwner && existingOwner !== workspace.uid) { + throw new Error(`${resource.kind}/${name} belongs to another workspace`); + } + return { + ...resource, + metadata: { + ...resource.metadata, + name, + namespace, + labels: { + ...resource.metadata?.labels, + ...LABELS, + [WORKSPACE_PROJECT_LABEL]: workspace.project, + [WORKSPACE_UID_LABEL]: workspace.uid, + }, + }, + }; +} + +function labelExternal( + workspace: Workspace, + resource: KubernetesObject, + namespace: string, +): KubernetesObject { + const name = resourceName(resource); + if (resource.metadata?.namespace !== namespace) { + throw new Error( + `${resource.kind}/${name} is outside expected namespace ${namespace}`, + ); + } + const existingOwner = resource.metadata?.labels?.[WORKSPACE_UID_LABEL]; + if (existingOwner && existingOwner !== workspace.uid) { + throw new Error(`${resource.kind}/${name} belongs to another workspace`); + } + const { status: _status, ...body } = resource as KubernetesObject & { + status?: unknown; + }; + return { + ...body, + metadata: { + ...body.metadata, + labels: { + ...body.metadata?.labels, + [WORKSPACE_PROJECT_LABEL]: workspace.project, + [WORKSPACE_UID_LABEL]: workspace.uid, + }, + }, + }; +} + +function deploymentNames(deployments: V1Deployment[]): string[] { + return deployments + .map((deployment) => deployment.metadata?.name) + .filter((name): name is string => Boolean(name)); +} + +function selectTargets(all: string[], requested?: string[]): string[] { + if (!requested) return all; + const unique = [...new Set(requested)]; + const available = new Set(all); + for (const name of unique) { + if (!available.has(name)) + throw new Error(`No managed deployment named ${name}`); + } + return unique; +} + +export function createManagementService(dependencies: ManagementDependencies) { + async function namespaceSafety( + workspace: Workspace, + ): Promise { + validateWorkspace(workspace); + const namespace = await dependencies.readNamespace(workspace.project); + if (!namespace) return { project: workspace.project, status: "missing" }; + const managed = + namespace.labels?.["app.kubernetes.io/managed-by"] === + LABELS["app.kubernetes.io/managed-by"]; + if (!managed) { + return { + project: workspace.project, + status: "external", + namespaceUid: namespace.uid, + }; + } + if (namespace.labels?.[WORKSPACE_UID_LABEL] !== workspace.uid) { + return { + project: workspace.project, + status: "different-workspace", + namespaceUid: namespace.uid, + }; + } + return { + project: workspace.project, + status: "owned", + namespaceUid: namespace.uid, + }; + } + + async function assertSafe( + workspace: Workspace, + allowMissing = false, + ): Promise { + const safety = await namespaceSafety(workspace); + if ( + safety.status === "owned" || + (allowMissing && safety.status === "missing") + ) { + return safety; + } + throw new Error( + `Namespace ${workspace.project} is ${safety.status}; refusing workspace mutation`, + ); + } + + async function targets(workspace: Workspace, names?: string[]) { + await assertSafe(workspace); + return selectTargets( + deploymentNames(await dependencies.listDeployments(workspace.project)), + names, + ); + } + + async function ownExternalResources( + workspace: Workspace, + namespace: string, + resources: KubernetesObject[], + ): Promise { + for (const resource of resources) { + await dependencies.applyResource( + labelExternal(workspace, resource, namespace), + ); + } + } + + async function deleteResources( + workspace: Workspace, + resources: ResourceIdentity[], + ): Promise { + await assertSafe(workspace); + for (const resource of resources) { + if (!resource.uid || resource.workspaceUid !== workspace.uid) { + throw new Error( + `${resource.kind}/${resource.name} has an invalid workspace deletion identity`, + ); + } + } + for (const resource of resources) { + await dependencies.deleteResource(resource); + } + } + + async function planDown( + workspace: Workspace, + full = false, + ): Promise { + const safety = await assertSafe(workspace); + const projectResources = sortResources( + await dependencies.listProjectResources(workspace.project), + ); + const retainedResources = full + ? [] + : projectResources.filter( + (resource) => + resource.kind === "Ingress" || + resource.kind === "PersistentVolumeClaim", + ); + const deleteResources = full + ? [...projectResources] + : projectResources.filter( + (resource) => !retainedResources.includes(resource), + ); + + if (full) { + deleteResources.push( + ...(await dependencies.listDatabaseResources(workspace.project)), + ...(await dependencies.listStorageResources(workspace.project)), + ); + if (!safety.namespaceUid) { + throw new Error( + `Namespace ${workspace.project} has no UID deletion precondition`, + ); + } + deleteResources.push({ + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: workspace.project, + uid: safety.namespaceUid, + labels: { + ...LABELS, + [WORKSPACE_PROJECT_LABEL]: workspace.project, + [WORKSPACE_UID_LABEL]: workspace.uid, + }, + }, + }); + } + + return { + full, + retained: retainedResources.map((item) => identity(workspace, item)), + delete: sortResources(deleteResources) + .reverse() + .map((item) => identity(workspace, item)), + }; + } + + return { + namespaceSafety, + + async graphStatus( + workspace: Workspace, + options: { includeIdle?: boolean } = {}, + ): Promise { + await assertSafe(workspace); + return buildNamespaceGraphs( + await dependencies.fetchGraphObjects(workspace.project), + options, + ); + }, + + async stop(workspace: Workspace, names?: string[]): Promise { + const selected = await targets(workspace, names); + for (const name of selected) { + await dependencies.scaleDeployment(workspace.project, name, 0); + } + return selected; + }, + + async restart(workspace: Workspace, names?: string[]): Promise { + const selected = await targets(workspace, names); + for (const name of selected) { + await dependencies.restartDeployment(workspace.project, name); + } + return selected; + }, + + async rollback( + workspace: Workspace, + names?: string[], + timeoutMs?: number, + ): Promise { + await assertSafe(workspace); + const candidates = await dependencies.planRollback( + workspace.project, + names, + ); + for (const candidate of candidates) { + await dependencies.rollbackDeployment(workspace.project, candidate); + } + for (const candidate of candidates) { + await dependencies.waitForDeployment( + workspace.project, + candidate.name, + timeoutMs, + ); + } + return candidates; + }, + + databaseCredentialsMetadata( + compose: ComposeSpecification, + ): CredentialMetadata[] { + return getComposePostgresClaims(compose).map((claim: PostgresClaim) => ({ + service: claim.service, + provider: "postgres", + principal: claim.username, + resource: claim.database, + secretNamespace: DATABASE_NAMESPACE, + secretName: claim.secretName, + })); + }, + + async reconcileDatabases( + workspace: Workspace, + compose: ComposeSpecification, + ) { + await assertSafe(workspace, true); + const environment = await dependencies.reconcileDatabases( + workspace.project, + compose, + ); + await ownExternalResources( + workspace, + DATABASE_NAMESPACE, + await dependencies.listDatabaseResources(workspace.project), + ); + return environment; + }, + + async getDatabaseCredentials(workspace: Workspace, username: string) { + await assertSafe(workspace); + const databases = await dependencies.listDatabaseResources( + workspace.project, + ); + const database = databases.find( + (resource) => + resource.kind === "Database" && + (resource as KubernetesObject & { spec?: { owner?: string } }).spec + ?.owner === username, + ); + if (!database) { + throw new Error( + `Database role ${username} is not managed by this workspace`, + ); + } + assertResourceOwnership(workspace, database); + return dependencies.getDatabaseCredentials(username); + }, + + storageCredentialsMetadata( + compose: ComposeSpecification, + ): CredentialMetadata[] { + return getComposeS3Claims(compose).map((claim) => ({ + service: claim.service, + provider: "s3", + principal: claim.key, + resource: claim.bucket, + secretNamespace: STORAGE_NAMESPACE, + })); + }, + + async reconcileStorage( + workspace: Workspace, + compose: ComposeSpecification, + ) { + await assertSafe(workspace, true); + const environment = await dependencies.reconcileStorage( + workspace.project, + compose, + ); + await ownExternalResources( + workspace, + STORAGE_NAMESPACE, + await dependencies.listStorageResources(workspace.project), + ); + return environment; + }, + + async getStorageCredentials(workspace: Workspace, claim: S3Claim) { + await assertSafe(workspace); + const resources = await dependencies.listStorageResources( + workspace.project, + ); + const key = resources.find( + (resource) => + resource.kind === "GarageKey" && + resource.metadata?.name === claim.key, + ); + const bucket = resources.find( + (resource) => + resource.kind === "GarageBucket" && + resource.metadata?.name === claim.bucket, + ); + if (!key || !bucket) { + throw new Error( + `S3 claim ${claim.key}/${claim.bucket} is not managed by this workspace`, + ); + } + assertResourceOwnership(workspace, key); + assertResourceOwnership(workspace, bucket); + return dependencies.getStorageCredentials(claim); + }, + + async planResources( + workspace: Workspace, + desired: KubernetesObject[], + ): Promise { + await assertSafe(workspace, true); + const labeled = sortResources( + desired.map((item) => labelDesired(workspace, item)), + ); + const stale = await dependencies.findStaleResources( + workspace.project, + labeled, + ); + return { + desired: labeled, + stale: sortResources(stale) + .reverse() + .map((item) => identity(workspace, item)), + }; + }, + + async applyResources( + workspace: Workspace, + resources: KubernetesObject[], + ): Promise { + await assertSafe(workspace, true); + const applied: KubernetesObject[] = []; + for (const resource of sortResources( + resources.map((item) => labelDesired(workspace, item)), + )) { + applied.push(await dependencies.applyResource(resource)); + } + return applied; + }, + + async waitForResources( + workspace: Workspace, + deploymentTargets: string[], + timeoutMs?: number, + ): Promise { + const selected = await targets(workspace, deploymentTargets); + for (const name of selected) { + await dependencies.waitForDeployment( + workspace.project, + name, + timeoutMs, + ); + } + }, + + deleteResources, + + planDown, + + async down(workspace: Workspace, full = false): Promise { + const plan = await planDown(workspace, full); + await deleteResources(workspace, plan.delete); + return plan; + }, + }; +} diff --git a/server/materialize.ts b/server/materialize.ts new file mode 100644 index 0000000..f2ec638 --- /dev/null +++ b/server/materialize.ts @@ -0,0 +1,161 @@ +import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { + chmod, + lstat, + mkdir, + open, + rename, + rm, + symlink, +} from "node:fs/promises"; +import { + basename, + dirname, + isAbsolute, + join, + relative, + resolve, +} from "node:path"; +import { + BUILD_PROTOCOL_VERSION, + assertSha256Digest, + type Sha256Digest, + type WorkspaceFile, + type WorkspaceManifest, +} from "../shared/build-protocol"; + +export interface MaterializeCas { + get(digest: Sha256Digest): Promise; + has(digest: Sha256Digest): Promise; +} + +function safePath(path: string): boolean { + return ( + path.length > 0 && + !isAbsolute(path) && + !path.includes("\\") && + !path.includes("\0") && + path + .split("/") + .every((part) => part !== "" && part !== "." && part !== "..") + ); +} + +export function parseWorkspaceManifest(data: Uint8Array): WorkspaceManifest { + let value: unknown; + try { + value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(data)); + } catch { + throw new Error("Workspace manifest is not valid UTF-8 JSON"); + } + const manifest = value as Partial; + if ( + manifest.version !== BUILD_PROTOCOL_VERSION || + !Array.isArray(manifest.files) + ) { + throw new Error("Unsupported workspace manifest"); + } + const paths = new Set(); + for (const file of manifest.files as WorkspaceFile[]) { + if ( + !file || + !safePath(file.path) || + (file.type !== "file" && file.type !== "symlink") || + !Number.isSafeInteger(file.size) || + file.size < 0 || + ![0o644, 0o755, 0o777].includes(file.mode) + ) { + throw new Error("Workspace manifest contains an invalid file"); + } + assertSha256Digest(file.digest); + if (paths.has(file.path)) + throw new Error(`Duplicate workspace path: ${file.path}`); + for (const parent of dirname(file.path).split("/")) { + if (parent && paths.has(parent)) { + throw new Error(`Workspace path conflicts with a file: ${file.path}`); + } + } + paths.add(file.path); + } + for (const path of paths) { + if ([...paths].some((other) => other.startsWith(`${path}/`))) { + throw new Error(`Workspace path conflicts with a directory: ${path}`); + } + } + return manifest as WorkspaceManifest; +} + +function safeSymlinkTarget(filePath: string, target: string): boolean { + if ( + !target || + isAbsolute(target) || + target.includes("\\") || + target.includes("\0") + ) + return false; + const resolved = resolve("/workspace", dirname(filePath), target); + return resolved === "/workspace" || resolved.startsWith("/workspace/"); +} + +export async function materializeWorkspace( + cas: MaterializeCas, + manifestDigest: Sha256Digest, + destination: string, +): Promise { + assertSha256Digest(manifestDigest); + const manifest = parseWorkspaceManifest(await cas.get(manifestDigest)); + const missing: Sha256Digest[] = []; + for (const file of manifest.files) + if (!(await cas.has(file.digest))) missing.push(file.digest); + if (missing.length) + throw new Error(`Workspace blobs are missing: ${missing.join(", ")}`); + + await mkdir(dirname(destination), { recursive: true }); + try { + await lstat(destination); + throw new Error(`Workspace destination already exists: ${destination}`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const temporary = join( + dirname(destination), + `.${basename(destination)}.${process.pid}.${randomUUID()}.tmp`, + ); + await mkdir(temporary, { mode: 0o755 }); + try { + for (const file of manifest.files) { + const target = join(temporary, file.path); + if (relative(temporary, target).startsWith("..")) + throw new Error("Unsafe workspace path"); + await mkdir(dirname(target), { recursive: true, mode: 0o755 }); + const data = await cas.get(file.digest); + if (data.byteLength !== file.size) { + throw new Error(`Workspace blob size mismatch for ${file.path}`); + } + if (file.type === "symlink") { + const link = new TextDecoder("utf-8", { fatal: true }).decode(data); + if (!safeSymlinkTarget(file.path, link)) + throw new Error(`Unsafe symlink target for ${file.path}`); + await symlink(link, target); + } else { + const handle = await open( + target, + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, + file.mode, + ); + try { + await handle.writeFile(data); + } finally { + await handle.close(); + } + await chmod(target, file.mode); + } + } + await rename(temporary, destination); + } catch (error) { + await rm(temporary, { recursive: true, force: true }); + throw error; + } + return manifest; +} diff --git a/server/operation-store.ts b/server/operation-store.ts new file mode 100644 index 0000000..d407c3a --- /dev/null +++ b/server/operation-store.ts @@ -0,0 +1,398 @@ +import { createHash, randomUUID } from "node:crypto"; +import { KUBER_API_VERSION, type ObjectMeta } from "./workspace-store"; +import type { OperationError, OperationState } from "../shared/api"; +import { redactString, REDACTED } from "./redact"; + +export const MAX_IDEMPOTENCY_KEY_BYTES = 256; + +export type { OperationError, OperationState } from "../shared/api"; + +export interface Operation { + apiVersion: typeof KUBER_API_VERSION; + kind: "Operation"; + metadata: ObjectMeta & { workspaceUid?: string }; + spec: { + workspaceId: string; + action: string; + idempotencyKey: string; + requestHash: string; + }; + status: { + state: OperationState; + startedAt?: string; + finishedAt?: string; + result?: unknown; + error?: OperationError; + }; +} + +export interface CreateOperationInput { + workspaceId: string; + workspaceUid?: string; + action: string; + idempotencyKey: string; + request?: unknown; +} + +export class OperationValidationError extends Error { + readonly code = "OPERATION_INVALID"; +} + +export class OperationConflictError extends Error { + readonly code = "OPERATION_CONFLICT"; +} + +export class OperationNotFoundError extends Error { + readonly code = "OPERATION_NOT_FOUND"; +} + +/** createIdempotent must atomically index workspaceId + idempotencyKey. */ +export interface OperationPersistence { + createIdempotent(operation: Operation): Promise; + get(id: string): Promise; + list(workspaceId?: string): Promise; + replace(operation: Operation, expectedResourceVersion: string): Promise; +} + +export interface OperationStore { + create(input: CreateOperationInput): Promise; + get(id: string): Promise; + list(workspaceId?: string): Promise; + transition( + id: string, + state: OperationState, + options?: { result?: unknown; error?: OperationError }, + ): Promise; +} + +export interface WorkspaceLease { + workspaceId: string; + holder: string; + expiresAt: string; + renew(ttlMs?: number): Promise; + release(): Promise; +} + +export interface WorkspaceLeaseProvider { + acquire( + workspaceId: string, + holder: string, + ttlMs?: number, + ): Promise; +} + +/** + * Marks any operation left in a non-terminal state by a previous process as + * failed. On process restart no synchronous operation can legitimately still be + * in flight (startup runs before serving), so pending/running records are stale. + * Best-effort: individual conflicts or failures are logged and skipped so one + * bad record cannot block recovery. + */ +export async function recoverStaleOperations( + store: Pick, +): Promise { + const operations = await store.list(); + let recovered = 0; + for (const operation of operations) { + if ( + operation.status.state !== "pending" && + operation.status.state !== "running" + ) + continue; + const name = operation.metadata.name; + try { + await store.transition(name, "failed", { + error: { + code: "OPERATION_INTERRUPTED", + message: + "Server restarted while the operation was in progress; retry to proceed", + }, + }); + recovered += 1; + } catch (error) { + console.error( + `Failed to mark stale operation '${name}' as failed during startup recovery`, + error, + ); + } + } + return recovered; +} + +const TRANSITIONS: Record> = { + pending: new Set(["running", "failed", "cancelled"]), + running: new Set(["succeeded", "failed", "cancelled"]), + succeeded: new Set(), + failed: new Set(), + cancelled: new Set(), +}; + +function clone(value: T): T { + return structuredClone(value); +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") { + const encoded = JSON.stringify(value); + if (encoded === undefined) + throw new OperationValidationError( + "Operation request must be JSON serializable", + ); + return encoded; + } + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + return `{${Object.keys(value as Record) + .sort() + .map( + (key) => + `${JSON.stringify(key)}:${canonicalJson((value as Record)[key])}`, + ) + .join(",")}}`; +} + +function hashRequest(input: CreateOperationInput): string { + let request: string; + try { + request = canonicalJson(input.request ?? null); + } catch { + throw new OperationValidationError( + "Operation request must be JSON serializable", + ); + } + return createHash("sha256") + .update(`${input.action}\0${request}`) + .digest("hex"); +} + +const SENSITIVE_KEY = + /(?:password|passwd|token|secret|credential|private.?key|access.?key|connection.?string|database.?url)/i; + +/** Produces a persistence-safe result while retaining useful operation status. */ +export function sanitizeOperationResult( + value: unknown, + action?: string, +): unknown { + if (action === "databases.reconcile" || action === "storage.reconcile") { + return { redacted: true }; + } + if (typeof value === "string") return redactString(value); + if (Array.isArray(value)) + return value.map((item) => sanitizeOperationResult(item)); + if (!value || typeof value !== "object") return value; + const record = value as Record; + const isDataResource = + record.kind === "Secret" || record.kind === "ConfigMap"; + return Object.fromEntries( + Object.entries(record).map(([key, item]) => [ + key, + SENSITIVE_KEY.test(key) || + (isDataResource && (key === "data" || key === "stringData")) + ? REDACTED + : sanitizeOperationResult(item), + ]), + ); +} + +export class PersistentOperationStore implements OperationStore { + constructor( + private readonly persistence: OperationPersistence, + private readonly now: () => Date = () => new Date(), + private readonly uid: () => string = randomUUID, + ) {} + + async create(input: CreateOperationInput): Promise { + if (!input.workspaceId || !input.action.trim()) { + throw new OperationValidationError( + "Workspace ID and action are required", + ); + } + const keyBytes = Buffer.byteLength(input.idempotencyKey); + if (!input.idempotencyKey || keyBytes > MAX_IDEMPOTENCY_KEY_BYTES) { + throw new OperationValidationError( + `Idempotency key must be 1-${MAX_IDEMPOTENCY_KEY_BYTES} bytes`, + ); + } + const now = this.now().toISOString(); + const id = this.uid(); + const operation: Operation = { + apiVersion: KUBER_API_VERSION, + kind: "Operation", + metadata: { + name: `operation-${id}`, + uid: id, + resourceVersion: "1", + creationTimestamp: now, + ...(input.workspaceUid && { workspaceUid: input.workspaceUid }), + }, + spec: { + workspaceId: input.workspaceId, + action: input.action, + idempotencyKey: input.idempotencyKey, + requestHash: hashRequest(input), + }, + status: { state: "pending" }, + }; + const stored = await this.persistence.createIdempotent(operation); + if (stored.spec.requestHash !== operation.spec.requestHash) { + throw new OperationConflictError( + "Idempotency key was already used for a different request", + ); + } + return clone(stored); + } + + async get(id: string) { + const operation = await this.persistence.get(id); + return operation && clone(operation); + } + + async list(workspaceId?: string) { + return clone(await this.persistence.list(workspaceId)); + } + + async transition( + id: string, + state: OperationState, + options: { result?: unknown; error?: OperationError } = {}, + ): Promise { + const current = await this.persistence.get(id); + if (!current) + throw new OperationNotFoundError(`Operation '${id}' not found`); + if (!TRANSITIONS[current.status.state].has(state)) { + throw new OperationConflictError( + `Cannot transition operation from ${current.status.state} to ${state}`, + ); + } + if (state === "succeeded" && options.error) { + throw new OperationValidationError( + "Successful operations cannot have errors", + ); + } + if (state === "failed" && !options.error) { + throw new OperationValidationError("Failed operations require an error"); + } + const timestamp = this.now().toISOString(); + const operation: Operation = clone(current); + operation.metadata.resourceVersion = String( + Number(current.metadata.resourceVersion) + 1, + ); + operation.status = { + state, + ...(current.status.startedAt && { startedAt: current.status.startedAt }), + ...(state === "running" && { startedAt: timestamp }), + ...(["succeeded", "failed", "cancelled"].includes(state) && { + finishedAt: timestamp, + }), + ...(options.result !== undefined && { + result: clone( + sanitizeOperationResult(options.result, current.spec.action), + ), + }), + ...(options.error && { error: clone(options.error) }), + }; + await this.persistence.replace(operation, current.metadata.resourceVersion); + return clone(operation); + } +} + +export class MemoryOperationPersistence implements OperationPersistence { + private readonly operations = new Map(); + private readonly idempotency = new Map(); + + async createIdempotent(operation: Operation) { + const key = `${operation.spec.workspaceId}\0${operation.spec.idempotencyKey}`; + const existingId = this.idempotency.get(key); + if (existingId) return clone(this.operations.get(existingId)!); + this.operations.set(operation.metadata.name, clone(operation)); + this.idempotency.set(key, operation.metadata.name); + return clone(operation); + } + + async get(id: string) { + const operation = this.operations.get(id); + return operation && clone(operation); + } + + async list(workspaceId?: string) { + return [...this.operations.values()] + .filter((operation) => + workspaceId ? operation.spec.workspaceId === workspaceId : true, + ) + .sort((a, b) => + a.metadata.creationTimestamp.localeCompare( + b.metadata.creationTimestamp, + ), + ) + .map(clone); + } + + async replace(operation: Operation, expectedResourceVersion: string) { + const current = this.operations.get(operation.metadata.name); + if (!current) throw new OperationNotFoundError("Operation not found"); + if (current.metadata.resourceVersion !== expectedResourceVersion) { + throw new OperationConflictError("Operation was concurrently modified"); + } + this.operations.set(operation.metadata.name, clone(operation)); + } +} + +export class MemoryOperationStore extends PersistentOperationStore { + constructor(now?: () => Date, uid?: () => string) { + super(new MemoryOperationPersistence(), now, uid); + } +} + +export class MemoryWorkspaceLeaseProvider implements WorkspaceLeaseProvider { + private readonly leases = new Map< + string, + { holder: string; expiresAt: number; token: string } + >(); + + constructor(private readonly now: () => number = Date.now) {} + + async acquire(workspaceId: string, holder: string, ttlMs = 30_000) { + if (!workspaceId || !holder || !Number.isFinite(ttlMs) || ttlMs <= 0) { + throw new OperationValidationError("Invalid workspace lease request"); + } + const current = this.leases.get(workspaceId); + if ( + current && + current.expiresAt > this.now() && + current.holder !== holder + ) { + return undefined; + } + const token = randomUUID(); + this.leases.set(workspaceId, { + holder, + token, + expiresAt: this.now() + ttlMs, + }); + + const lease: WorkspaceLease = { + workspaceId, + holder, + expiresAt: new Date(this.now() + ttlMs).toISOString(), + renew: async (nextTtl = ttlMs) => { + const active = this.leases.get(workspaceId); + if ( + !active || + active.token !== token || + active.expiresAt <= this.now() || + nextTtl <= 0 + ) { + return false; + } + active.expiresAt = this.now() + nextTtl; + lease.expiresAt = new Date(active.expiresAt).toISOString(); + return true; + }, + release: async () => { + if (this.leases.get(workspaceId)?.token === token) { + this.leases.delete(workspaceId); + } + }, + }; + return lease; + } +} diff --git a/server/redact.ts b/server/redact.ts new file mode 100644 index 0000000..25cd0c1 --- /dev/null +++ b/server/redact.ts @@ -0,0 +1,19 @@ +export const REDACTED = "[REDACTED]"; + +// scheme://user:password@host — only redacts when a password is present, so +// ordinary URLs (with or without a userinfo) are left untouched. +const URL_CREDENTIALS = /\b([a-z][a-z0-9+.-]*:\/\/)[^/\s@]+:[^/\s@]*@/gi; + +// AWS access key ID (AKIA prefix + 16 uppercase alphanumeric chars). +const AWS_ACCESS_KEY_ID = /\bAKIA[0-9A-Z]{16}\b/g; + +// OpenSSH and other PEM private key headers embedded anywhere in a string. +const PRIVATE_KEY_HEADER = /-----BEGIN [A-Z ]*PRIVATE KEY-----/g; + +/** Redact common credential-bearing substrings while leaving everything else intact. */ +export function redactString(value: string): string { + return value + .replace(AWS_ACCESS_KEY_ID, REDACTED) + .replace(PRIVATE_KEY_HEADER, REDACTED) + .replace(URL_CREDENTIALS, (_match, scheme) => `${scheme}${REDACTED}@`); +} diff --git a/server/registry.ts b/server/registry.ts new file mode 100644 index 0000000..62e6955 --- /dev/null +++ b/server/registry.ts @@ -0,0 +1,175 @@ +import { createHash } from "node:crypto"; +import { + assertSha256Digest, + type Sha256Digest, +} from "../shared/build-protocol"; + +const ACCEPT = [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.docker.distribution.manifest.v2+json", +].join(", "); + +export type RegistryCredentials = { username: string; password: string }; +export type RegistryFetch = ( + input: string | URL | Request, + init?: RequestInit, +) => Promise; +export type RegistryResolveOptions = { + fetch?: RegistryFetch; + credentials?: RegistryCredentials; + insecure?: boolean; + origin?: string; +}; + +export type ParsedImageReference = { + registry: string; + repository: string; + reference: string; + digest?: Sha256Digest; +}; + +function validateRepository(repository: string, image: string): void { + if ( + !repository || + repository.split("/").some((part) => !part || part === "." || part === "..") + ) + throw new Error(`Invalid image reference: ${image}`); +} + +export function parseImageReference(image: string): ParsedImageReference { + const slash = image.indexOf("/"); + if (slash <= 0) + throw new Error("Image reference must include a registry host"); + const registry = image.slice(0, slash); + let repositoryAndReference = image.slice(slash + 1); + if ( + !repositoryAndReference || + !registry || + /[/?#@]/.test(registry) || + /\s/.test(image) + ) + throw new Error(`Invalid image reference: ${image}`); + + const at = repositoryAndReference.lastIndexOf("@"); + if (at !== -1) { + const value = repositoryAndReference.slice(at + 1); + assertSha256Digest(value); + repositoryAndReference = repositoryAndReference.slice(0, at); + validateRepository(repositoryAndReference, image); + return { + registry, + repository: repositoryAndReference, + reference: value, + digest: value, + }; + } + const lastSlash = repositoryAndReference.lastIndexOf("/"); + const colon = repositoryAndReference.lastIndexOf(":"); + const reference = + colon > lastSlash ? repositoryAndReference.slice(colon + 1) : "latest"; + const repository = + colon > lastSlash + ? repositoryAndReference.slice(0, colon) + : repositoryAndReference; + validateRepository(repository, image); + if (!reference) throw new Error(`Invalid image reference: ${image}`); + return { registry, repository, reference }; +} + +function bearerParameters( + challenge: string, +): Record | undefined { + const match = /^Bearer\s+(.+)$/i.exec(challenge.trim()); + if (!match?.[1]) return; + const values: Record = {}; + const expression = /([a-z][a-z0-9_-]*)=(?:"((?:\\.|[^"])*)"|([^,\s]+))/gi; + for (const item of match[1].matchAll(expression)) + values[item[1]!.toLowerCase()] = (item[2] ?? item[3] ?? "").replace( + /\\"/g, + '"', + ); + return values.realm ? values : undefined; +} + +async function responseError(response: Response): Promise { + const detail = (await response.text()).slice(0, 512).trim(); + return new Error( + `Registry request failed (${response.status})${detail ? `: ${detail}` : ""}`, + ); +} + +export async function resolveRegistryDigest( + image: string, + options: RegistryResolveOptions = {}, +): Promise { + const parsed = parseImageReference(image); + if (parsed.digest) return parsed.digest; + const fetcher: RegistryFetch = options.fetch ?? globalThis.fetch; + const scheme = options.insecure ? "http" : "https"; + const repository = parsed.repository + .split("/") + .map(encodeURIComponent) + .join("/"); + const origin = (options.origin ?? `${scheme}://${parsed.registry}`).replace( + /\/+$/, + "", + ); + const manifestUrl = `${origin}/v2/${repository}/manifests/${encodeURIComponent(parsed.reference)}`; + const headers = new Headers({ accept: ACCEPT }); + if (options.credentials) { + headers.set( + "authorization", + `Basic ${Buffer.from(`${options.credentials.username}:${options.credentials.password}`).toString("base64")}`, + ); + } + + let response = await fetcher(manifestUrl, { headers }); + if (response.status === 401) { + const challenge = bearerParameters( + response.headers.get("www-authenticate") ?? "", + ); + if (!challenge) throw await responseError(response); + const tokenUrl = new URL(challenge.realm!); + if (tokenUrl.protocol !== "https:" && !options.insecure) + throw new Error("Registry bearer token realm must use HTTPS"); + if (tokenUrl.protocol !== "https:" && tokenUrl.protocol !== "http:") + throw new Error("Registry bearer token realm must use HTTP or HTTPS"); + if (challenge.service) + tokenUrl.searchParams.set("service", challenge.service); + tokenUrl.searchParams.set( + "scope", + challenge.scope ?? `repository:${parsed.repository}:pull`, + ); + const tokenHeaders = new Headers(); + if (options.credentials) + tokenHeaders.set( + "authorization", + `Basic ${Buffer.from(`${options.credentials.username}:${options.credentials.password}`).toString("base64")}`, + ); + const tokenResponse = await fetcher(tokenUrl, { headers: tokenHeaders }); + if (!tokenResponse.ok) throw await responseError(tokenResponse); + const payload = (await tokenResponse.json()) as { + token?: unknown; + access_token?: unknown; + }; + const token = payload.token ?? payload.access_token; + if (typeof token !== "string" || !token) + throw new Error("Registry token response did not contain a token"); + headers.set("authorization", `Bearer ${token}`); + response = await fetcher(manifestUrl, { headers }); + } + if (!response.ok) throw await responseError(response); + + const body = new Uint8Array(await response.arrayBuffer()); + const advertised = response.headers + .get("docker-content-digest") + ?.trim() + ?.toLowerCase(); + if (advertised !== undefined) { + assertSha256Digest(advertised); + return advertised; + } + return `sha256:${createHash("sha256").update(body).digest("hex")}`; +} diff --git a/server/workspace-store.ts b/server/workspace-store.ts new file mode 100644 index 0000000..b519719 --- /dev/null +++ b/server/workspace-store.ts @@ -0,0 +1,377 @@ +import { randomUUID } from "node:crypto"; + +export const KUBER_API_VERSION = "kuber.astrxl.dev/v2" as const; +export const MAX_WORKSPACE_CONFIG_BYTES = 768 * 1024; +export const MAX_SOURCE_REFERENCE_BYTES = 64 * 1024; +export const MAX_REVISION_PAYLOAD_BYTES = 900 * 1024; + +export const WORKSPACE_ID_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; +export const RESERVED_WORKSPACE_IDS: ReadonlySet = new Set([ + "kuber-system", + "database", + "garage-system", + "routing", +]); + +export interface ObjectMeta { + name: string; + uid: string; + resourceVersion: string; + creationTimestamp: string; + labels?: Record; + annotations?: Record; +} + +export interface WorkspaceSourceReference { + uri: string; + digest: string; + revision?: string; +} + +export interface WorkspaceSpec { + source: WorkspaceSourceReference; + config?: unknown; +} + +export interface WorkspaceStatus { + latestRevision: number; +} + +export interface Workspace { + apiVersion: typeof KUBER_API_VERSION; + kind: "Workspace"; + metadata: ObjectMeta; + spec: WorkspaceSpec; + status: WorkspaceStatus; +} + +export interface WorkspaceRevision { + apiVersion: typeof KUBER_API_VERSION; + kind: "WorkspaceRevision"; + metadata: ObjectMeta & { + workspaceUid: string; + }; + spec: Readonly & { + workspaceId: string; + revision: number; + }; +} + +export interface CreateWorkspaceInput { + id: string; + source: WorkspaceSourceReference; + config?: unknown; + labels?: Record; + annotations?: Record; +} + +export interface UpdateWorkspaceInput { + source: WorkspaceSourceReference; + config?: unknown; + labels?: Record; + annotations?: Record; +} + +export class WorkspaceValidationError extends Error { + readonly code = "WORKSPACE_INVALID"; +} + +export class WorkspaceConflictError extends Error { + readonly code = "WORKSPACE_CONFLICT"; +} + +export class WorkspaceNotFoundError extends Error { + readonly code = "WORKSPACE_NOT_FOUND"; +} + +/** Persistence must atomically compare resourceVersion and append the revision. */ +export interface WorkspacePersistence { + get(id: string): Promise; + list(): Promise; + create(workspace: Workspace, revision: WorkspaceRevision): Promise; + replace( + workspace: Workspace, + revision: WorkspaceRevision, + expectedResourceVersion: string, + ): Promise; + getRevision( + workspaceId: string, + revision: number, + ): Promise; + listRevisions(workspaceId: string): Promise; +} + +export interface WorkspaceStore { + create(input: CreateWorkspaceInput): Promise; + get(id: string): Promise; + list(): Promise; + update( + id: string, + input: UpdateWorkspaceInput, + ifMatch: string, + ): Promise; + getRevision( + id: string, + revision: number, + ): Promise; + listRevisions(id: string): Promise; +} + +export interface WorkspaceStoreOptions { + now?: () => Date; + uid?: () => string; +} + +export function validateWorkspaceId(id: string): void { + if (!WORKSPACE_ID_PATTERN.test(id)) { + throw new WorkspaceValidationError( + "Workspace ID must be a lowercase DNS label of at most 63 characters", + ); + } + if (id.startsWith("kube-") || RESERVED_WORKSPACE_IDS.has(id)) { + throw new WorkspaceValidationError(`Workspace ID '${id}' is reserved`); + } +} + +export function workspaceEtag(workspace: Pick): string { + return `"${workspace.metadata.resourceVersion}"`; +} + +export function resourceVersionFromEtag(etag: string): string { + const value = etag.trim(); + const match = /^(?:W\/)?"([^"\\]+)"$/.exec(value); + if (!match?.[1]) { + throw new WorkspaceValidationError("If-Match must contain a valid ETag"); + } + return match[1]; +} + +function encodedSize(value: unknown): number { + let serialized: string; + try { + serialized = JSON.stringify(value); + } catch { + throw new WorkspaceValidationError( + "Workspace payload must be JSON serializable", + ); + } + if (serialized === undefined) { + throw new WorkspaceValidationError( + "Workspace payload must be JSON serializable", + ); + } + return Buffer.byteLength(serialized); +} + +function validateSpec(spec: WorkspaceSpec): void { + if (!spec.source || typeof spec.source !== "object") { + throw new WorkspaceValidationError("A source reference is required"); + } + if (!spec.source.uri?.trim() || !spec.source.digest?.trim()) { + throw new WorkspaceValidationError("Source uri and digest are required"); + } + const source = spec.source as unknown as Record; + for (const field of ["blob", "content", "data", "archive", "files"]) { + if (field in source) { + throw new WorkspaceValidationError( + "Inline source blobs are not supported; provide a source reference", + ); + } + } + if (encodedSize(spec.source) > MAX_SOURCE_REFERENCE_BYTES) { + throw new WorkspaceValidationError("Source reference is too large"); + } + if (encodedSize(spec.config ?? null) > MAX_WORKSPACE_CONFIG_BYTES) { + throw new WorkspaceValidationError("Workspace config is too large"); + } + if (encodedSize(spec) > MAX_REVISION_PAYLOAD_BYTES) { + throw new WorkspaceValidationError( + "Workspace revision payload is too large", + ); + } +} + +function clone(value: T): T { + return structuredClone(value); +} + +function revisionFor( + workspace: Workspace, + uid: () => string, + creationTimestamp: string, +): WorkspaceRevision { + const revision = workspace.status.latestRevision; + return { + apiVersion: KUBER_API_VERSION, + kind: "WorkspaceRevision", + metadata: { + name: `${workspace.metadata.name}-r${revision}`, + uid: uid(), + workspaceUid: workspace.metadata.uid, + resourceVersion: workspace.metadata.resourceVersion, + creationTimestamp, + }, + spec: clone({ + ...workspace.spec, + workspaceId: workspace.metadata.name, + revision, + }), + }; +} + +export class PersistentWorkspaceStore implements WorkspaceStore { + private readonly now: () => Date; + private readonly uid: () => string; + + constructor( + private readonly persistence: WorkspacePersistence, + options: WorkspaceStoreOptions = {}, + ) { + this.now = options.now ?? (() => new Date()); + this.uid = options.uid ?? randomUUID; + } + + async create(input: CreateWorkspaceInput): Promise { + validateWorkspaceId(input.id); + const spec = clone({ source: input.source, config: input.config }); + validateSpec(spec); + const workspace: Workspace = { + apiVersion: KUBER_API_VERSION, + kind: "Workspace", + metadata: { + name: input.id, + uid: this.uid(), + resourceVersion: "1", + creationTimestamp: this.now().toISOString(), + ...(input.labels && { labels: clone(input.labels) }), + ...(input.annotations && { annotations: clone(input.annotations) }), + }, + spec, + status: { latestRevision: 1 }, + }; + await this.persistence.create( + workspace, + revisionFor(workspace, this.uid, workspace.metadata.creationTimestamp), + ); + return clone(workspace); + } + + async get(id: string): Promise { + const workspace = await this.persistence.get(id); + return workspace && clone(workspace); + } + + async list(): Promise { + return clone(await this.persistence.list()); + } + + async update( + id: string, + input: UpdateWorkspaceInput, + ifMatch: string, + ): Promise { + validateWorkspaceId(id); + const current = await this.persistence.get(id); + if (!current) + throw new WorkspaceNotFoundError(`Workspace '${id}' not found`); + const expected = resourceVersionFromEtag(ifMatch); + if (expected !== current.metadata.resourceVersion) { + throw new WorkspaceConflictError("Workspace ETag does not match"); + } + const spec = clone({ source: input.source, config: input.config }); + validateSpec(spec); + const workspace: Workspace = { + ...clone(current), + metadata: { + ...clone(current.metadata), + resourceVersion: String(Number(current.metadata.resourceVersion) + 1), + ...(input.labels !== undefined && { labels: clone(input.labels) }), + ...(input.annotations !== undefined && { + annotations: clone(input.annotations), + }), + }, + spec, + status: { latestRevision: current.status.latestRevision + 1 }, + }; + await this.persistence.replace( + workspace, + revisionFor(workspace, this.uid, this.now().toISOString()), + expected, + ); + return clone(workspace); + } + + async getRevision(id: string, revision: number) { + const value = await this.persistence.getRevision(id, revision); + return value && clone(value); + } + + async listRevisions(id: string) { + return clone(await this.persistence.listRevisions(id)); + } +} + +export class MemoryWorkspacePersistence implements WorkspacePersistence { + private readonly workspaces = new Map(); + private readonly revisions = new Map< + string, + Map + >(); + + async get(id: string) { + const value = this.workspaces.get(id); + return value && clone(value); + } + + async list() { + return [...this.workspaces.values()] + .sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)) + .map(clone); + } + + async create(workspace: Workspace, revision: WorkspaceRevision) { + if (this.workspaces.has(workspace.metadata.name)) { + throw new WorkspaceConflictError("Workspace already exists"); + } + this.workspaces.set(workspace.metadata.name, clone(workspace)); + this.revisions.set( + workspace.metadata.name, + new Map([[revision.spec.revision, clone(revision)]]), + ); + } + + async replace( + workspace: Workspace, + revision: WorkspaceRevision, + expectedResourceVersion: string, + ) { + const current = this.workspaces.get(workspace.metadata.name); + if (!current) throw new WorkspaceNotFoundError("Workspace not found"); + if (current.metadata.resourceVersion !== expectedResourceVersion) { + throw new WorkspaceConflictError("Workspace was concurrently modified"); + } + const revisions = this.revisions.get(workspace.metadata.name); + if (!revisions || revisions.has(revision.spec.revision)) { + throw new WorkspaceConflictError("Workspace revision already exists"); + } + this.workspaces.set(workspace.metadata.name, clone(workspace)); + revisions.set(revision.spec.revision, clone(revision)); + } + + async getRevision(id: string, revision: number) { + const value = this.revisions.get(id)?.get(revision); + return value && clone(value); + } + + async listRevisions(id: string) { + return [...(this.revisions.get(id)?.values() ?? [])] + .sort((a, b) => a.spec.revision - b.spec.revision) + .map(clone); + } +} + +export class MemoryWorkspaceStore extends PersistentWorkspaceStore { + constructor(options: WorkspaceStoreOptions = {}) { + super(new MemoryWorkspacePersistence(), options); + } +} diff --git a/shared/api.ts b/shared/api.ts new file mode 100644 index 0000000..2ac7f2b --- /dev/null +++ b/shared/api.ts @@ -0,0 +1,245 @@ +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = + | JsonPrimitive + | JsonValue[] + | { [key: string]: JsonValue }; + +export type ApiProblemDetails = { + type?: string; + title: string; + status: number; + detail?: string; + message?: string; + /** Stable machine-readable identifier, unlike title or detail. */ + code: string; + requestId?: string; + operationId?: string; + errors?: Record; +}; + +export type Page = { + items: T[]; + nextCursor?: string; +}; + +export const KUBER_API_VERSION = "kuber.astrxl.dev/v2" as const; + +export type ObjectMeta = { + name: string; + uid: string; + resourceVersion: string; + creationTimestamp: string; + labels?: Record; + annotations?: Record; +}; + +export type WorkspaceSourceReference = { + uri: string; + digest: string; + revision?: string; +}; + +export type WorkspaceSpec = { + source: WorkspaceSourceReference; + config?: unknown; +}; + +export type Workspace = { + apiVersion: typeof KUBER_API_VERSION; + kind: "Workspace"; + metadata: ObjectMeta; + spec: WorkspaceSpec; + status: { latestRevision: number }; +}; + +export type CreateWorkspaceRequest = { + id: string; + source: WorkspaceSourceReference; + config?: unknown; + labels?: Record; + annotations?: Record; +}; +export type UpdateWorkspaceRequest = Omit; +export type CreateWorkspaceResponse = Workspace; +export type ListWorkspacesResponse = Page; +export type GetWorkspaceResponse = Workspace; +export type DeleteWorkspaceResponse = OperationAcceptedResponse; +export type AdoptWorkspaceResponse = { + workspaceId: string; + workspaceUid: string; + resourcesAdopted: number; +}; +export type AdoptPlatformRequest = { workspaceUid: string }; +export type AdoptPlatformResponse = AdoptWorkspaceResponse; + +export type WorkspaceState = + | "pending" + | "ready" + | "degraded" + | "stopped" + | "deleting" + | "failed"; + +export type ServiceStatus = { + name: string; + state: "pending" | "running" | "stopped" | "failed" | "unknown"; + readyReplicas: number; + desiredReplicas: number; + image?: string; + message?: string; +}; + +export type WorkspaceStatusResponse = { + workspaceId: string; + state: WorkspaceState; + services: ServiceStatus[]; + observedAt: string; +}; + +export type LifecycleAction = "start" | "stop" | "restart" | "delete"; +export type LifecycleRequest = { + action: LifecycleAction; + services?: string[]; +}; +export type LifecycleResponse = OperationAcceptedResponse; + +export type ResourceObject = { + apiVersion: string; + kind: string; + metadata: { + name: string; + namespace?: string; + labels?: Record; + annotations?: Record; + }; +} & Record; + +export type ResourceChange = { + apiVersion: string; + kind: string; + name: string; + action: "create" | "update" | "delete" | "unchanged"; +}; +export type ReconcileResourcesRequest = { + resources: ResourceObject[]; + prune?: boolean; + dryRun?: boolean; +}; +export type ReconcileResourcesResponse = { + operationId?: string; + changes: ResourceChange[]; +}; + +export type DatabaseEngine = "postgres"; +export type DatabaseClaim = { + name: string; + engine: DatabaseEngine; + database?: string; + username?: string; +}; +export type ReconcileDatabasesRequest = { claims: DatabaseClaim[] }; +export type DatabaseBinding = { + claim: string; + service: string; + environment: Record; +}; +export type ReconcileDatabasesResponse = { + bindings: DatabaseBinding[]; + operationId?: string; +}; + +export type StorageKind = "s3"; +export type StorageClaim = { + name: string; + kind: StorageKind; + bucket?: string; + region?: string; +}; +export type ReconcileStorageRequest = { claims: StorageClaim[] }; +export type StorageBinding = { + claim: string; + service: string; + environment: Record; +}; +export type ReconcileStorageResponse = { + bindings: StorageBinding[]; + operationId?: string; +}; + +export type OperationState = + | "pending" + | "running" + | "succeeded" + | "failed" + | "cancelled"; +export type OperationError = { code: string; message: string }; +export type Operation = { + apiVersion: typeof KUBER_API_VERSION; + kind: "Operation"; + metadata: ObjectMeta & { workspaceUid?: string }; + spec: { workspaceId: string; action: string }; + status: { + state: OperationState; + startedAt?: string; + finishedAt?: string; + result?: unknown; + error?: OperationError; + }; +}; +export type OperationAcceptedResponse = { + operationId: string; + operation?: Operation; +}; +export type GetOperationResponse = Operation; +export type CancelOperationResponse = Operation; +export type OperationEvent = { + operationId: string; + sequence: number; + timestamp: string; + type: "status" | "progress" | "log" | "result" | "error"; + data: JsonValue; +}; + +export type UserRole = "admin" | "operator" | "viewer" | (string & {}); +export type User = { + username: string; + roles: UserRole[]; + disabled: boolean; + createdAt?: string; + updatedAt?: string; +}; +export type CreateUserRequest = { + username: string; + password: string; + roles: UserRole[]; +}; +export type UpdateUserRequest = { + password?: string; + roles?: UserRole[]; + disabled?: boolean; +}; +export type UserResponse = User; +export type ListUsersResponse = Page; + +export type AuditEvent = { + id: string; + timestamp: string; + actor: string; + action: string; + resourceType: string; + resourceId?: string; + workspaceId?: string; + requestId?: string; + operationId?: string; + outcome: "success" | "failure"; + metadata?: Record; +}; +export type ListAuditEventsRequest = { + cursor?: string; + limit?: number; + actor?: string; + workspaceId?: string; + since?: string; + until?: string; +}; +export type ListAuditEventsResponse = Page; diff --git a/shared/artifacts.ts b/shared/artifacts.ts new file mode 100644 index 0000000..a27c75f --- /dev/null +++ b/shared/artifacts.ts @@ -0,0 +1,226 @@ +import { existsSync, readFileSync, realpathSync, statSync } from "node:fs"; +import { + dirname, + isAbsolute, + normalize, + relative, + resolve, + sep, +} from "node:path"; + +export interface ArtifactProvider { + isFile(path: string, options?: ArtifactReadOptions): boolean; + readText(path: string, options?: ArtifactReadOptions): string; +} + +export type ArtifactReadOptions = { + expandHome?: boolean; +}; + +export type ArtifactBundle = { + version: 1; + files: Record; +}; + +export type ArtifactLimits = { + maxArtifactCount?: number; + maxArtifactBytes?: number; + maxTotalBytes?: number; +}; + +export type LocalArtifactProviderOptions = ArtifactLimits & { + workspace: string; + strict?: boolean; +}; + +export type BundleArtifactProviderOptions = ArtifactLimits; + +export const DEFAULT_ARTIFACT_LIMITS = { + maxArtifactCount: 100, + maxArtifactBytes: 1024 * 1024, + maxTotalBytes: 4 * 1024 * 1024, +} as const; + +function artifactError( + message: string, + code?: string, +): Error & { code?: string } { + return Object.assign(new Error(message), code ? { code } : {}); +} + +function toWorkspacePath(path: string): string { + if ( + !path || + path.includes("\0") || + path.includes("\\") || + path.startsWith("~") || + isAbsolute(path) + ) { + throw artifactError(`Artifact path must be workspace-relative: ${path}`); + } + + const normalized = normalize(path); + if (normalized === ".." || normalized.startsWith(`..${sep}`)) { + throw artifactError(`Artifact path escapes the workspace: ${path}`); + } + + return normalized.replace(/^\.\//, ""); +} + +function assertWithinWorkspace(workspace: string, path: string): void { + const relation = relative(workspace, path); + if ( + relation === ".." || + relation.startsWith(`..${sep}`) || + isAbsolute(relation) + ) { + throw artifactError(`Artifact path escapes the workspace: ${path}`); + } +} + +function assertRealPathWithinWorkspace(workspace: string, path: string): void { + let existingPath = path; + while (!existsSync(existingPath)) { + const parent = dirname(existingPath); + if (parent === existingPath) break; + existingPath = parent; + } + assertWithinWorkspace(workspace, realpathSync(existingPath)); +} + +function assertPositiveLimit(name: string, value: number | undefined): void { + if (value !== undefined && (!Number.isSafeInteger(value) || value < 0)) { + throw new Error(`${name} must be a non-negative safe integer`); + } +} + +class ArtifactBudget { + readonly #limits: ArtifactLimits; + #count = 0; + #bytes = 0; + + constructor(limits: ArtifactLimits) { + assertPositiveLimit("maxArtifactCount", limits.maxArtifactCount); + assertPositiveLimit("maxArtifactBytes", limits.maxArtifactBytes); + assertPositiveLimit("maxTotalBytes", limits.maxTotalBytes); + this.#limits = limits; + } + + add(path: string, content: string): void { + const bytes = Buffer.byteLength(content); + if ( + this.#limits.maxArtifactBytes !== undefined && + bytes > this.#limits.maxArtifactBytes + ) { + throw artifactError( + `Artifact ${path} exceeds the ${this.#limits.maxArtifactBytes} byte limit`, + ); + } + if ( + this.#limits.maxArtifactCount !== undefined && + this.#count + 1 > this.#limits.maxArtifactCount + ) { + throw artifactError( + `Artifact count exceeds the ${this.#limits.maxArtifactCount} limit`, + ); + } + if ( + this.#limits.maxTotalBytes !== undefined && + this.#bytes + bytes > this.#limits.maxTotalBytes + ) { + throw artifactError( + `Artifact bytes exceed the ${this.#limits.maxTotalBytes} byte limit`, + ); + } + + this.#count += 1; + this.#bytes += bytes; + } +} + +export class LocalArtifactProvider implements ArtifactProvider { + readonly #workspace: string; + readonly #strict: boolean; + readonly #budget: ArtifactBudget; + + constructor(options: LocalArtifactProviderOptions) { + this.#workspace = options.strict + ? realpathSync(options.workspace) + : resolve(options.workspace); + this.#strict = options.strict ?? false; + this.#budget = new ArtifactBudget(options); + } + + #resolve(path: string, options: ArtifactReadOptions): string { + if (!this.#strict) { + return options.expandHome && path.startsWith("~") + ? resolve(process.env.HOME ?? "", path.slice(1)) + : resolve(this.#workspace, path); + } + + const candidate = resolve(this.#workspace, toWorkspacePath(path)); + assertWithinWorkspace(this.#workspace, candidate); + assertRealPathWithinWorkspace(this.#workspace, candidate); + return candidate; + } + + isFile(path: string, options: ArtifactReadOptions = {}): boolean { + const resolved = this.#resolve(path, options); + return existsSync(resolved) && statSync(resolved).isFile(); + } + + readText(path: string, options: ArtifactReadOptions = {}): string { + const resolved = this.#resolve(path, options); + if (this.#strict) { + // Resolve again at read time so a symlink swap cannot bypass confinement. + assertWithinWorkspace(this.#workspace, realpathSync(resolved)); + } + const content = readFileSync(resolved, "utf8"); + this.#budget.add(path, content); + return content; + } +} + +export class BundleArtifactProvider implements ArtifactProvider { + readonly #files = new Map(); + + constructor( + bundle: ArtifactBundle, + options: BundleArtifactProviderOptions = {}, + ) { + if (bundle.version !== 1) + throw new Error("Unsupported artifact bundle version"); + + const budget = new ArtifactBudget({ + ...DEFAULT_ARTIFACT_LIMITS, + ...options, + }); + for (const [path, content] of Object.entries(bundle.files)) { + const normalized = toWorkspacePath(path); + if (this.#files.has(normalized)) { + throw artifactError(`Duplicate artifact path: ${path}`); + } + budget.add(normalized, content); + this.#files.set(normalized, content); + } + } + + isFile(path: string): boolean { + return this.#files.has(toWorkspacePath(path)); + } + + readText(path: string): string { + const normalized = toWorkspacePath(path); + const content = this.#files.get(normalized); + if (content === undefined) { + throw artifactError(`Artifact not found: ${path}`, "ENOENT"); + } + return content; + } +} + +export function createArtifactBundle( + files: Record, +): ArtifactBundle { + return { version: 1, files: { ...files } }; +} diff --git a/shared/build-protocol.ts b/shared/build-protocol.ts new file mode 100644 index 0000000..0a2cd75 --- /dev/null +++ b/shared/build-protocol.ts @@ -0,0 +1,63 @@ +export const BUILD_PROTOCOL_VERSION = 1 as const; + +export type Sha256Digest = `sha256:${string}`; +export type BuildArchitecture = "amd64" | "arm64"; + +export type WorkspaceFile = { + path: string; + type: "file" | "symlink"; + digest: Sha256Digest; + size: number; + mode: 0o644 | 0o755 | 0o777; +}; + +export type WorkspaceManifest = { + version: typeof BUILD_PROTOCOL_VERSION; + files: WorkspaceFile[]; +}; + +export type BuildSpec = { + architecture: BuildArchitecture; + image: string; + context: string; + dockerfile?: string; + target?: string; + buildArgs: string[]; + workspace: Sha256Digest; +}; + +export type BuildRequest = { + version: typeof BUILD_PROTOCOL_VERSION; + id: string; + project: string; + service: string; + spec: BuildSpec; +}; + +export type BuildState = "queued" | "running" | "succeeded" | "failed"; + +export type BuildStatus = { + version: typeof BUILD_PROTOCOL_VERSION; + id: string; + state: BuildState; + createdAt: string; + startedAt?: string; + finishedAt?: string; + digest?: Sha256Digest; + error?: string; +}; + +export type BuildEvent = + | { type: "status"; status: BuildStatus } + | { type: "log"; id: string; sequence: number; message: string }; + +export function isSha256Digest(value: unknown): value is Sha256Digest { + return typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value); +} + +export function assertSha256Digest( + value: unknown, +): asserts value is Sha256Digest { + if (!isSha256Digest(value)) + throw new Error(`Invalid SHA-256 digest: ${value}`); +} diff --git a/tests/command/administration.test.ts b/tests/command/administration.test.ts new file mode 100644 index 0000000..6c5e182 --- /dev/null +++ b/tests/command/administration.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, test } from "bun:test"; +import { listAuditEvents } from "../../command/audit"; +import { main } from "../../command/main"; +import { getOperation, listOperations } from "../../command/operations"; +import { + addUser, + deleteUser, + listUsers, + revokeUserSessions, + setUserDisabled, + updateUser, +} from "../../command/users"; +import type { ApiRequestInit } from "../../lib/api"; + +type Call = { path: string; init?: ApiRequestInit }; + +function requestReturning(result: T, calls: Call[]) { + return async (path: string, init?: ApiRequestInit): Promise => { + calls.push({ path, init }); + return result as unknown as R; + }; +} + +const user = { + username: "alice", + roles: ["admin"], + disabled: false, + updatedAt: "2026-09-02T10:00:00.000Z", +}; + +describe("user administration commands", () => { + test("lists and creates users through authenticated API routes", async () => { + const calls: Call[] = []; + expect( + await listUsers(requestReturning({ items: [user] }, calls)), + ).toContain("alice"); + expect( + await addUser( + "alice", + "secret", + ["admin"], + requestReturning(user, calls), + ), + ).toContain("admin"); + + expect(calls).toEqual([ + { path: "/users", init: undefined }, + { + path: "/users", + init: { + method: "POST", + json: { username: "alice", password: "secret", roles: ["admin"] }, + }, + }, + ]); + }); + + test("updates roles, passwords, and enabled state with PATCH", async () => { + const calls: Call[] = []; + const request = requestReturning(user, calls); + await updateUser( + "alice/example", + { roles: ["operator"], password: "new" }, + request, + ); + await setUserDisabled("alice", true, request); + await setUserDisabled("alice", false, request); + + expect(calls).toEqual([ + { + path: "/users/alice%2Fexample", + init: { + method: "PATCH", + json: { roles: ["operator"], password: "new" }, + }, + }, + { + path: "/users/alice", + init: { method: "PATCH", json: { disabled: true } }, + }, + { + path: "/users/alice", + init: { method: "PATCH", json: { disabled: false } }, + }, + ]); + }); + + test("requires confirmation for deletion and supports session revocation", async () => { + const calls: Call[] = []; + const request = requestReturning(undefined, calls); + expect(await deleteUser("alice", false, request)).toBe( + "Deletion cancelled", + ); + expect(await deleteUser("alice", true, request)).toBe("Deleted user alice"); + expect( + await revokeUserSessions( + "alice", + requestReturning({ username: "alice", revoked: 2 }, calls), + ), + ).toBe("Revoked 2 sessions for alice"); + + expect(calls).toEqual([ + { path: "/users/alice", init: { method: "DELETE" } }, + { + path: "/users/alice/sessions/revoke", + init: { method: "POST" }, + }, + ]); + }); +}); + +const operation = { + metadata: { + name: "operation-1", + creationTimestamp: "2026-09-02T10:00:00.000Z", + }, + spec: { workspaceId: "team/shop", action: "restart" }, + status: { state: "succeeded", result: { deployments: ["web"] } }, +}; + +describe("operations and audit commands", () => { + test("lists filtered operations and gets operation details", async () => { + const calls: Call[] = []; + const listing = await listOperations( + "team/shop", + requestReturning({ items: [operation] }, calls), + ); + const detail = await getOperation( + "operation/1", + requestReturning(operation, calls), + ); + + expect(listing).toContain("restart"); + expect(detail).toContain('Result: {"deployments":["web"]}'); + expect(calls).toEqual([ + { path: "/operations?workspaceId=team%2Fshop", init: undefined }, + { path: "/operations/operation%2F1", init: undefined }, + ]); + }); + + test("lists audit events with an optional workspace filter", async () => { + const calls: Call[] = []; + const output = await listAuditEvents( + "team/shop", + requestReturning( + { + items: [ + { + metadata: { + name: "audit-1", + creationTimestamp: "2026-09-02T10:00:00.000Z", + }, + spec: { + actor: { username: "alice" }, + action: "workspace.restart", + workspaceId: "team/shop", + outcome: "success", + }, + }, + ], + }, + calls, + ), + ); + + expect(output).toContain("alice"); + expect(output).toContain("workspace.restart"); + expect(calls).toEqual([ + { path: "/audit?workspaceId=team%2Fshop", init: undefined }, + ]); + }); + + test("registers administration command groups", async () => { + const subCommands = await Promise.resolve(main.subCommands); + expect(Object.keys(subCommands ?? {})).toEqual( + expect.arrayContaining(["users", "operations", "audit"]), + ); + }); +}); diff --git a/tests/command/api-migration.test.ts b/tests/command/api-migration.test.ts new file mode 100644 index 0000000..01eb2bb --- /dev/null +++ b/tests/command/api-migration.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, test } from "bun:test"; +import { getDatabaseCredentials } from "../../command/db"; +import { runDown } from "../../command/down"; +import { runPs } from "../../command/ps"; +import { runRestart } from "../../command/restart"; +import { runRollback } from "../../command/rollback"; +import { getRemoteS3Credentials } from "../../command/s3"; +import { runStop } from "../../command/stop"; +import type { ApiRequestInit } from "../../lib/api"; + +type Call = { path: string; init?: ApiRequestInit }; + +function requestReturning(result: T, calls: Call[]) { + return async (path: string, init?: ApiRequestInit): Promise => { + calls.push({ path, init }); + return result as unknown as R; + }; +} + +describe("workspace API command runners", () => { + test("ps requests structured status and renders it locally", async () => { + const calls: Call[] = []; + const output = await runPs( + "shop/demo", + true, + requestReturning( + [ + { + namespace: "shop-demo", + roots: [ + { + id: "Deployment/web", + status: { label: "1/1", level: "good" as const }, + children: [], + }, + ], + }, + ], + calls, + ), + ); + + expect(calls).toEqual([ + { + path: "/workspaces/shop%2Fdemo/status?includeIdle=true", + init: undefined, + }, + ]); + expect(output).toContain("Deployment/web"); + expect(output).toContain("\u001b[42m"); + }); + + test("stop and restart use lifecycle actions with optional targeting", async () => { + const calls: Call[] = []; + const request = requestReturning({ deployments: ["web"] }, calls); + + await runStop("shop", request); + await runRestart("shop", undefined, request); + await runRestart("shop", "worker", request); + + expect(calls).toEqual([ + { + path: "/workspaces/shop/lifecycle", + init: { method: "POST", json: { action: "stop" } }, + }, + { + path: "/workspaces/shop/lifecycle", + init: { method: "POST", json: { action: "restart" } }, + }, + { + path: "/workspaces/shop/lifecycle", + init: { + method: "POST", + json: { action: "restart", services: ["worker"] }, + }, + }, + ]); + }); + + test("rollback and down delegate complete server-side operations", async () => { + const calls: Call[] = []; + await runRollback( + "shop", + "web", + 45_000, + requestReturning({ deployments: ["web"] }, calls), + ); + await runDown( + "shop", + true, + requestReturning({ full: true, retained: [], delete: [] }, calls), + ); + + expect(calls).toEqual([ + { + path: "/workspaces/shop/rollback", + init: { + method: "POST", + json: { services: ["web"], timeoutMs: 45_000 }, + }, + }, + { + path: "/workspaces/shop/down", + init: { method: "POST", json: { full: true } }, + }, + ]); + }); + + test("database and S3 credential requests send resolved local claims", async () => { + const calls: Call[] = []; + const databaseClaim = { + service: "web", + username: "app", + database: "app", + secretName: "postgres-app", + }; + const s3Claim = { service: "web", key: "assets", bucket: "assets" }; + + await getDatabaseCredentials( + "shop", + databaseClaim, + requestReturning({ username: "app", password: "secret" }, calls), + ); + await getRemoteS3Credentials( + "shop", + s3Claim, + requestReturning({ AWS_ACCESS_KEY_ID: "key" }, calls), + ); + + expect(calls).toEqual([ + { + path: "/workspaces/shop/databases/credentials", + init: { method: "POST", json: { claim: databaseClaim } }, + }, + { + path: "/workspaces/shop/storage/credentials", + init: { method: "POST", json: { claim: s3Claim } }, + }, + ]); + }); +}); diff --git a/tests/command/exec.test.ts b/tests/command/exec.test.ts new file mode 100644 index 0000000..350497d --- /dev/null +++ b/tests/command/exec.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, test } from "bun:test"; +import { runExec, type ExecSessionOpener } from "../../command/exec"; +import type { ExecApiSession, ExecOutputFrame } from "../../lib/exec-api"; + +function fakeSession(...frames: ExecOutputFrame[]): ExecApiSession { + return { + sendStdin() {}, + resize() {}, + close() {}, + async *[Symbol.asyncIterator]() { + yield* frames; + }, + }; +} + +describe("exec API command runner", () => { + test("opens the authenticated workspace endpoint and returns remote status", async () => { + let call: Parameters | undefined; + const opener: ExecSessionOpener = async (...args) => { + call = args; + return fakeSession({ type: "exit", exitCode: 23 }); + }; + const controller = new AbortController(); + const result = await runExec( + "shop/demo", + "api", + ["sh", "-c", "exit 23"], + controller.signal, + opener, + ); + + expect(result).toBe(23); + expect(call?.[0]).toBe("shop/demo"); + expect(call?.[1]).toMatchObject({ + deployment: "api", + command: ["sh", "-c", "exit 23"], + tty: false, + }); + expect(call?.[2]).toBe(controller.signal); + }); + + test("validates required arguments before opening a connection", async () => { + let calls = 0; + const opener: ExecSessionOpener = async () => { + calls += 1; + return fakeSession(); + }; + const signal = new AbortController().signal; + await expect(runExec("shop", "", ["sh"], signal, opener)).rejects.toThrow( + "Deployment name", + ); + await expect(runExec("shop", "api", [], signal, opener)).rejects.toThrow( + "Command is required", + ); + expect(calls).toBe(0); + }); +}); diff --git a/tests/command/logs.test.ts b/tests/command/logs.test.ts new file mode 100644 index 0000000..d99e61a --- /dev/null +++ b/tests/command/logs.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, test } from "bun:test"; +import { + runLogs, + type LogApiEvent, + type LogEventWriter, + type LogsApiStream, +} from "../../command/logs"; + +describe("logs API command runner", () => { + test("uses the authenticated workspace NDJSON route and preserves prefixes", async () => { + const calls: Array<{ + path: string; + signal?: AbortSignal; + timeout?: number; + }> = []; + const events: LogApiEvent[] = [ + { type: "heartbeat", timestamp: "now" }, + { + type: "log", + targetName: "web", + pod: "web-1", + container: "web", + message: "ready", + }, + { + type: "error", + pod: "web-1", + container: "web", + message: "disconnected", + retryable: true, + }, + ]; + const stream: LogsApiStream = async function* (path, init, options) { + calls.push({ + path, + signal: init?.signal ?? undefined, + timeout: options?.timeoutMs, + }); + yield* events as never[]; + }; + const output: Parameters[] = []; + const controller = new AbortController(); + await runLogs( + "shop/demo", + "web api", + true, + controller.signal, + stream, + (...entry) => output.push(entry), + ); + + expect(calls).toEqual([ + { + path: "/workspaces/shop%2Fdemo/logs?service=web+api&follow=true", + signal: controller.signal, + timeout: 0, + }, + ]); + expect(output).toEqual([ + ["web", "ready", false], + ["web api", "disconnected", true], + ]); + }); + + test("omits follow for finite collection and passes cancellation", async () => { + const controller = new AbortController(); + let receivedSignal: AbortSignal | null | undefined; + const stream: LogsApiStream = async function* (path, init, options) { + expect(path).toBe("/workspaces/shop/logs"); + expect(options?.timeoutMs).toBeUndefined(); + receivedSignal = init?.signal; + yield* []; + }; + await runLogs( + "shop", + undefined, + false, + controller.signal, + stream, + () => {}, + ); + expect(receivedSignal).toBe(controller.signal); + }); +}); diff --git a/tests/command/metadata.test.ts b/tests/command/metadata.test.ts index 00cec13..fc02a6a 100644 --- a/tests/command/metadata.test.ts +++ b/tests/command/metadata.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "bun:test"; +import { login, logout, whoami } from "../../command/auth"; import { db } from "../../command/db"; import { down } from "../../command/down"; import { exec } from "../../command/exec"; @@ -23,6 +24,9 @@ type Command = { describe("CLI command definitions", () => { const commands = [ ["up", "Create and start deployments", up], + ["login", "Log in to kuber.astrxl.dev", login], + ["logout", "Log out of kuber.astrxl.dev", logout], + ["whoami", "Show the current kuber user", whoami], ["start", "Start deployments without rebuilding images", start], ["stop", "Scale managed deployments to zero", stop], ["restart", "Roll out a restart for managed deployments", restart], @@ -30,7 +34,8 @@ describe("CLI command definitions", () => { ["s3", "Inspect managed S3 storage", s3], ["down", "Delete managed resources except ingress and PVCs", down], ["ps", "List deployments", ps], - ["logs", "Show deployment logs", logs], ["exec", "Execute a command inside a deployment pod", exec], + ["logs", "Show deployment logs", logs], + ["exec", "Execute a command inside a deployment pod", exec], ["export", "Write rendered manifests to a YAML file", exportCommand], ["db", "Inspect managed postgres databases", db], ] as const; @@ -74,6 +79,9 @@ describe("CLI command definitions", () => { expect((main as Command).args?.config).toMatchObject({ type: "string", }); + expect((login as Command).args?.persist).toMatchObject({ + type: "boolean", + }); }); test("defines both database inspection subcommands", () => { diff --git a/tests/command/up-api.test.ts b/tests/command/up-api.test.ts new file mode 100644 index 0000000..cb0e092 --- /dev/null +++ b/tests/command/up-api.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, test } from "bun:test"; +import type { ApiRequestInit } from "../../lib/api"; +import { KuberApiError } from "../../lib/api"; +import type { ApiRequester } from "../../lib/build"; +import { + ensureWorkspace, + reconcileResources, + workspaceAdoptionRoute, +} from "../../command/up"; +import { workspaceManifestDigest } from "../../lib/workspace"; + +const manifest = { version: 1 as const, files: [] }; +const snapshot = { + manifest, + digest: workspaceManifestDigest(manifest), + blobs: [], +}; + +describe("up API pipeline", () => { + test("creates workspace metadata without embedding source blobs", async () => { + const calls: Array<{ path: string; init?: ApiRequestInit }> = []; + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + calls.push({ path, init }); + if (!init) throw new KuberApiError("missing", 404); + return { + metadata: { name: "shop", uid: "uid", resourceVersion: "1" }, + } as T; + }; + await ensureWorkspace( + "shop", + { services: { web: { image: "nginx" } } }, + snapshot, + request, + ); + + expect(calls.map(({ path }) => path)).toEqual([ + "/workspaces/shop", + "/workspaces", + ]); + const body = calls[1]!.init?.json as Record; + expect(body).toMatchObject({ + id: "shop", + source: { uri: `cas://${snapshot.digest}`, digest: snapshot.digest }, + }); + expect(JSON.stringify(body)).not.toContain('"blob"'); + expect(JSON.stringify(body)).not.toContain('"files"'); + }); + + test("updates workspace metadata with the current resource-version ETag", async () => { + const calls: Array<{ path: string; init?: ApiRequestInit }> = []; + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + calls.push({ path, init }); + return { + metadata: { + name: "shop", + uid: "uid", + resourceVersion: init ? "8" : "7", + }, + } as T; + }; + await ensureWorkspace("shop", { services: {} }, snapshot, request); + expect(calls[1]?.init?.method).toBe("PUT"); + expect(new Headers(calls[1]?.init?.headers).get("if-match")).toBe('"7"'); + }); + + test("plans, applies, runs the hook, waits, then deletes stale identities", async () => { + const order: string[] = []; + const desired = [ + { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: "web" } }, + ]; + const stale = [ + { + apiVersion: "v1", + kind: "Secret", + name: "old", + uid: "secret-uid", + workspaceUid: "workspace-uid", + }, + ]; + const request: ApiRequester = async (path: string) => { + order.push(path.split("/").at(-1)!); + if (path.endsWith("/plan")) return { desired, stale } as T; + return {} as T; + }; + + await reconcileResources( + "shop", + desired, + 42_000, + () => { + order.push("hook"); + }, + request, + ); + expect(order).toEqual(["plan", "apply", "hook", "wait", "delete"]); + }); + + test("fails closed with the precise missing adoption route", async () => { + const request: ApiRequester = async (path: string) => { + if (path.endsWith("/plan")) + throw new Error( + "Namespace shop is external; refusing workspace mutation", + ); + return {} as T; + }; + await expect( + reconcileResources("shop", [], 1, undefined, request), + ).rejects.toThrow(`POST ${workspaceAdoptionRoute("shop")}`); + }); +}); diff --git a/tests/lib/api.test.ts b/tests/lib/api.test.ts new file mode 100644 index 0000000..201041d --- /dev/null +++ b/tests/lib/api.test.ts @@ -0,0 +1,270 @@ +import { afterEach, describe, expect, mock, test } from "bun:test"; +import { + DEFAULT_API_TIMEOUT_MS, + KuberApiError, + apiRequest, + apiStreamNdjson, + apiUpload, +} from "../../lib/api"; +import type { KuberSession } from "../../lib/session"; + +const originalFetch = globalThis.fetch; +const session: KuberSession = { + token: "secret-token", + expiresAt: "2099-01-01T00:00:00.000Z", + user: { username: "operator", roles: ["operator"] }, +}; + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +describe("authenticated API transport", () => { + test("sends authentication and serializes JSON", async () => { + const fetchMock = mock( + async (input: Parameters[0], init?: RequestInit) => { + expect(input).toBe("https://api.test/workspaces"); + expect(new Headers(init?.headers).get("authorization")).toBe( + "Bearer secret-token", + ); + expect(new Headers(init?.headers).get("content-type")).toBe( + "application/json", + ); + expect(init?.body).toBe('{"name":"demo"}'); + return Response.json({ id: "workspace-1" }); + }, + ); + globalThis.fetch = fetchMock as unknown as typeof fetch; + + await expect( + apiRequest<{ id: string }>( + "/workspaces", + { method: "POST", json: { name: "demo" } }, + { baseUrl: "https://api.test", session }, + ), + ).resolves.toEqual({ id: "workspace-1" }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + test("preserves unauthenticated callers and empty responses", async () => { + globalThis.fetch = mock(async (_input, init) => { + expect(new Headers(init?.headers).has("authorization")).toBe(false); + return new Response(null, { status: 204 }); + }) as unknown as typeof fetch; + + await expect( + apiRequest( + "/logout", + { method: "POST" }, + { + authenticated: false, + baseUrl: "https://api.test", + }, + ), + ).resolves.toBeUndefined(); + }); + + test("treats an empty successful JSON response as no content", async () => { + globalThis.fetch = mock( + async () => new Response("", { status: 200 }), + ) as unknown as typeof fetch; + + await expect( + apiRequest( + "/empty", + {}, + { + authenticated: false, + baseUrl: "https://api.test", + }, + ), + ).resolves.toBeUndefined(); + }); + + test("exposes typed problem details and correlation headers", async () => { + globalThis.fetch = mock(async () => + Response.json( + { + title: "Conflict", + status: 409, + detail: "Workspace already exists", + code: "WORKSPACE_EXISTS", + operationId: "op-7", + }, + { + status: 409, + headers: { "x-request-id": "req-4" }, + }, + ), + ) as unknown as typeof fetch; + + try { + await apiRequest( + "/workspaces", + {}, + { + authenticated: false, + baseUrl: "https://api.test", + }, + ); + throw new Error("expected request to fail"); + } catch (error) { + expect(error).toBeInstanceOf(KuberApiError); + expect(error).toMatchObject({ + message: "Workspace already exists", + status: 409, + code: "WORKSPACE_EXISTS", + requestId: "req-4", + operationId: "op-7", + }); + expect((error as KuberApiError).problem.title).toBe("Conflict"); + } + }); + + test("provides a stable fallback problem for non-JSON errors", async () => { + globalThis.fetch = mock( + async () => new Response("upstream failure", { status: 502 }), + ) as unknown as typeof fetch; + + await expect( + apiRequest( + "/status", + {}, + { + authenticated: false, + baseUrl: "https://api.test", + }, + ), + ).rejects.toMatchObject({ status: 502, code: "HTTP_502" }); + }); + + test("applies a finite timeout by default", async () => { + globalThis.fetch = mock(async (_input, init) => { + expect(DEFAULT_API_TIMEOUT_MS).toBeGreaterThan(0); + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + "abort", + () => reject(init.signal?.reason), + { + once: true, + }, + ); + }); + }) as unknown as typeof fetch; + + await expect( + apiRequest( + "/slow", + {}, + { + authenticated: false, + baseUrl: "https://api.test", + timeoutMs: 5, + }, + ), + ).rejects.toMatchObject({ name: "TimeoutError" }); + }); + + test("propagates caller aborts to fetch", async () => { + const controller = new AbortController(); + const reason = new DOMException("cancelled", "AbortError"); + globalThis.fetch = mock(async (_input, init) => { + if (init?.signal?.aborted) throw init.signal.reason; + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + "abort", + () => reject(init.signal?.reason), + { + once: true, + }, + ); + }); + }) as unknown as typeof fetch; + + const request = apiRequest( + "/operations/1", + { signal: controller.signal }, + { + authenticated: false, + baseUrl: "https://api.test", + }, + ); + controller.abort(reason); + await expect(request).rejects.toBe(reason); + }); + + test("uploads binary chunks without assigning a JSON content type", async () => { + const bytes = new Uint8Array([1, 2, 3]); + globalThis.fetch = mock(async (_input, init) => { + const headers = new Headers(init?.headers); + expect(init?.method).toBe("PATCH"); + expect(headers.get("content-type")).toBe("application/octet-stream"); + expect(headers.get("upload-offset")).toBe("12"); + expect(init?.body).toBe(bytes); + return Response.json({ + uploadId: "upload-1", + offset: 15, + complete: false, + }); + }) as unknown as typeof fetch; + + await expect( + apiUpload<{ offset: number }>("/uploads/upload-1", bytes, { + offset: 12, + authenticated: false, + baseUrl: "https://api.test", + }), + ).resolves.toMatchObject({ offset: 15 }); + }); + + test("consumes split NDJSON records incrementally", async () => { + let source: ReadableStreamDefaultController | undefined; + const stream = new ReadableStream({ + start(controller) { + source = controller; + controller.enqueue(new TextEncoder().encode('{"sequence":1}\n{"seq')); + }, + }); + globalThis.fetch = mock(async (_input, init) => { + expect(new Headers(init?.headers).get("accept")).toBe( + "application/x-ndjson", + ); + return new Response(stream); + }) as unknown as typeof fetch; + + const records = apiStreamNdjson<{ sequence: number }>( + "/operations/1/events", + {}, + { + authenticated: false, + baseUrl: "https://api.test", + }, + ); + const first = await records.next(); + expect(first.value).toEqual({ sequence: 1 }); + source?.enqueue(new TextEncoder().encode('uence":2}\r\n\n')); + source?.close(); + expect((await records.next()).value).toEqual({ sequence: 2 }); + expect((await records.next()).done).toBe(true); + }); + + test("rejects malformed NDJSON records", async () => { + globalThis.fetch = mock( + async () => new Response('{"ok":true}\nnot-json\n'), + ) as unknown as typeof fetch; + + const consume = async () => { + for await (const _record of apiStreamNdjson( + "/events", + {}, + { + authenticated: false, + baseUrl: "https://api.test", + }, + )) { + // Consume the complete stream. + } + }; + await expect(consume()).rejects.toBeInstanceOf(SyntaxError); + }); +}); diff --git a/tests/lib/apply.test.ts b/tests/lib/apply.test.ts index 35c91f3..367e3b0 100644 --- a/tests/lib/apply.test.ts +++ b/tests/lib/apply.test.ts @@ -19,6 +19,11 @@ describe("resource ordering", () => { resource("Secret"), resource("PersistentVolumeClaim"), resource("StorageClass"), + resource("RoleBinding"), + resource("ClusterRoleBinding"), + resource("Role"), + resource("ClusterRole"), + resource("ServiceAccount"), resource("Namespace"), resource("Ingress"), ]; @@ -26,6 +31,11 @@ describe("resource ordering", () => { "Namespace", "GarageBucket", "GarageKey", + "ServiceAccount", + "ClusterRole", + "Role", + "ClusterRoleBinding", + "RoleBinding", "StorageClass", "PersistentVolumeClaim", "Secret", diff --git a/tests/lib/build-api.test.ts b/tests/lib/build-api.test.ts new file mode 100644 index 0000000..d82034e --- /dev/null +++ b/tests/lib/build-api.test.ts @@ -0,0 +1,178 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ApiRequestInit } from "../../lib/api"; +import { + buildServices, + resolveBuildImages, + uploadWorkspaceSnapshot, + type ApiRequester, +} from "../../lib/build"; +import { + workspaceManifestDigest, + type WorkspaceSnapshot, +} from "../../lib/workspace"; +import type { BuildRequest } from "../../shared/build-protocol"; + +const directories: string[] = []; + +function emptySnapshot(): WorkspaceSnapshot { + const manifest = { version: 1 as const, files: [] }; + return { manifest, digest: workspaceManifestDigest(manifest), blobs: [] }; +} + +afterEach(async () => { + await Promise.all( + directories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +describe("authenticated build API pipeline", () => { + test("negotiates and uploads the manifest through resumable blob routes", async () => { + const snapshot = emptySnapshot(); + const calls: Array<{ path: string; init?: ApiRequestInit }> = []; + let negotiations = 0; + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + calls.push({ path, init }); + if (path === "/snapshots/negotiate") { + negotiations += 1; + return ( + negotiations === 1 + ? { + workspace: snapshot.digest, + missing: [snapshot.digest], + ready: false, + } + : { workspace: snapshot.digest, missing: [], ready: true } + ) as T; + } + if (init?.method === "POST" && !path.endsWith("/complete")) + return { offset: 0, complete: false } as T; + if (init?.method === "PATCH") + return { offset: (init.body as Uint8Array).byteLength } as T; + return { complete: true } as T; + }; + + await uploadWorkspaceSnapshot(snapshot, request); + + expect( + calls.map(({ path, init }) => [init?.method ?? "GET", path]), + ).toEqual([ + ["POST", "/snapshots/negotiate"], + ["POST", `/blobs/${encodeURIComponent(snapshot.digest)}/uploads`], + ["PATCH", `/blobs/${encodeURIComponent(snapshot.digest)}/uploads`], + [ + "POST", + `/blobs/${encodeURIComponent(snapshot.digest)}/uploads/complete`, + ], + ["POST", "/snapshots/negotiate"], + ]); + expect(new Headers(calls[2]!.init?.headers).get("upload-offset")).toBe("0"); + }); + + test("submits, reconciles, reports logs, and returns the server image reference", async () => { + const root = await mkdtemp(join(tmpdir(), "kuber-build-api-")); + directories.push(root); + const git = Bun.spawn(["git", "init", "-q", root]); + expect(await git.exited).toBe(0); + const snapshot = emptySnapshot(); + const calls: Array<{ path: string; init?: ApiRequestInit }> = []; + let submitted: BuildRequest | undefined; + const output: string[] = []; + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + calls.push({ path, init }); + if (path === "/snapshots/negotiate") + return { workspace: snapshot.digest, missing: [], ready: true } as T; + if (path === "/builds") { + submitted = init?.json as BuildRequest; + return { + version: 1, + id: submitted.id, + state: "queued", + createdAt: "2026-01-01T00:00:00Z", + } as T; + } + if (path.includes("/events")) + return [ + { + type: "log", + id: submitted!.id, + sequence: 1, + message: "build log\n", + }, + ] as T; + if (path.endsWith("/reconcile")) + return { + version: 1, + id: submitted!.id, + state: "succeeded", + createdAt: "2026-01-01T00:00:00Z", + digest: `sha256:${"a".repeat(64)}`, + } as T; + if (path.endsWith("/result")) + return { + image: "registry.server/kuber/shop-web", + digest: `sha256:${"a".repeat(64)}`, + reference: `registry.server/kuber/shop-web@sha256:${"a".repeat(64)}`, + } as T; + throw new Error(`Unexpected request ${path}`); + }; + + const result = await buildServices( + "shop", + { + services: { web: { build: { context: ".", args: { MODE: "prod" } } } }, + }, + root, + { + progress: (message) => { + output.push(message); + }, + }, + { request, snapshot, sleep: async () => {}, pollIntervalMs: 0 }, + ); + + expect(submitted?.spec).toMatchObject({ + architecture: "arm64", + context: ".", + buildArgs: ["MODE=prod"], + workspace: snapshot.digest, + }); + expect(result).toEqual({ + built: ["web"], + changed: ["web"], + images: { + web: `registry.server/kuber/shop-web@sha256:${"a".repeat(64)}`, + }, + }); + expect(output).toContain("build log"); + expect(calls.some(({ path }) => path.endsWith("/result"))).toBe(true); + }); + + test("no-build image resolution uses only the resolve route", async () => { + const calls: string[] = []; + const images = await resolveBuildImages( + "shop", + { services: { web: { build: "." }, cache: { image: "redis" } } }, + { + request: async (path: string, init?: ApiRequestInit) => { + calls.push(`${init?.method}:${path}:${JSON.stringify(init?.json)}`); + return { reference: "registry/web@sha256:immutable" } as T; + }, + }, + ); + expect(images).toEqual({ web: "registry/web@sha256:immutable" }); + expect(calls).toEqual([ + 'POST:/images/resolve:{"project":"shop","service":"web"}', + ]); + }); +}); diff --git a/tests/lib/config.test.ts b/tests/lib/config.test.ts index d1fbaf4..7bd3317 100644 --- a/tests/lib/config.test.ts +++ b/tests/lib/config.test.ts @@ -4,7 +4,6 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import type { V1Deployment } from "@kubernetes/client-node"; import { - DEFAULT_BUILDERS, DEFAULT_REGISTRY, DEFAULT_ROLLOUT_TIMEOUT_MS, extractConfigArgument, @@ -35,7 +34,6 @@ describe("kuber config", () => { expect(await loadConfig(cwd)).toMatchObject({ project: cwd.split("/").at(-1), registry: DEFAULT_REGISTRY, - builders: DEFAULT_BUILDERS, rolloutTimeoutMs: DEFAULT_ROLLOUT_TIMEOUT_MS, configFile: undefined, }); @@ -50,7 +48,6 @@ describe("kuber config", () => { project: "configured-project", composeFile: "deploy/compose.yml", registry: "registry.example.com/", - builders: { amd64: "user@amd", remoteRoot: "build-root" }, rolloutTimeoutMs: 42_000, async preBuild() {}, };`, @@ -61,11 +58,6 @@ describe("kuber config", () => { project: "configured-project", composeFile: join(cwd, "deploy", "compose.yml"), registry: "registry.example.com", - builders: { - amd64: "user@amd", - arm64: DEFAULT_BUILDERS.arm64, - remoteRoot: "build-root", - }, rolloutTimeoutMs: 42_000, configFile, }); diff --git a/tests/lib/convert-artifacts.test.ts b/tests/lib/convert-artifacts.test.ts new file mode 100644 index 0000000..834bc28 --- /dev/null +++ b/tests/lib/convert-artifacts.test.ts @@ -0,0 +1,191 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { composeToKubernetes, envFromToSecrets } from "../../lib/convert"; +import type { ComposeSpecification, Service } from "../../schema/docker.d"; +import { + BundleArtifactProvider, + LocalArtifactProvider, + createArtifactBundle, +} from "../../shared/artifacts"; + +const temporaryDirectories: string[] = []; + +async function temporaryDirectory(): Promise { + const path = await mkdtemp(join(tmpdir(), "kuber-artifacts-")); + temporaryDirectories.push(path); + return path; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +describe("conversion artifacts", () => { + test("bundle rendering matches the default local filesystem rendering", async () => { + const workspace = await temporaryDirectory(); + const env = "MODE=production\nTOKEN=a=b\n"; + const config = "enabled=true\n"; + await writeFile(join(workspace, ".env"), env); + await writeFile(join(workspace, "app.conf"), config); + + const compose = { + services: { + app: { + image: "app", + env_file: ".env", + volumes: ["./app.conf:/etc/app.conf:ro"], + }, + }, + } as ComposeSpecification; + + const local = await composeToKubernetes("project", compose, workspace); + const bundle = JSON.parse( + JSON.stringify(createArtifactBundle({ ".env": env, "app.conf": config })), + ); + const bundled = await composeToKubernetes( + "project", + compose, + workspace, + {}, + {}, + new BundleArtifactProvider(bundle), + ); + + expect(bundled).toEqual(local); + }); + + test("bundle providers preserve optional and required missing-file behavior", async () => { + const provider = new BundleArtifactProvider(createArtifactBundle({})); + + expect( + await envFromToSecrets( + "project", + "app", + { env_file: [{ path: "missing.env", required: false }] } as Service, + process.cwd(), + {}, + provider, + ), + ).toEqual([]); + await expect( + envFromToSecrets( + "project", + "app", + { env_file: "missing.env" } as Service, + process.cwd(), + {}, + provider, + ), + ).rejects.toThrow("Artifact not found"); + }); + + test("default local providers preserve env-file tilde path behavior", async () => { + const workspace = await temporaryDirectory(); + await mkdir(join(workspace, "~")); + await writeFile(join(workspace, "~", "legacy.env"), "LEGACY=yes\n"); + + const [secret] = await envFromToSecrets( + "project", + "app", + { env_file: "~/legacy.env" } as Service, + workspace, + ); + expect(secret?.stringData).toEqual({ LEGACY: "yes" }); + }); + + test("strict local providers reject traversal and absolute paths", async () => { + const workspace = await temporaryDirectory(); + const provider = new LocalArtifactProvider({ workspace, strict: true }); + + await expect( + envFromToSecrets( + "project", + "app", + { env_file: "../outside.env" } as Service, + workspace, + {}, + provider, + ), + ).rejects.toThrow("escapes the workspace"); + await expect( + envFromToSecrets( + "project", + "app", + { env_file: join(workspace, "absolute.env") } as Service, + workspace, + {}, + provider, + ), + ).rejects.toThrow("workspace-relative"); + }); + + test("strict local providers reject symlinks escaping the workspace", async () => { + const workspace = await temporaryDirectory(); + const outside = await temporaryDirectory(); + await writeFile(join(outside, "secret.env"), "TOKEN=secret\n"); + await symlink(join(outside, "secret.env"), join(workspace, "secret.env")); + + await expect( + envFromToSecrets( + "project", + "app", + { env_file: "secret.env" } as Service, + workspace, + {}, + new LocalArtifactProvider({ workspace, strict: true }), + ), + ).rejects.toThrow("escapes the workspace"); + }); + + test("strict local providers reject missing files below escaping symlinks", async () => { + const workspace = await temporaryDirectory(); + const outside = await temporaryDirectory(); + await symlink(outside, join(workspace, "outside")); + + await expect( + envFromToSecrets( + "project", + "app", + { + env_file: [{ path: "outside/missing.env", required: false }], + } as Service, + workspace, + {}, + new LocalArtifactProvider({ workspace, strict: true }), + ), + ).rejects.toThrow("escapes the workspace"); + }); + + test("bundle providers reject unsafe paths and byte or count excesses", () => { + expect( + () => + new BundleArtifactProvider(createArtifactBundle({ "../secret": "x" })), + ).toThrow("escapes the workspace"); + expect( + () => + new BundleArtifactProvider(createArtifactBundle({ config: "four" }), { + maxArtifactBytes: 3, + }), + ).toThrow("exceeds the 3 byte limit"); + expect( + () => + new BundleArtifactProvider( + createArtifactBundle({ first: "1", second: "2" }), + { maxArtifactCount: 1 }, + ), + ).toThrow("Artifact count exceeds the 1 limit"); + expect( + () => + new BundleArtifactProvider( + createArtifactBundle({ first: "12", second: "34" }), + { maxTotalBytes: 3 }, + ), + ).toThrow("Artifact bytes exceed the 3 byte limit"); + }); +}); diff --git a/tests/lib/exec-api.test.ts b/tests/lib/exec-api.test.ts new file mode 100644 index 0000000..c088554 --- /dev/null +++ b/tests/lib/exec-api.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, test } from "bun:test"; +import { openExecSession, type ExecWebSocket } from "../../lib/exec-api"; + +class FakeSocket implements ExecWebSocket { + binaryType: "arraybuffer" | "blob" = "blob"; + readyState = 0; + sent: string[] = []; + closes: Array<[number | undefined, string | undefined]> = []; + private listeners = new Map void>>(); + + send(data: string) { + this.sent.push(data); + } + + close(code?: number, reason?: string) { + this.closes.push([code, reason]); + this.readyState = 3; + } + + addEventListener(type: string, listener: (event: any) => void) { + let values = this.listeners.get(type); + if (!values) { + values = new Set(); + this.listeners.set(type, values); + } + values.add(listener); + } + + removeEventListener(type: string, listener: (event: any) => void) { + this.listeners.get(type)?.delete(listener); + } + + emit(type: string, event: any = {}) { + if (type === "open") this.readyState = 1; + for (const listener of [...(this.listeners.get(type) ?? [])]) + listener(event); + } +} + +const session = { + token: "secret-token", + expiresAt: "2099-01-01T00:00:00.000Z", + user: { username: "user", roles: [] }, +}; + +describe("exec WebSocket API", () => { + test("authenticates the upgrade and maps protocol byte frames", async () => { + const socket = new FakeSocket(); + let connection: + | { url: string; headers: Readonly> } + | undefined; + const opening = openExecSession( + "shop/demo", + { + deployment: "api", + command: ["sh", "-c", "echo ok"], + tty: true, + columns: 120, + rows: 40, + }, + new AbortController().signal, + { + baseUrl: "https://api.test/api/v2", + session, + socketFactory(url, headers) { + connection = { url, headers }; + return socket; + }, + }, + ); + socket.emit("open"); + const client = await opening; + + expect(connection).toEqual({ + url: "wss://api.test/api/v2/workspaces/shop%2Fdemo/exec", + headers: { + authorization: "Bearer secret-token", + }, + }); + expect(JSON.parse(socket.sent[0]!)).toEqual({ + type: "start", + version: 1, + deployment: "api", + command: ["sh", "-c", "echo ok"], + tty: true, + columns: 120, + rows: 40, + }); + + client.sendStdin(new TextEncoder().encode("hello"), true); + client.resize(80, 24); + + const iterator = client[Symbol.asyncIterator](); + socket.emit("message", { + data: JSON.stringify({ + type: "stdout", + data: "b3V0", + encoding: "base64", + }), + }); + expect(await iterator.next()).toEqual({ + done: false, + value: { type: "stdout", data: new TextEncoder().encode("out") }, + }); + + expect(JSON.parse(socket.sent[1]!)).toEqual({ + type: "stdin", + data: "aGVsbG8=", + encoding: "base64", + eof: true, + }); + expect(JSON.parse(socket.sent[2]!)).toEqual({ + type: "resize", + columns: 80, + rows: 24, + }); + + socket.emit("message", { + data: JSON.stringify({ type: "exit", exitCode: 7 }), + }); + expect((await iterator.next()).value).toEqual({ + type: "exit", + exitCode: 7, + }); + socket.emit("close"); + expect((await iterator.next()).done).toBe(true); + }); + + test("closes an active connection when cancelled", async () => { + const socket = new FakeSocket(); + const controller = new AbortController(); + const opening = openExecSession( + "shop", + { deployment: "api", command: ["sh"], tty: false }, + controller.signal, + { session, socketFactory: () => socket }, + ); + socket.emit("open"); + const client = await opening; + const next = client[Symbol.asyncIterator]().next(); + controller.abort(); + expect((await next).done).toBe(true); + expect(socket.closes).toContainEqual([1000, "aborted"]); + }); + + test("rejects malformed server frames and closes with protocol error", async () => { + const socket = new FakeSocket(); + const opening = openExecSession( + "shop", + { deployment: "api", command: ["sh"], tty: false }, + new AbortController().signal, + { session, socketFactory: () => socket }, + ); + socket.emit("open"); + const client = await opening; + const next = client[Symbol.asyncIterator]().next(); + socket.emit("message", { + data: JSON.stringify({ type: "stdout", data: 1 }), + }); + await expect(next).rejects.toThrow("Invalid exec output frame"); + expect(socket.closes).toContainEqual([1002, "invalid frame"]); + }); +}); diff --git a/tests/lib/render-api.test.ts b/tests/lib/render-api.test.ts new file mode 100644 index 0000000..42deb6b --- /dev/null +++ b/tests/lib/render-api.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, test } from "bun:test"; +import { renderResources } from "../../lib/render"; + +describe("side-effect-free export rendering", () => { + test("resolves build images and renders locally without provider calls", async () => { + const paths: string[] = []; + const resources = await renderResources( + "shop", + { services: { web: { build: "." } } }, + process.cwd(), + { + request: async (path: string) => { + paths.push(path); + return { reference: "registry/web@sha256:immutable" } as T; + }, + }, + ); + const deployment = resources.find( + (resource) => resource.kind === "Deployment", + ); + expect(paths).toEqual(["/images/resolve"]); + expect((deployment as any)?.spec?.template.spec.containers[0].image).toBe( + "registry/web@sha256:immutable", + ); + }); + + test("clearly rejects exports requiring generated credentials", async () => { + await expect( + renderResources("shop", { + services: { web: { image: "app", volumes: ["postgresql:app"] } }, + }), + ).rejects.toThrow("Export is side-effect-free"); + }); +}); diff --git a/tests/lib/session.test.ts b/tests/lib/session.test.ts new file mode 100644 index 0000000..c70f89c --- /dev/null +++ b/tests/lib/session.test.ts @@ -0,0 +1,71 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + getSessionPath, + readSession, + removeSessions, + writeSession, + type KuberSession, +} from "../../lib/session"; + +const originalRuntimeDirectory = process.env.XDG_RUNTIME_DIR; +const originalConfigDirectory = process.env.XDG_CONFIG_HOME; +const directories: string[] = []; + +async function setupDirectories(): Promise { + const root = await mkdtemp(join(tmpdir(), "kuber-session-test-")); + directories.push(root); + process.env.XDG_RUNTIME_DIR = join(root, "runtime"); + process.env.XDG_CONFIG_HOME = join(root, "config"); +} + +afterEach(async () => { + if (originalRuntimeDirectory === undefined) + delete process.env.XDG_RUNTIME_DIR; + else process.env.XDG_RUNTIME_DIR = originalRuntimeDirectory; + if (originalConfigDirectory === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = originalConfigDirectory; + await Promise.all( + directories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +describe("API sessions", () => { + const session: KuberSession = { + token: "token", + expiresAt: "2999-01-01T00:00:00.000Z", + user: { username: "dmgnr", roles: ["admin"] }, + }; + + test("writes runtime sessions with private permissions", async () => { + await setupDirectories(); + const path = await writeSession(session, false); + expect(path).toBe(getSessionPath(false)); + expect((await stat(path)).mode & 0o777).toBe(0o600); + expect(JSON.parse(await readFile(path, "utf8"))).toEqual(session); + expect(await readSession()).toEqual(session); + }); + + test("selects persistent storage and removes the other session", async () => { + await setupDirectories(); + await writeSession(session, false); + const path = await writeSession(session, true); + expect(path).toBe(getSessionPath(true)); + expect(await readSession()).toEqual(session); + await removeSessions(); + expect(await readSession()).toBeUndefined(); + }); + + test("discards expired sessions", async () => { + await setupDirectories(); + await writeSession( + { ...session, expiresAt: "2000-01-01T00:00:00.000Z" }, + false, + ); + expect(await readSession()).toBeUndefined(); + }); +}); diff --git a/tests/lib/workspace.test.ts b/tests/lib/workspace.test.ts new file mode 100644 index 0000000..03f0de5 --- /dev/null +++ b/tests/lib/workspace.test.ts @@ -0,0 +1,192 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { execFile } from "node:child_process"; +import { + mkdtemp, + readFile, + readlink, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { + enumerateWorkspace, + materializeWorkspace, + serializeWorkspaceManifest, + validateWorkspaceManifest, + validateWorkspacePath, + workspaceManifestDigest, +} from "../../lib/workspace"; +import { + BUILD_PROTOCOL_VERSION, + type WorkspaceManifest, +} from "../../shared/build-protocol"; + +const run = promisify(execFile); +const temporaryDirectories: string[] = []; + +async function temporaryDirectory(prefix: string): Promise { + const path = await mkdtemp(join(tmpdir(), prefix)); + temporaryDirectories.push(path); + return path; +} + +async function repository(): Promise { + const root = await temporaryDirectory("kuber-workspace-"); + await run("git", ["init", "-q", root]); + await run("git", ["-C", root, "config", "user.email", "test@example.com"]); + await run("git", ["-C", root, "config", "user.name", "Test"]); + return root; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +describe("workspace snapshots", () => { + test("captures working tracked, untracked, and ignored dotenv files deterministically", async () => { + const root = await repository(); + await writeFile(join(root, ".gitignore"), "ignored*\n.env*\nsub/.env*\n"); + await writeFile(join(root, "tracked.txt"), "committed"); + await writeFile(join(root, "script.sh"), "#!/bin/sh\n"); + await run("chmod", ["755", join(root, "script.sh")]); + await run("git", [ + "-C", + root, + "add", + ".gitignore", + "tracked.txt", + "script.sh", + ]); + await run("git", ["-C", root, "commit", "-qm", "initial"]); + + await writeFile(join(root, "tracked.txt"), "working tree"); + await writeFile(join(root, "untracked.txt"), "untracked"); + await writeFile(join(root, "ignored.bin"), "excluded"); + await writeFile(join(root, ".env.local"), "SECRET=root"); + await run("mkdir", [join(root, "sub")]); + await writeFile(join(root, "sub/.env.test"), "SECRET=sub"); + await run("ln", ["-s", "tracked.txt", join(root, "link")]); + + const first = await enumerateWorkspace(root); + const second = await enumerateWorkspace(root); + expect(first).toEqual(second); + expect(first.manifest.files.map((file) => file.path)).toEqual([ + ".env.local", + ".gitignore", + "link", + "script.sh", + "sub/.env.test", + "tracked.txt", + "untracked.txt", + ]); + expect( + first.manifest.files.find((file) => file.path === "script.sh")?.mode, + ).toBe(0o755); + expect( + first.manifest.files.find((file) => file.path === "link")?.type, + ).toBe("symlink"); + expect( + first.manifest.files.some((file) => file.path === "ignored.bin"), + ).toBe(false); + + const destination = join( + await temporaryDirectory("kuber-materialized-parent-"), + "tree", + ); + const blobs = new Map(first.blobs.map((blob) => [blob.digest, blob.data])); + await materializeWorkspace(destination, first.manifest, async (digest) => + blobs.get(digest)!, + ); + expect(await readFile(join(destination, "tracked.txt"), "utf8")).toBe( + "working tree", + ); + expect(await readFile(join(destination, ".env.local"), "utf8")).toBe( + "SECRET=root", + ); + expect(await readlink(join(destination, "link"))).toBe("tracked.txt"); + expect((await stat(join(destination, "script.sh"))).mode & 0o777).toBe( + 0o755, + ); + }); + + test("omits tracked files deleted in the worktree", async () => { + const root = await repository(); + await writeFile(join(root, "deleted"), "value"); + await run("git", ["-C", root, "add", "deleted"]); + await run("git", ["-C", root, "commit", "-qm", "initial"]); + await rm(join(root, "deleted")); + expect((await enumerateWorkspace(root)).manifest.files).toEqual([]); + }); + + test("rejects escaping symlinks and special files", async () => { + const symlinkRoot = await repository(); + await run("ln", ["-s", "../outside", join(symlinkRoot, "escape")]); + await expect(enumerateWorkspace(symlinkRoot)).rejects.toThrow( + "Symlink escapes workspace", + ); + + const specialRoot = await repository(); + await run("mkfifo", [join(specialRoot, "pipe")]); + await expect(enumerateWorkspace(specialRoot)).rejects.toThrow( + "Special files", + ); + }); + + test("rejects traversal, unsorted manifests, ancestor collisions, and corrupt blobs", async () => { + expect(() => validateWorkspacePath("../secret")).toThrow( + "Unsafe workspace path", + ); + const digest = `sha256:${"a".repeat(64)}` as const; + const unsorted: WorkspaceManifest = { + version: BUILD_PROTOCOL_VERSION, + files: [ + { path: "b", type: "file", digest, size: 0, mode: 0o644 }, + { path: "a", type: "file", digest, size: 0, mode: 0o644 }, + ], + }; + expect(() => validateWorkspaceManifest(unsorted)).toThrow( + "bytewise sorted", + ); + const canonical = { + version: BUILD_PROTOCOL_VERSION, + files: [{ path: "a", type: "file", digest, size: 0, mode: 0o644 }], + } satisfies WorkspaceManifest; + const reordered = JSON.parse( + `{"files":[{"mode":420,"size":0,"digest":"${digest}","type":"file","path":"a"}],"version":1}`, + ) as WorkspaceManifest; + expect(workspaceManifestDigest(reordered)).toBe( + workspaceManifestDigest(canonical), + ); + expect( + JSON.parse(Buffer.from(serializeWorkspaceManifest(reordered)).toString()), + ).toEqual(canonical); + expect(() => + validateWorkspaceManifest({ + version: BUILD_PROTOCOL_VERSION, + files: [ + { path: "a", type: "symlink", digest, size: 0, mode: 0o777 }, + { path: "a/b", type: "file", digest, size: 0, mode: 0o644 }, + ], + }), + ).toThrow("used as a directory"); + + const parent = await temporaryDirectory("kuber-materialized-invalid-"); + await expect( + materializeWorkspace( + join(parent, "tree"), + { + version: BUILD_PROTOCOL_VERSION, + files: [{ path: "file", type: "file", digest, size: 1, mode: 0o644 }], + }, + async () => Buffer.from("wrong"), + ), + ).rejects.toThrow("Blob verification failed"); + }); +}); diff --git a/tests/server/app.test.ts b/tests/server/app.test.ts new file mode 100644 index 0000000..59c6491 --- /dev/null +++ b/tests/server/app.test.ts @@ -0,0 +1,471 @@ +import { describe, expect, spyOn, test } from "bun:test"; +import { cleanupExpiredSessions, createApp } from "../../server/app"; +import { hashToken, MemoryAuthStore } from "../../server/auth"; +import { MemoryAuditStore } from "../../server/audit-store"; +import type { ManagementService } from "../../server/management"; +import { + MemoryOperationStore, + MemoryWorkspaceLeaseProvider, +} from "../../server/operation-store"; +import { MemoryWorkspaceStore } from "../../server/workspace-store"; + +function request( + path: string, + init: RequestInit = {}, + token?: string, +): Request { + const headers = new Headers(init.headers); + if (token) headers.set("authorization", `Bearer ${token}`); + return new Request(`https://kuber.astrxl.dev${path}`, { ...init, headers }); +} + +describe("kuber API authentication", () => { + test("logs in, resolves identity, and revokes the session", async () => { + const store = new MemoryAuthStore(); + await store.putUser({ + username: "dmgnr", + passwordHash: "stored-hash", + roles: ["admin"], + }); + const app = createApp({ + store, + now: () => Date.parse("2026-09-02T00:00:00.000Z"), + verifyPassword: async (password, hash) => + password === "correct" && hash === "stored-hash", + }); + + const login = await app( + request("/api/v2/login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + username: "dmgnr", + password: "correct", + persistent: false, + }), + }), + ); + expect(login.status).toBe(200); + const session = (await login.json()) as { + token: string; + expiresAt: string; + }; + expect(session.expiresAt).toBe("2026-09-03T00:00:00.000Z"); + + const me = await app(request("/api/v2/me", {}, session.token)); + expect(await me.json()).toEqual({ username: "dmgnr", roles: ["admin"] }); + + const logout = await app( + request("/api/v2/logout", { method: "POST" }, session.token), + ); + expect(logout.status).toBe(204); + expect((await app(request("/api/v2/me", {}, session.token))).status).toBe( + 401, + ); + }); + + test("rejects invalid credentials and unauthenticated requests", async () => { + const store = new MemoryAuthStore(); + await store.putUser({ + username: "dmgnr", + passwordHash: "stored-hash", + roles: ["admin"], + }); + const app = createApp({ + store, + verifyPassword: async () => false, + }); + + const login = await app( + request("/api/v2/login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ username: "dmgnr", password: "wrong" }), + }), + ); + expect(login.status).toBe(401); + expect((await app(request("/api/v2/me"))).status).toBe(401); + }); + + test("provides an unauthenticated health endpoint", async () => { + const app = createApp({ store: new MemoryAuthStore() }); + const response = await app(request("/api/v2/health")); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ status: "ok" }); + }); + + test("provides an explicit expired-session startup cleanup helper", async () => { + const store = new MemoryAuthStore(); + await store.putUser({ + username: "user", + passwordHash: "hash", + roles: ["viewer"], + }); + await store.putSession({ + tokenHash: hashToken("expired"), + username: "user", + authVersion: 1, + expiresAt: "2026-09-01T00:00:00.000Z", + }); + expect( + await cleanupExpiredSessions( + store, + Date.parse("2026-09-02T00:00:00.000Z"), + ), + ).toBe(1); + }); + + test("rate limits repeated failed logins", async () => { + const app = createApp({ + store: new MemoryAuthStore(), + now: () => 0, + }); + const login = () => + app( + request("/api/v2/login", { + method: "POST", + body: JSON.stringify({ username: "missing", password: "wrong" }), + }), + ); + + for (let attempt = 0; attempt < 5; attempt += 1) { + expect((await login()).status).toBe(401); + } + const limited = await login(); + expect(limited.status).toBe(429); + expect(limited.headers.get("retry-after")).toBe("300"); + }); +}); + +async function authenticatedStore(role: "viewer" | "operator" | "admin") { + const store = new MemoryAuthStore(); + await store.putUser({ username: role, passwordHash: "hash", roles: [role] }); + await store.putSession({ + tokenHash: hashToken("token"), + username: role, + authVersion: 1, + expiresAt: "2030-01-01T00:00:00.000Z", + }); + return store; +} + +describe("kuber v2 HTTP routes", () => { + test("uses exact origins, request IDs, and problem+json errors", async () => { + const app = createApp({ + store: new MemoryAuthStore(), + allowedOrigins: ["https://console.example"], + requestId: () => "generated-id", + }); + const denied = await app( + request("/api/v2/health", { + headers: { origin: "https://console.example.evil" }, + }), + ); + expect(denied.status).toBe(403); + expect(denied.headers.get("content-type")).toContain( + "application/problem+json", + ); + expect(denied.headers.get("x-request-id")).toBe("generated-id"); + expect(await denied.json()).toMatchObject({ + code: "ORIGIN_NOT_ALLOWED", + requestId: "generated-id", + }); + + const allowed = await app( + request("/api/v2/health", { + headers: { + origin: "https://console.example", + "x-request-id": "caller-id", + }, + }), + ); + expect(allowed.headers.get("access-control-allow-origin")).toBe( + "https://console.example", + ); + expect(allowed.headers.get("x-request-id")).toBe("caller-id"); + }); + + test("enforces capabilities and supports user CRUD with revocation", async () => { + const viewerStore = await authenticatedStore("viewer"); + const viewerApp = createApp({ store: viewerStore }); + expect( + (await viewerApp(request("/api/v2/users", {}, "token"))).status, + ).toBe(403); + + const store = await authenticatedStore("admin"); + const auditStore = new MemoryAuditStore(); + const app = createApp({ + store, + auditStore, + hashPassword: async (password) => `hashed:${password}`, + }); + const created = await app( + request( + "/api/v2/users", + { + method: "POST", + body: JSON.stringify({ + username: "alice", + password: "secret", + roles: ["operator"], + }), + }, + "token", + ), + ); + expect(created.status).toBe(201); + expect(await created.json()).toEqual({ + username: "alice", + roles: ["operator"], + disabled: false, + }); + expect((await store.getUser("alice"))?.passwordHash).toBe("hashed:secret"); + + const revoke = await app( + request( + "/api/v2/users/alice/sessions/revoke", + { method: "POST" }, + "token", + ), + ); + expect(revoke.status).toBe(200); + expect(await revoke.json()).toEqual({ username: "alice", revoked: 0 }); + expect((await auditStore.list()).map((event) => event.spec.action)).toEqual( + ["user.create", "sessions.revoke"], + ); + }); + + test("provides workspace ETags and idempotent synchronous operations", async () => { + const store = await authenticatedStore("operator"); + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + now: () => new Date("2026-09-02T00:00:00.000Z"), + }); + const operationStore = new MemoryOperationStore( + () => new Date("2026-09-02T00:00:00.000Z"), + () => "operation-uid", + ); + let stops = 0; + const management = { + stop: async () => { + stops += 1; + return ["api"]; + }, + } as unknown as ManagementService; + const app = createApp({ + store, + workspaceStore, + operationStore, + management, + }); + + const created = await app( + request( + "/api/v2/workspaces", + { + method: "POST", + body: JSON.stringify({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }), + }, + "token", + ), + ); + expect(created.status).toBe(201); + expect(created.headers.get("etag")).toBe('"1"'); + + const missingPrecondition = await app( + request( + "/api/v2/workspaces/demo", + { + method: "PUT", + body: JSON.stringify({ + source: { uri: "oci://example/demo", digest: "sha256:def" }, + }), + }, + "token", + ), + ); + expect(missingPrecondition.status).toBe(428); + + const stop = () => + app( + request( + "/api/v2/workspaces/demo/lifecycle", + { + method: "POST", + headers: { "idempotency-key": "stop-once" }, + body: JSON.stringify({ action: "stop", services: ["api"] }), + }, + "token", + ), + ); + expect((await stop()).status).toBe(200); + expect((await stop()).status).toBe(200); + expect(stops).toBe(1); + expect(await operationStore.list("demo")).toHaveLength(1); + }); + + test("rejects JSON bodies over the configured limit", async () => { + const app = createApp({ + store: await authenticatedStore("admin"), + workspaceStore: new MemoryWorkspaceStore(), + jsonBodyLimit: 32, + }); + const result = await app( + request( + "/api/v2/workspaces", + { method: "POST", body: JSON.stringify({ value: "x".repeat(64) }) }, + "token", + ), + ); + expect(result.status).toBe(413); + expect(await result.json()).toMatchObject({ code: "BODY_TOO_LARGE" }); + }); + + test("does not expose request internals or corrupt success when auditing fails", async () => { + const reported = spyOn(console, "error").mockImplementation(() => {}); + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const operationStore = new MemoryOperationStore( + undefined, + () => "operation-uid", + ); + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore, + management: { stop: async () => ["api"] } as unknown as ManagementService, + auditStore: { + append: async () => { + throw new Error("audit unavailable"); + }, + list: async () => [], + }, + }); + const result = await app( + request( + "/api/v2/workspaces/demo/lifecycle", + { + method: "POST", + headers: { "idempotency-key": "private-request" }, + body: JSON.stringify({ action: "stop", password: "do-not-store" }), + }, + "token", + ), + ); + expect(result.status).toBe(200); + const body = (await result.json()) as Record; + expect(body.operation.spec).toEqual({ + workspaceId: "demo", + action: "workspace.stop", + }); + expect(body.operation.status.state).toBe("succeeded"); + expect(reported).toHaveBeenCalledTimes(1); + reported.mockRestore(); + expect( + await operationStore.get("operation-operation-uid"), + ).not.toHaveProperty("spec.request"); + }); + + test("fails and identifies operations that cannot acquire the workspace lease", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const operationStore = new MemoryOperationStore(undefined, () => "blocked"); + const leases = new MemoryWorkspaceLeaseProvider(); + await leases.acquire("demo", "other"); + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore, + leases, + management: { stop: async () => [] } as unknown as ManagementService, + }); + const result = await app( + request( + "/api/v2/workspaces/demo/lifecycle", + { method: "POST", body: JSON.stringify({ action: "stop" }) }, + "token", + ), + ); + expect(result.status).toBe(409); + expect(await result.json()).toMatchObject({ + code: "WORKSPACE_BUSY", + operationId: "operation-blocked", + }); + expect((await operationStore.get("operation-blocked"))?.status.state).toBe( + "failed", + ); + }); + + test("routes workspace adoption and keeps platform adoption admin-only", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const adopted: string[] = []; + const adoption = { + adopt: async (workspaceId: string, workspaceUid: string) => { + adopted.push(`${workspaceId}:${workspaceUid}`); + return { workspaceId, workspaceUid, resourcesAdopted: 2 }; + }, + adoptPlatform: async (workspaceUid: string) => ({ + workspaceId: "kuber-system", + workspaceUid, + resourcesAdopted: 1, + }), + }; + const operatorApp = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + adoption, + }); + const regular = await operatorApp( + request("/api/v2/workspaces/demo/adopt", { method: "POST" }, "token"), + ); + expect(regular.status).toBe(200); + expect(adopted).toEqual(["demo:workspace-uid"]); + expect( + ( + await operatorApp( + request( + "/api/v2/platform/kuber-system/adopt", + { + method: "POST", + body: JSON.stringify({ workspaceUid: "platform" }), + }, + "token", + ), + ) + ).status, + ).toBe(403); + + const adminApp = createApp({ + store: await authenticatedStore("admin"), + workspaceStore, + adoption, + }); + const platform = await adminApp( + request( + "/api/v2/platform/kuber-system/adopt", + { method: "POST", body: JSON.stringify({ workspaceUid: "platform" }) }, + "token", + ), + ); + expect(platform.status).toBe(200); + }); +}); diff --git a/tests/server/audit-store.test.ts b/tests/server/audit-store.test.ts new file mode 100644 index 0000000..42a6212 --- /dev/null +++ b/tests/server/audit-store.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, test } from "bun:test"; +import { + AUDIT_REDACTED, + MemoryAuditStore, + redactAuditValue, +} from "../../server/audit-store"; + +describe("audit store", () => { + test("recursively redacts secrets without changing the input", async () => { + const details = { + authorization: "Bearer visible-before-redaction", + nested: [{ password: "hunter2", note: "safe" }], + header: "Bearer another-secret", + }; + const store = new MemoryAuditStore(); + const event = await store.append({ + actor: { username: "admin" }, + action: "workspace.update", + workspaceId: "demo", + outcome: "success", + details, + }); + expect(event.spec.details).toEqual({ + authorization: AUDIT_REDACTED, + nested: [{ password: AUDIT_REDACTED, note: "safe" }], + header: AUDIT_REDACTED, + }); + expect(details.nested[0]?.password).toBe("hunter2"); + }); + + test("is append-only and returns defensive copies", async () => { + const store = new MemoryAuditStore(); + const event = await store.append({ + actor: { username: "admin" }, + action: "workspace.create", + workspaceId: "demo", + outcome: "success", + }); + event.spec.action = "tampered"; + expect((await store.list("demo"))[0]?.spec.action).toBe("workspace.create"); + expect(redactAuditValue({ api_key: "key", ordinary: "value" })).toEqual({ + api_key: AUDIT_REDACTED, + ordinary: "value", + }); + }); + + test("redacts credential-bearing URL strings at value level", () => { + expect( + redactAuditValue("git clone https://alice:s3cret@github.com/org/repo.git"), + ).toBe("git clone https://[REDACTED]@github.com/org/repo.git"); + }); + + test("redacts AWS access key IDs at value level", () => { + expect( + redactAuditValue("connection used AKIAIOSFODNN7EXAMPLE to attach volume"), + ).toBe("connection used [REDACTED] to attach volume"); + }); + + test("redacts OpenSSH / private key headers at value level", () => { + expect( + redactAuditValue( + "ssh key\n-----BEGIN OPENSSH PRIVATE KEY-----\nabc123\n-----END OPENSSH PRIVATE KEY-----", + ), + ).toBe( + "ssh key\n[REDACTED]\nabc123\n-----END OPENSSH PRIVATE KEY-----", + ); + }); + + test("preserves ordinary URLs and arbitrary identifiers", () => { + expect( + redactAuditValue("deploy from https://registry.example.com/v2/app"), + ).toBe("deploy from https://registry.example.com/v2/app"); + expect( + redactAuditValue("user alice@example.com recovered the AKIA-referencing doc"), + ).toBe("user alice@example.com recovered the AKIA-referencing doc"); + expect(redactAuditValue("id abc-123-def")).toBe("id abc-123-def"); + }); +}); diff --git a/tests/server/auth.test.ts b/tests/server/auth.test.ts new file mode 100644 index 0000000..83afe79 --- /dev/null +++ b/tests/server/auth.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, test } from "bun:test"; +import { + MemoryAuthStore, + hashToken, + tokenHashesEqual, +} from "../../server/auth"; +import { + CAPABILITIES, + capabilitiesForRoles, + hasCapability, +} from "../../server/authorization"; + +const expiresAt = "2026-09-03T00:00:00.000Z"; + +describe("authorization", () => { + test("grants named capabilities through deny-by-default roles", () => { + expect([...capabilitiesForRoles(["viewer"])]).toEqual(["kubernetes:read"]); + expect(hasCapability(["operator"], "kubernetes:write")).toBe(true); + expect(hasCapability(["operator"], "users:read")).toBe(false); + expect([...capabilitiesForRoles(["unknown"])]).toEqual([]); + expect([...capabilitiesForRoles(["admin"])]).toEqual([...CAPABILITIES]); + }); +}); + +describe("memory auth store", () => { + test("creates, lists, updates, and deletes users", async () => { + const store = new MemoryAuthStore(); + const bob = await store.createUser({ + username: "bob", + passwordHash: "hash-b", + roles: ["viewer"], + }); + await store.createUser({ + username: "alice", + passwordHash: "hash-a", + roles: ["operator"], + }); + + expect(bob.authVersion).toBe(1); + expect((await store.listUsers()).map((user) => user.username)).toEqual([ + "alice", + "bob", + ]); + await expect(store.createUser({ ...bob })).rejects.toThrow( + "User already exists", + ); + + const updated = await store.updateUser("bob", { + roles: ["admin"], + disabled: true, + }); + expect(updated).toMatchObject({ + roles: ["admin"], + disabled: true, + authVersion: 2, + }); + expect(await store.updateUser("missing", {})).toBeUndefined(); + expect(await store.deleteUser("bob")).toBe(true); + expect(await store.deleteUser("bob")).toBe(false); + }); + + test("references user authVersion and ignores legacy role snapshots", async () => { + const store = new MemoryAuthStore(); + await store.putUser({ + username: "alice", + passwordHash: "hash", + roles: ["viewer"], + }); + const tokenHash = hashToken("token"); + await store.putSession({ + tokenHash, + username: "alice", + roles: ["admin"], + expiresAt, + }); + + expect(store.sessions.get(tokenHash)).toEqual({ + tokenHash, + username: "alice", + authVersion: 1, + expiresAt, + }); + await store.updateUser("alice", { roles: ["operator"] }); + expect(await store.getSession(tokenHash)).toBeUndefined(); + await expect( + store.putSession({ + tokenHash: hashToken("stale"), + username: "alice", + authVersion: 1, + expiresAt, + }), + ).rejects.toThrow("stale"); + }); + + test("revokes user sessions and cleans up expired sessions", async () => { + const store = new MemoryAuthStore(); + for (const username of ["alice", "bob"]) { + await store.putUser({ + username, + passwordHash: "hash", + roles: ["viewer"], + }); + } + await store.putSession({ + tokenHash: hashToken("alice-expired"), + username: "alice", + authVersion: 1, + expiresAt: "2026-09-01T00:00:00.000Z", + }); + await store.putSession({ + tokenHash: hashToken("alice-active"), + username: "alice", + authVersion: 1, + expiresAt, + }); + await store.putSession({ + tokenHash: hashToken("bob-expired"), + username: "bob", + authVersion: 1, + expiresAt: "2026-09-01T00:00:00.000Z", + }); + + expect( + await store.listExpiredSessions(Date.parse("2026-09-02T00:00:00.000Z")), + ).toHaveLength(2); + expect( + await store.deleteExpiredSessions(Date.parse("2026-09-02T00:00:00.000Z")), + ).toBe(2); + expect(await store.revokeUserSessions("alice")).toBe(1); + expect(store.sessions.size).toBe(0); + }); + + test("rejects invalid domain records and malformed token hashes", async () => { + const store = new MemoryAuthStore(); + await expect( + store.putUser({ + username: " alice", + passwordHash: "hash", + roles: ["viewer"], + }), + ).rejects.toThrow("Username"); + await expect( + store.putUser({ + username: "alice", + passwordHash: "hash", + roles: ["root"] as never, + }), + ).rejects.toThrow("valid role"); + expect(tokenHashesEqual("zz", "zz")).toBe(false); + }); +}); diff --git a/tests/server/build-controller.test.ts b/tests/server/build-controller.test.ts new file mode 100644 index 0000000..ad55229 --- /dev/null +++ b/tests/server/build-controller.test.ts @@ -0,0 +1,483 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { lstat, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + BuildConflictError, + BuildController, + BuildValidationError, + type BuildJobObservation, + type BuildKubernetesOperations, +} from "../../server/build-controller"; +import { MemoryBuildStore } from "../../server/build-store"; +import { FilesystemCas } from "../../server/cas"; +import type { KubernetesJob } from "../../server/build-job"; +import { + BUILD_PROTOCOL_VERSION, + type BuildRequest, + type Sha256Digest, + type WorkspaceManifest, +} from "../../shared/build-protocol"; + +const roots: string[] = []; +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +function digest(value: Uint8Array | string): Sha256Digest { + return `sha256:${createHash("sha256").update(value).digest("hex")}`; +} + +class FakeKubernetes implements BuildKubernetesOperations { + jobs: KubernetesJob[] = []; + deleted: string[] = []; + observation: BuildJobObservation | undefined = { phase: "queued" }; + logs = ""; + async createJob(job: KubernetesJob) { + this.jobs.push(job); + } + async getJob() { + return this.observation; + } + async getJobLogs() { + return this.logs; + } + async deleteJob(_namespace: string, name: string) { + this.deleted.push(name); + } +} + +async function fixture(maxLogBytes = 1024) { + const root = await mkdtemp(join(tmpdir(), "kuber-controller-")); + roots.push(root); + const cas = new FilesystemCas(join(root, "cas")); + const store = new MemoryBuildStore(); + const kubernetes = new FakeKubernetes(); + const source = Buffer.from("FROM scratch\n"); + const sourceDigest = await cas.put(source); + const manifest: WorkspaceManifest = { + version: BUILD_PROTOCOL_VERSION, + files: [ + { + path: "Dockerfile", + type: "file", + digest: sourceDigest, + size: source.byteLength, + mode: 0o644, + }, + ], + }; + const workspace = await cas.put(Buffer.from(JSON.stringify(manifest))); + let now = 0; + const controller = new BuildController({ + cas, + store, + kubernetes, + namespace: "builds", + workspaceRoot: join(root, "workspaces"), + workspaceClaimName: "workspaces", + cacheImage: "registry.test/cache/app", + maxLogBytes, + now: () => new Date(Date.UTC(2026, 8, 2, 0, 0, now++)), + resolveDigest: async () => `sha256:${"f".repeat(64)}`, + }); + const request: BuildRequest = { + version: BUILD_PROTOCOL_VERSION, + id: "request-one", + project: "demo", + service: "web", + spec: { + architecture: "amd64", + image: "registry.test/demo/web:latest", + context: ".", + buildArgs: [], + workspace, + }, + }; + return { + root, + cas, + store, + kubernetes, + controller, + request, + workspace, + sourceDigest, + }; +} + +async function internalFixture(maxLogBytes = 1024) { + const root = await mkdtemp(join(tmpdir(), "kuber-controller-internal-")); + roots.push(root); + const cas = new FilesystemCas(join(root, "cas")); + const store = new MemoryBuildStore(); + const kubernetes = new FakeKubernetes(); + const source = Buffer.from("FROM scratch\n"); + const sourceDigest = await cas.put(source); + const manifest: WorkspaceManifest = { + version: BUILD_PROTOCOL_VERSION, + files: [ + { + path: "Dockerfile", + type: "file", + digest: sourceDigest, + size: source.byteLength, + mode: 0o644, + }, + ], + }; + const workspace = await cas.put(Buffer.from(JSON.stringify(manifest))); + let now = 0; + const controller = new BuildController({ + cas, + store, + kubernetes, + namespace: "builds", + workspaceRoot: join(root, "workspaces"), + workspaceClaimName: "workspaces", + cacheImage: (request) => + `cncf-distribution-svc.registry.svc.cluster.local:5000/kuber/cache-${request.project}-${request.service}`, + imageName: (request) => + `registry.neko-piranha.ts.net/kuber/${request.project}-${request.service}:latest`, + pushImage: (request) => + `cncf-distribution-svc.registry.svc.cluster.local:5000/kuber/${request.project}-${request.service}:latest`, + pushRegistryInsecure: true, + maxLogBytes, + now: () => new Date(Date.UTC(2026, 8, 2, 0, 0, now++)), + resolveDigest: async () => `sha256:${"f".repeat(64)}`, + }); + const request: BuildRequest = { + version: BUILD_PROTOCOL_VERSION, + id: "request-internal", + project: "demo", + service: "web", + spec: { + architecture: "amd64", + image: "registry.neko-piranha.ts.net/kuber/demo-web:latest", + context: ".", + buildArgs: [], + workspace, + }, + }; + return { + root, + cas, + store, + kubernetes, + controller, + request, + workspace, + sourceDigest, + }; +} + +describe("build controller", () => { + test("negotiates snapshots and resumes verified blob uploads", async () => { + const { controller, cas } = await fixture(); + const content = Buffer.from("resumable"); + const expected = digest(content); + expect( + await controller.beginBlobUpload(expected, content.byteLength), + ).toMatchObject({ offset: 0, complete: false }); + expect( + await controller.uploadBlobChunk(expected, 0, content.subarray(0, 3)), + ).toMatchObject({ offset: 3 }); + expect( + await controller.beginBlobUpload(expected, content.byteLength), + ).toMatchObject({ offset: 3 }); + await expect( + controller.uploadBlobChunk(expected, 0, content), + ).rejects.toBeInstanceOf(BuildConflictError); + await controller.uploadBlobChunk(expected, 3, content.subarray(3)); + expect(await controller.completeBlobUpload(expected)).toMatchObject({ + complete: true, + offset: content.byteLength, + }); + expect( + await controller.uploadBlobChunk(expected, 0, Buffer.from("retry")), + ).toMatchObject({ + complete: true, + size: content.byteLength, + offset: content.byteLength, + }); + expect(Buffer.from(await cas.get(expected))).toEqual(content); + + const bad = `sha256:${"0".repeat(64)}` as Sha256Digest; + await controller.beginBlobUpload(bad, 1); + await controller.uploadBlobChunk(bad, 0, Buffer.from("x")); + await expect(controller.completeBlobUpload(bad)).rejects.toBeInstanceOf( + BuildValidationError, + ); + }); + + test("reports the manifest and source blobs missing during negotiation", async () => { + const { controller, cas } = await fixture(); + const absent = `sha256:${"1".repeat(64)}` as Sha256Digest; + expect(await controller.negotiateSnapshot(absent)).toEqual({ + workspace: absent, + missing: [absent], + ready: false, + }); + const manifest = Buffer.from( + JSON.stringify({ + version: 1, + files: [ + { path: "x", type: "file", digest: absent, size: 1, mode: 420 }, + ], + }), + ); + const workspace = await cas.put(manifest); + expect(await controller.negotiateSnapshot(workspace)).toEqual({ + workspace, + missing: [absent], + ready: false, + }); + }); + + test("submits once, blocks competing image builds, captures bounded logs, and resolves immutable results", async () => { + const { controller, kubernetes, request } = await fixture(8); + expect(await controller.submitBuild(request)).toMatchObject({ + state: "queued", + }); + expect( + await controller.submitBuild(structuredClone(request)), + ).toMatchObject({ state: "queued" }); + expect(kubernetes.jobs).toHaveLength(1); + const jobSpec = kubernetes.jobs[0]!.spec as any; + expect( + jobSpec.template.spec.containers[0].volumeMounts, + ).toContainEqual( + expect.objectContaining({ + name: "workspace", + mountPath: "/workspace", + subPath: `workspaces/${kubernetes.jobs[0]!.metadata.name}`, + }), + ); + await expect( + controller.submitBuild({ ...request, id: "request-two" }), + ).rejects.toBeInstanceOf(BuildConflictError); + await expect( + controller.submitBuild({ ...request, project: "changed" }), + ).rejects.toBeInstanceOf(BuildConflictError); + + kubernetes.logs = "old-line\nnew-line\n"; + kubernetes.observation = { + phase: "running", + startedAt: "2026-09-02T00:00:03.000Z", + }; + expect(await controller.reconcileBuild(request.id)).toMatchObject({ + state: "running", + }); + const logs = (await controller.getBuildEvents(request.id)).filter( + (event) => event.type === "log", + ); + expect( + logs.reduce( + (bytes, event) => bytes + Buffer.byteLength(event.message), + 0, + ), + ).toBeLessThanOrEqual(8); + + kubernetes.logs += "done\n"; + kubernetes.observation = { + phase: "succeeded", + finishedAt: "2026-09-02T00:00:04.000Z", + }; + const status = await controller.reconcileBuild(request.id); + expect(status).toMatchObject({ + state: "succeeded", + digest: `sha256:${"f".repeat(64)}`, + }); + expect(await controller.getBuildResult(request.id)).toEqual({ + image: "registry.test/demo/web", + digest: `sha256:${"f".repeat(64)}`, + reference: `registry.test/demo/web@sha256:${"f".repeat(64)}`, + }); + expect(await controller.reconcileBuild(request.id)).toEqual(status); + }); + + test("cancels idempotently and cleans up only terminal build resources", async () => { + const { controller, kubernetes, request, root } = await fixture(); + await controller.submitBuild(request); + await expect(controller.cleanupBuild(request.id)).rejects.toBeInstanceOf( + BuildConflictError, + ); + expect(await controller.cancelBuild(request.id)).toMatchObject({ + state: "failed", + error: "Build cancelled", + }); + const deletes = kubernetes.deleted.length; + expect(await controller.cancelBuild(request.id)).toMatchObject({ + error: "Build cancelled", + }); + expect(kubernetes.deleted).toHaveLength(deletes); + await controller.cleanupBuild(request.id); + expect(kubernetes.deleted.length).toBe(deletes + 1); + await expect( + lstat(join(root, "workspaces", kubernetes.jobs[0]!.metadata.name)), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + + test("pushes to internal registry, records canonical result, and uses insecure flags", async () => { + const { controller, kubernetes, request } = await internalFixture(); + await controller.submitBuild(request); + expect(kubernetes.jobs).toHaveLength(1); + const container = (kubernetes.jobs[0]!.spec as any).template.spec + .containers[0]; + expect(container.args).toContain( + "--output=type=image,name=cncf-distribution-svc.registry.svc.cluster.local:5000/kuber/demo-web:latest,push=true,registry.insecure=true", + ); + expect(container.args).toContain( + "--import-cache=type=registry,ref=cncf-distribution-svc.registry.svc.cluster.local:5000/kuber/cache-demo-web,registry.insecure=true", + ); + expect(container.args).toContain( + "--export-cache=type=registry,ref=cncf-distribution-svc.registry.svc.cluster.local:5000/kuber/cache-demo-web,mode=max,registry.insecure=true", + ); + + kubernetes.observation = { + phase: "succeeded", + finishedAt: "2026-09-02T00:00:04.000Z", + }; + const status = await controller.reconcileBuild(request.id); + expect(status).toMatchObject({ state: "succeeded" }); + const result = await controller.getBuildResult(request.id); + expect(result.image).toBe("registry.neko-piranha.ts.net/kuber/demo-web"); + expect(result.reference).toBe( + `registry.neko-piranha.ts.net/kuber/demo-web@sha256:${"f".repeat(64)}`, + ); + }); + + test("does not add insecure flags when pushRegistryInsecure is false", async () => { + const root = await mkdtemp(join(tmpdir(), "kuber-controller-secure-")); + roots.push(root); + const cas = new FilesystemCas(join(root, "cas")); + const store = new MemoryBuildStore(); + const kubernetes = new FakeKubernetes(); + const source = Buffer.from("FROM scratch\n"); + const sourceDigest = await cas.put(source); + const manifest: WorkspaceManifest = { + version: BUILD_PROTOCOL_VERSION, + files: [ + { + path: "Dockerfile", + type: "file", + digest: sourceDigest, + size: source.byteLength, + mode: 0o644, + }, + ], + }; + const workspace = await cas.put(Buffer.from(JSON.stringify(manifest))); + let now = 0; + const controller = new BuildController({ + cas, + store, + kubernetes, + namespace: "builds", + workspaceRoot: join(root, "workspaces"), + workspaceClaimName: "workspaces", + cacheImage: "registry.test/cache/app", + imageName: (request) => + `registry.test/${request.project}-${request.service}:latest`, + pushImage: (request) => + `internal.registry:5000/${request.project}-${request.service}:latest`, + maxLogBytes: 1024, + now: () => new Date(Date.UTC(2026, 8, 2, 0, 0, now++)), + resolveDigest: async () => `sha256:${"f".repeat(64)}`, + }); + const request: BuildRequest = { + version: BUILD_PROTOCOL_VERSION, + id: "request-secure", + project: "demo", + service: "web", + spec: { + architecture: "amd64", + image: "registry.test/demo-web:latest", + context: ".", + buildArgs: [], + workspace, + }, + }; + await controller.submitBuild(request); + const container = (kubernetes.jobs[0]!.spec as any).template.spec + .containers[0]; + for (const arg of container.args) { + expect(arg).not.toContain("registry.insecure"); + } + }); + + test("pushImage from request cannot redirect to a different image in results", async () => { + const root = await mkdtemp(join(tmpdir(), "kuber-controller-redirect-")); + roots.push(root); + const cas = new FilesystemCas(join(root, "cas")); + const store = new MemoryBuildStore(); + const kubernetes = new FakeKubernetes(); + const source = Buffer.from("FROM scratch\n"); + const sourceDigest = await cas.put(source); + const manifest: WorkspaceManifest = { + version: BUILD_PROTOCOL_VERSION, + files: [ + { + path: "Dockerfile", + type: "file", + digest: sourceDigest, + size: source.byteLength, + mode: 0o644, + }, + ], + }; + const workspace = await cas.put(Buffer.from(JSON.stringify(manifest))); + let now = 0; + const controller = new BuildController({ + cas, + store, + kubernetes, + namespace: "builds", + workspaceRoot: join(root, "workspaces"), + workspaceClaimName: "workspaces", + cacheImage: "canonical.test/cache/app", + imageName: (request) => + `canonical.test/${request.project}-${request.service}:latest`, + pushImage: (request) => + `internal.test/${request.project}-${request.service}:latest`, + maxLogBytes: 1024, + now: () => new Date(Date.UTC(2026, 8, 2, 0, 0, now++)), + resolveDigest: async () => `sha256:${"f".repeat(64)}`, + }); + const request: BuildRequest = { + version: BUILD_PROTOCOL_VERSION, + id: "request-redirect", + project: "demo", + service: "web", + spec: { + architecture: "amd64", + image: "canonical.test/demo-web:latest", + context: ".", + buildArgs: [], + workspace, + }, + }; + await controller.submitBuild(request); + const container = (kubernetes.jobs[0]!.spec as any).template.spec + .containers[0]; + expect( + container.args.some((arg: string) => + arg.includes("name=internal.test/demo-web:latest"), + ), + ).toBe(true); + + kubernetes.observation = { + phase: "succeeded", + finishedAt: "2026-09-02T00:00:04.000Z", + }; + await controller.reconcileBuild(request.id); + const result = await controller.getBuildResult(request.id); + expect(result.image).toBe("canonical.test/demo-web"); + expect(result.reference).toBe( + `canonical.test/demo-web@sha256:${"f".repeat(64)}`, + ); + }); +}); diff --git a/tests/server/build-job.test.ts b/tests/server/build-job.test.ts new file mode 100644 index 0000000..cddbb22 --- /dev/null +++ b/tests/server/build-job.test.ts @@ -0,0 +1,169 @@ +import { describe, expect, test } from "bun:test"; +import { createBuildJob } from "../../server/build-job"; +import type { BuildSpec } from "../../shared/build-protocol"; + +function spec(architecture: "arm64" | "amd64"): BuildSpec { + return { + architecture, + image: "registry.example.com/kuber/demo-web:latest", + context: "apps/web", + dockerfile: "docker/Web.Dockerfile", + target: "production", + buildArgs: ["NODE_ENV=production"], + workspace: `sha256:${"a".repeat(64)}`, + }; +} + +describe("BuildKit Job generation", () => { + test.each(["arm64", "amd64"] as const)( + "generates a rootless %s job", + (architecture) => { + const job = createBuildJob({ + name: `build-${architecture}`, + namespace: "kuber-system", + spec: spec(architecture), + workspaceClaimName: "build-workspaces", + workspaceSubPath: "snapshot", + cacheImage: "registry.example.com/cache/demo-web", + registrySecretName: "registry-auth", + nodeSelector: { "kubernetes.io/arch": "wrong", pool: "builders" }, + }); + const jobSpec = job.spec as any; + const pod = jobSpec.template.spec; + const container = pod.containers[0]; + expect(pod.nodeSelector["kubernetes.io/arch"]).toBe(architecture); + expect(pod.nodeSelector.pool).toBe("builders"); + expect(pod.automountServiceAccountToken).toBe(false); + expect(pod.securityContext).toMatchObject({ + runAsNonRoot: true, + runAsUser: 1000, + seccompProfile: { type: "Unconfined" }, + }); + expect(container.securityContext).toEqual({ + runAsNonRoot: true, + runAsUser: 1000, + allowPrivilegeEscalation: true, + seccompProfile: { type: "Unconfined" }, + appArmorProfile: { type: "Unconfined" }, + }); + expect(container.env).toContainEqual({ + name: "BUILDKITD_FLAGS", + value: "--oci-worker-no-process-sandbox", + }); + expect(container.args).toContain(`--opt=platform=linux/${architecture}`); + expect(container.args).toContain( + "--import-cache=type=registry,ref=registry.example.com/cache/demo-web", + ); + expect(container.args).toContain( + "--export-cache=type=registry,ref=registry.example.com/cache/demo-web,mode=max", + ); + expect(container.args).toContain( + "--output=type=image,name=registry.example.com/kuber/demo-web:latest,push=true", + ); + expect(container.volumeMounts).toContainEqual({ + name: "workspace", + mountPath: "/workspace", + readOnly: true, + subPath: "snapshot", + }); + expect(pod.volumes).toContainEqual({ + name: "registry-auth", + secret: { + secretName: "registry-auth", + items: [{ key: ".dockerconfigjson", path: "config.json" }], + }, + }); + expect( + jobSpec.template.metadata.annotations[ + "container.apparmor.security.beta.kubernetes.io/buildkit" + ], + ).toBe("unconfined"); + }, + ); + + test("uses pushImage for output when set", () => { + const job = createBuildJob({ + name: "build-push", + namespace: "default", + spec: spec("amd64"), + workspaceClaimName: "workspace", + cacheImage: "registry.example.com/cache/demo-web", + pushImage: "internal.registry:5000/kuber/demo-web:latest", + }); + const container = (job.spec as any).template.spec.containers[0]; + expect(container.args).toContain( + "--output=type=image,name=internal.registry:5000/kuber/demo-web:latest,push=true", + ); + expect(container.args).not.toContainEqual( + expect.stringContaining("--output=type=image,name=registry.example.com"), + ); + }); + + test("adds insecure flags when configured", () => { + const job = createBuildJob({ + name: "build-insecure", + namespace: "default", + spec: spec("amd64"), + workspaceClaimName: "workspace", + cacheImage: "internal.registry:5000/cache/demo-web", + pushImage: "internal.registry:5000/kuber/demo-web:latest", + pushRegistryInsecure: true, + cacheRegistryInsecure: true, + }); + const container = (job.spec as any).template.spec.containers[0]; + expect(container.args).toContain( + "--import-cache=type=registry,ref=internal.registry:5000/cache/demo-web,registry.insecure=true", + ); + expect(container.args).toContain( + "--export-cache=type=registry,ref=internal.registry:5000/cache/demo-web,mode=max,registry.insecure=true", + ); + expect(container.args).toContain( + "--output=type=image,name=internal.registry:5000/kuber/demo-web:latest,push=true,registry.insecure=true", + ); + }); + + test("no insecure flags when not configured", () => { + const job = createBuildJob({ + name: "build-secure", + namespace: "default", + spec: spec("amd64"), + workspaceClaimName: "workspace", + cacheImage: "registry.example.com/cache/demo-web", + }); + const container = (job.spec as any).template.spec.containers[0]; + for (const arg of container.args) { + expect(arg).not.toContain("registry.insecure"); + } + }); + + test("rejects traversal and invalid Kubernetes names", () => { + expect(() => + createBuildJob({ + name: "Invalid_Name", + namespace: "default", + spec: spec("arm64"), + workspaceClaimName: "workspace", + cacheImage: "cache", + }), + ).toThrow("DNS label"); + expect(() => + createBuildJob({ + name: "valid", + namespace: "default", + spec: { ...spec("arm64"), context: "../outside" }, + workspaceClaimName: "workspace", + cacheImage: "cache", + }), + ).toThrow("safe workspace-relative"); + expect(() => + createBuildJob({ + name: "valid", + namespace: "default", + spec: spec("arm64"), + workspaceClaimName: "workspace", + workspaceSubPath: "../outside", + cacheImage: "cache", + }), + ).toThrow("Workspace subPath"); + }); +}); diff --git a/tests/server/build-store.test.ts b/tests/server/build-store.test.ts new file mode 100644 index 0000000..8f391e6 --- /dev/null +++ b/tests/server/build-store.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from "bun:test"; +import { + BUILD_RECORD_API_VERSION, + BuildStoreConflictError, + MemoryBuildStore, + type BuildRecord, +} from "../../server/build-store"; +import { + BUILD_PROTOCOL_VERSION, + type BuildRequest, + type Sha256Digest, +} from "../../shared/build-protocol"; + +const workspace = `sha256:${"a".repeat(64)}` as Sha256Digest; + +function build(id: string, imageKey = "project\0service\0image"): BuildRecord { + const request: BuildRequest = { + version: BUILD_PROTOCOL_VERSION, + id, + project: "project", + service: "service", + spec: { + architecture: "amd64", + image: "registry.test/app:latest", + context: ".", + buildArgs: [], + workspace, + }, + }; + const status = { + version: BUILD_PROTOCOL_VERSION, + id, + state: "queued" as const, + createdAt: "2026-09-02T00:00:00.000Z", + }; + return { + apiVersion: BUILD_RECORD_API_VERSION, + kind: "BuildRecord", + metadata: { + name: id, + resourceVersion: "1", + creationTimestamp: status.createdAt, + labels: {}, + }, + spec: { request, imageKey, jobName: `job-${id}`, workspaceSubPath: id }, + status: { + ...status, + logBytes: 0, + logOffset: 0, + nextSequence: 1, + events: [{ type: "status", status }], + }, + }; +} + +describe("build store", () => { + test("atomically enforces idempotency and one active image build", async () => { + const store = new MemoryBuildStore(); + expect((await store.createBuild(build("one"))).created).toBe(true); + expect((await store.createBuild(build("one"))).created).toBe(false); + await expect( + store.createBuild(build("one", "different-key")), + ).rejects.toBeInstanceOf(BuildStoreConflictError); + await expect(store.createBuild(build("two"))).rejects.toBeInstanceOf( + BuildStoreConflictError, + ); + + const first = (await store.getBuild("one"))!; + const digest = `sha256:${"b".repeat(64)}` as Sha256Digest; + first.metadata.resourceVersion = "2"; + Object.assign(first.status, { + state: "succeeded", + digest, + finishedAt: first.status.createdAt, + }); + await store.replaceBuild(first, "1"); + expect((await store.createBuild(build("two"))).created).toBe(true); + + const changed = (await store.getBuild("one"))!; + changed.metadata.resourceVersion = "3"; + changed.status.digest = `sha256:${"c".repeat(64)}`; + await expect(store.replaceBuild(changed, "2")).rejects.toThrow("immutable"); + }); + + test("optimistically updates resumable upload records without leaking mutable data", async () => { + const store = new MemoryBuildStore(); + const digest = `sha256:${"d".repeat(64)}` as Sha256Digest; + const upload = { + apiVersion: BUILD_RECORD_API_VERSION, + kind: "BuildUpload" as const, + metadata: { + name: "upload", + resourceVersion: "1", + creationTimestamp: "now", + }, + spec: { digest, size: 3 }, + status: { offset: 0, data: new Uint8Array() }, + }; + await store.createUpload(upload); + upload.status.data = Buffer.from("mutated"); + expect((await store.getUpload(digest))?.status.data.byteLength).toBe(0); + const next = (await store.getUpload(digest))!; + next.metadata.resourceVersion = "2"; + next.status = { offset: 3, data: Buffer.from("abc") }; + await store.replaceUpload(next, "1"); + await expect(store.replaceUpload(next, "1")).rejects.toBeInstanceOf( + BuildStoreConflictError, + ); + }); +}); diff --git a/tests/server/cas.test.ts b/tests/server/cas.test.ts new file mode 100644 index 0000000..140eda6 --- /dev/null +++ b/tests/server/cas.test.ts @@ -0,0 +1,71 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { FilesystemCas } from "../../server/cas"; +import type { Sha256Digest } from "../../shared/build-protocol"; + +const roots: string[] = []; +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +function digest(value: string): Sha256Digest { + return `sha256:${createHash("sha256").update(value).digest("hex")}`; +} + +describe("filesystem CAS", () => { + test("stores, deduplicates, and verifies blobs", async () => { + const root = await mkdtemp(join(tmpdir(), "kuber-cas-")); + roots.push(root); + const cas = new FilesystemCas(root); + const expected = digest("hello"); + expect(await cas.put(Buffer.from("hello"), expected)).toBe(expected); + expect(await cas.has(expected)).toBe(true); + expect(Buffer.from(await cas.get(expected)).toString()).toBe("hello"); + expect( + await Promise.all( + Array.from({ length: 8 }, () => cas.put(Buffer.from("hello"))), + ), + ).toEqual(Array(8).fill(expected)); + await expect(cas.put(Buffer.from("other"), expected)).rejects.toThrow( + "digest mismatch", + ); + }); + + test("detects corruption on reads and existing writes", async () => { + const root = await mkdtemp(join(tmpdir(), "kuber-cas-")); + roots.push(root); + const cas = new FilesystemCas(root); + const expected = await cas.put(Buffer.from("hello")); + const hex = expected.slice(7); + await writeFile( + join(root, "sha256", hex.slice(0, 2), hex.slice(2)), + "tampered", + ); + await expect(cas.get(expected)).rejects.toThrow("Corrupt CAS blob"); + await expect(cas.put(Buffer.from("hello"))).rejects.toThrow( + "Corrupt CAS blob", + ); + }); + + test("does not follow a blob-path symlink", async () => { + const root = await mkdtemp(join(tmpdir(), "kuber-cas-")); + const outside = await mkdtemp(join(tmpdir(), "kuber-cas-outside-")); + roots.push(root, outside); + const cas = new FilesystemCas(root); + const expected = digest("hello"); + const hex = expected.slice(7); + await writeFile(join(outside, "blob"), "hello"); + await mkdir(join(root, "sha256", hex.slice(0, 2)), { recursive: true }); + await symlink( + join(outside, "blob"), + join(root, "sha256", hex.slice(0, 2), hex.slice(2)), + ); + expect(await cas.has(expected)).toBe(false); + await expect(cas.get(expected)).rejects.toThrow(); + }); +}); diff --git a/tests/server/exec-service.test.ts b/tests/server/exec-service.test.ts new file mode 100644 index 0000000..96e6e11 --- /dev/null +++ b/tests/server/exec-service.test.ts @@ -0,0 +1,373 @@ +import { describe, expect, test } from "bun:test"; +import { + createExecService, + EXEC_MANAGED_BY_LABEL, + EXEC_MANAGED_BY_VALUE, + EXEC_WORKSPACE_UID_LABEL, + ExecOutputLimitError, + type ExecChunk, + type ExecDeployment, + type ExecPod, + type KubernetesExecBackend, + type KubernetesExecExit, + type KubernetesExecProcess, + type KubernetesExecRequest, +} from "../../server/exec-service"; + +const workspace = { project: "shop", uid: "workspace-1" }; + +async function* chunks(...values: ExecChunk[]): AsyncGenerator { + for (const value of values) yield value; +} + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +class FakeProcess implements KubernetesExecProcess { + stdout: AsyncIterable = chunks(); + stderr: AsyncIterable = chunks(); + status: Promise = Promise.resolve({ exitCode: 0 }); + stdin: Uint8Array[] = []; + resizes: Array<[number, number]> = []; + stdinClosed = false; + closeCalls = 0; + + writeStdin(data: Uint8Array) { + this.stdin.push(data); + } + + closeStdin() { + this.stdinClosed = true; + } + + resize(columns: number, rows: number) { + this.resizes.push([columns, rows]); + } + + wait() { + return this.status; + } + + close() { + this.closeCalls += 1; + } +} + +class FakeBackend implements KubernetesExecBackend { + deployment: ExecDeployment | undefined = { + name: "api", + uid: "deployment-1", + labels: { + [EXEC_MANAGED_BY_LABEL]: EXEC_MANAGED_BY_VALUE, + [EXEC_WORKSPACE_UID_LABEL]: workspace.uid, + }, + selector: { app: "api" }, + containers: ["api", "sidecar"], + }; + pods: ExecPod[] = [ + { + name: "api-old", + uid: "pod-old", + deploymentUid: "deployment-1", + phase: "Running", + containers: [{ name: "api", running: true, ready: false }], + }, + { + name: "api-new", + uid: "pod-new", + deploymentUid: "deployment-1", + phase: "Running", + containers: [ + { name: "api", running: true, ready: true }, + { name: "sidecar", running: true }, + ], + }, + ]; + process = new FakeProcess(); + requests: KubernetesExecRequest[] = []; + signals: AbortSignal[] = []; + selectors: Array>> = []; + + async getDeployment() { + return this.deployment; + } + + async listPods( + _namespace: string, + selector: Readonly>, + ) { + this.selectors.push(selector); + return this.pods; + } + + async exec(request: KubernetesExecRequest, signal: AbortSignal) { + this.requests.push(request); + this.signals.push(signal); + return this.process; + } +} + +function input(overrides: Record = {}) { + return { + workspace, + deployment: "api", + command: ["sh", "-c", "printf ok"], + ...overrides, + }; +} + +async function allFrames(iterable: AsyncIterable): Promise { + const result: T[] = []; + for await (const value of iterable) result.push(value); + return result; +} + +describe("ExecService target resolution", () => { + test("selects a ready running pod owned by the named managed deployment", async () => { + const backend = new FakeBackend(); + const target = await createExecService(backend).resolveTarget(input()); + expect(target).toEqual({ + namespace: "shop", + deployment: "api", + deploymentUid: "deployment-1", + pod: "api-new", + podUid: "pod-new", + container: "api", + }); + expect(backend.selectors).toEqual([{ app: "api" }]); + }); + + test("rejects unmanaged and differently owned deployments", async () => { + const backend = new FakeBackend(); + backend.deployment = { + ...backend.deployment!, + labels: { [EXEC_MANAGED_BY_LABEL]: EXEC_MANAGED_BY_VALUE }, + }; + await expect( + createExecService(backend).resolveTarget(input()), + ).rejects.toMatchObject({ + code: "EXEC_TARGET_FORBIDDEN", + }); + backend.deployment.labels = { + [EXEC_MANAGED_BY_LABEL]: "someone-else", + [EXEC_WORKSPACE_UID_LABEL]: workspace.uid, + }; + await expect( + createExecService(backend).resolveTarget(input()), + ).rejects.toMatchObject({ + code: "EXEC_TARGET_FORBIDDEN", + }); + }); + + test("does not trust selector collisions or terminating pods", async () => { + const backend = new FakeBackend(); + backend.pods = [ + { + name: "foreign", + uid: "foreign-pod", + deploymentUid: "different-deployment", + phase: "Running", + containers: [{ name: "api", running: true, ready: true }], + }, + { + name: "terminating", + uid: "terminating-pod", + deploymentUid: "deployment-1", + phase: "Running", + deletionTimestamp: "2026-09-02T00:00:00Z", + containers: [{ name: "api", running: true, ready: true }], + }, + ]; + await expect( + createExecService(backend).resolveTarget(input()), + ).rejects.toMatchObject({ + code: "EXEC_TARGET_NOT_READY", + }); + }); + + test("supports an explicit container and rejects one absent from the pod", async () => { + const backend = new FakeBackend(); + expect( + await createExecService(backend).resolveTarget( + input({ container: "sidecar" }), + ), + ).toMatchObject({ container: "sidecar" }); + await expect( + createExecService(backend).resolveTarget(input({ container: "missing" })), + ).rejects.toMatchObject({ code: "EXEC_TARGET_NOT_FOUND" }); + }); +}); + +describe("ExecService non-TTY execution", () => { + test("captures stdout, stderr, and the remote exit status", async () => { + const backend = new FakeBackend(); + backend.process.stdout = chunks("hel", new TextEncoder().encode("lo")); + backend.process.stderr = chunks("warning\n"); + backend.process.status = Promise.resolve({ + exitCode: 7, + reason: "NonZeroExitCode", + }); + const result = await createExecService(backend).execute(input()); + expect(result).toMatchObject({ + pod: "api-new", + container: "api", + stdout: "hello", + stderr: "warning\n", + exitCode: 7, + reason: "NonZeroExitCode", + }); + expect(backend.requests[0]).toMatchObject({ tty: false }); + }); + + test("enforces independent output caps and closes the process", async () => { + const backend = new FakeBackend(); + backend.process.stdout = chunks("123", "456"); + await expect( + createExecService(backend, { + maxOutputCapBytes: 10, + defaultOutputCapBytes: 10, + }).execute(input({ stdoutCapBytes: 5 })), + ).rejects.toBeInstanceOf(ExecOutputLimitError); + expect(backend.process.closeCalls).toBeGreaterThan(0); + expect(backend.signals[0]?.aborted).toBe(true); + }); + + test("propagates cancellation and cleans up the process", async () => { + const backend = new FakeBackend(); + const status = deferred(); + backend.process.status = status.promise; + const controller = new AbortController(); + const execution = createExecService(backend).execute( + input({ signal: controller.signal }), + ); + await Bun.sleep(1); + controller.abort(); + status.reject(new Error("cancelled")); + await expect(execution).rejects.toMatchObject({ code: "EXEC_ABORTED" }); + expect(backend.process.closeCalls).toBeGreaterThan(0); + }); +}); + +describe("ExecService validation", () => { + test("validates names, command count, command bytes, and requested caps before exec", async () => { + const backend = new FakeBackend(); + const service = createExecService(backend, { + maxCommandArguments: 2, + maxCommandBytes: 6, + maxArgumentBytes: 4, + maxOutputCapBytes: 10, + defaultOutputCapBytes: 10, + }); + await expect( + service.execute(input({ deployment: "Bad_Name" })), + ).rejects.toMatchObject({ + code: "EXEC_INVALID", + }); + await expect( + service.execute(input({ command: ["a", "b", "c"] })), + ).rejects.toThrow("more than 2"); + await expect( + service.execute(input({ command: ["12345"] })), + ).rejects.toThrow("argument exceeds"); + await expect( + service.execute(input({ command: ["1234", "1234"] })), + ).rejects.toThrow("Command exceeds"); + await expect( + service.execute(input({ command: ["ok"], stdoutCapBytes: 11 })), + ).rejects.toThrow("stdoutCapBytes"); + expect(backend.requests).toHaveLength(0); + }); +}); + +describe("ExecService interactive sessions", () => { + test("maps process streams, stdin, resize, EOF, and exit into duplex frames", async () => { + const backend = new FakeBackend(); + backend.process.stdout = chunks("out"); + backend.process.stderr = chunks(new TextEncoder().encode("err")); + backend.process.status = Promise.resolve({ + exitCode: 3, + message: "finished", + }); + const controller = new AbortController(); + const session = await createExecService(backend).openInteractive({ + ...input(), + signal: controller.signal, + }); + await session.send({ type: "stdin", data: "hello", eof: true }); + await session.send({ type: "resize", columns: 120, rows: 40 }); + const frames = await allFrames(session); + expect(frames.map((frame) => frame.type).sort()).toEqual([ + "exit", + "stderr", + "stdout", + ]); + expect(new TextDecoder().decode(backend.process.stdin[0])).toBe("hello"); + expect(backend.process.stdinClosed).toBe(true); + expect(backend.process.resizes).toEqual([[120, 40]]); + expect(frames.find((frame) => frame.type === "exit")).toMatchObject({ + exitCode: 3, + message: "finished", + }); + expect(backend.requests[0]).toMatchObject({ tty: true }); + }); + + test("supports streaming sessions without allocating a TTY", async () => { + const backend = new FakeBackend(); + const session = await createExecService(backend).openInteractive({ + ...input(), + tty: false, + signal: new AbortController().signal, + }); + await allFrames(session); + expect(backend.requests[0]).toMatchObject({ tty: false }); + }); + + test("aborts and closes without emitting an error frame when the API signal ends", async () => { + const backend = new FakeBackend(); + const status = deferred(); + backend.process.status = status.promise; + backend.process.stdout = (async function* () { + yield "before-abort"; + await status.promise; + })(); + const controller = new AbortController(); + const session = await createExecService(backend).openInteractive({ + ...input(), + signal: controller.signal, + }); + const iterator = session[Symbol.asyncIterator](); + expect((await iterator.next()).value?.type).toBe("stdout"); + controller.abort(); + status.reject(new Error("transport closed")); + expect((await iterator.next()).done).toBe(true); + await Bun.sleep(1); + expect(backend.process.closeCalls).toBeGreaterThan(0); + }); + + test("validates interactive input frames", async () => { + const backend = new FakeBackend(); + const status = deferred(); + backend.process.status = status.promise; + const session = await createExecService(backend, { + maxStdinFrameBytes: 3, + }).openInteractive({ + ...input(), + signal: new AbortController().signal, + }); + await expect(session.send({ type: "stdin", data: "four" })).rejects.toThrow( + "stdin frame", + ); + await expect( + session.send({ type: "resize", columns: 0, rows: 24 }), + ).rejects.toThrow("Terminal dimensions"); + status.resolve({ exitCode: 0 }); + await allFrames(session); + }); +}); diff --git a/tests/server/exec-websocket.test.ts b/tests/server/exec-websocket.test.ts new file mode 100644 index 0000000..0498af3 --- /dev/null +++ b/tests/server/exec-websocket.test.ts @@ -0,0 +1,347 @@ +import { describe, expect, test } from "bun:test"; +import { + authorizeExecConnection, + execProblem, + WireExecSession, + type ExecConnection, + type ExecWebSocketLink, +} from "../../server/app"; +import { hashToken, MemoryAuthStore } from "../../server/auth"; +import { + createExecService, + EXEC_MANAGED_BY_LABEL, + EXEC_MANAGED_BY_VALUE, + EXEC_WORKSPACE_UID_LABEL, + type ExecChunk, + type ExecDeployment, + type ExecPod, + type KubernetesExecBackend, + type KubernetesExecExit, + type KubernetesExecProcess, + type KubernetesExecRequest, +} from "../../server/exec-service"; +import { MemoryWorkspaceStore } from "../../server/workspace-store"; + +const now = () => Date.parse("2026-09-02T00:00:00.000Z"); + +const workspace = { project: "shop", uid: "workspace-1" }; + +async function* chunks(...values: ExecChunk[]): AsyncGenerator { + for (const value of values) yield value; +} + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +class FakeProcess implements KubernetesExecProcess { + stdout: AsyncIterable = chunks(); + stderr: AsyncIterable = chunks(); + status: Promise = Promise.resolve({ exitCode: 0 }); + stdin: Uint8Array[] = []; + resizes: Array<[number, number]> = []; + stdinClosed = false; + closeCalls = 0; + + writeStdin(data: Uint8Array) { + this.stdin.push(data); + } + + closeStdin() { + this.stdinClosed = true; + } + + resize(columns: number, rows: number) { + this.resizes.push([columns, rows]); + } + + wait() { + return this.status; + } + + close() { + this.closeCalls += 1; + } +} + +class FakeBackend implements KubernetesExecBackend { + deployment: ExecDeployment | undefined = { + name: "api", + uid: "deployment-1", + labels: { + [EXEC_MANAGED_BY_LABEL]: EXEC_MANAGED_BY_VALUE, + [EXEC_WORKSPACE_UID_LABEL]: workspace.uid, + }, + selector: { app: "api" }, + containers: ["api"], + }; + pods: ExecPod[] = [ + { + name: "api-0", + uid: "pod-0", + deploymentUid: "deployment-1", + phase: "Running", + containers: [{ name: "api", running: true, ready: true }], + }, + ]; + process = new FakeProcess(); + requests: KubernetesExecRequest[] = []; + + async getDeployment() { + return this.deployment; + } + + async listPods() { + return this.pods; + } + + async exec(request: KubernetesExecRequest) { + this.requests.push(request); + return this.process; + } +} + +class FakeSocket implements ExecWebSocketLink { + sent: string[] = []; + closes: Array<[number | undefined, string | undefined]> = []; + sendText(data: string) { + this.sent.push(data); + } + close(code?: number, reason?: string) { + this.closes.push([code, reason]); + } +} + +function workspaceRecord(uid = "workspace-1") { + return { + apiVersion: "kuber.astrxl.dev/v2", + kind: "Workspace", + metadata: { + name: "shop", + uid, + resourceVersion: "1", + creationTimestamp: "2026-09-02T00:00:00.000Z", + }, + spec: { + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }, + status: { latestRevision: 1 }, + } as const; +} + +function connection(role: "viewer" | "operator" | "admin" = "operator") { + return { + identity: { + user: { + username: role, + passwordHash: "hash", + roles: [role], + authVersion: 1, + }, + session: { + tokenHash: hashToken("token"), + username: role, + authVersion: 1, + expiresAt: "2030-01-01T00:00:00.000Z", + }, + }, + workspace: workspaceRecord(), + } as ExecConnection; +} + +async function authenticatedStore(role: "viewer" | "operator" | "admin" = "operator") { + const store = new MemoryAuthStore(); + await store.putUser({ username: role, passwordHash: "hash", roles: [role] }); + await store.putSession({ + tokenHash: hashToken("token"), + username: role, + authVersion: 1, + expiresAt: "2030-01-01T00:00:00.000Z", + }); + return store; +} + +function request(path = "/api/v2/workspaces/shop/exec") { + return new Request(`https://kuber.astrxl.dev${path}`, { + headers: { authorization: "Bearer token" }, + }); +} + +describe("authorizeExecConnection", () => { + test("resolves the workspace UID server-side for authorized operators", async () => { + const store = await authenticatedStore("operator"); + const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-1" }); + await workspaceStore.create({ + id: "shop", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const result = await authorizeExecConnection( + { store, workspaceStore, now }, + request(), + "shop", + ); + expect(result.identity.user.roles).toContain("operator"); + expect(result.workspace.metadata.uid).toBe("workspace-1"); + }); + + test("rejects unauthenticated upgrades with a bearer challenge", async () => { + const result = await authorizeExecConnection( + { store: new MemoryAuthStore(), now }, + new Request("https://kuber.astrxl.dev/api/v2/workspaces/shop/exec"), + "shop", + ).then( + () => null, + (error) => error, + ); + expect((result as { status: number }).status).toBe(401); + }); + + test("rejects operators without the exec capability", async () => { + const store = await authenticatedStore("viewer"); + const error = await authorizeExecConnection( + { store, now }, + request(), + "shop", + ).then( + () => null, + (error) => error, + ); + expect((error as { status: number }).status).toBe(403); + }); + + test("does not resolve unauthenticated unknown workspaces", async () => { + const store = await authenticatedStore("operator"); + const workspaceStore = new MemoryWorkspaceStore(); + const error = await authorizeExecConnection( + { store, workspaceStore, now }, + request(), + "missing", + ).then( + () => null, + (error) => error, + ); + expect((error as { status: number }).status).toBe(404); + }); +}); + +describe("WireExecSession", () => { + test("opens on start and maps stdin/resize/output/exit wire frames", async () => { + const backend = new FakeBackend(); + backend.process.stdout = chunks("out"); + backend.process.stderr = chunks(new TextEncoder().encode("err")); + backend.process.status = Promise.resolve({ exitCode: 3 }); + const socket = new FakeSocket(); + const session = new WireExecSession( + socket, + createExecService(backend), + connection(), + ); + + await session.receive( + JSON.stringify({ + type: "start", + version: 1, + deployment: "api", + command: ["sh", "-c", "echo hi"], + tty: true, + }), + ); + + await session.receive( + JSON.stringify({ + type: "stdin", + data: Buffer.from("hello").toString("base64"), + encoding: "base64", + }), + ); + await session.receive( + JSON.stringify({ type: "resize", columns: 100, rows: 40 }), + ); + + await Bun.sleep(1); + expect(new TextDecoder().decode(backend.process.stdin[0])).toBe("hello"); + expect(backend.process.resizes).toEqual([[100, 40]]); + + const frames = socket.sent.map((text) => JSON.parse(text)); + expect(frames).toContainEqual({ + type: "stdout", + data: Buffer.from("out").toString("base64"), + encoding: "base64", + }); + expect(frames).toContainEqual({ type: "exit", exitCode: 3 }); + expect(socket.closes[0]?.[0]).toBe(1000); + }); + + test("rejects an unknown first frame", async () => { + const socket = new FakeSocket(); + const session = new WireExecSession( + socket, + createExecService(new FakeBackend()), + connection(), + ); + await session.receive(JSON.stringify({ type: "resize", columns: 1, rows: 1 })); + expect(JSON.parse(socket.sent[0]!)).toMatchObject({ + type: "error", + code: "EXEC_INVALID", + }); + }); + + test("rejects oversize frames", async () => { + const socket = new FakeSocket(); + const session = new WireExecSession( + socket, + createExecService(new FakeBackend()), + connection(), + { maxFrameBytes: 4 }, + ); + await session.receive(JSON.stringify({ type: "close" })); + expect(JSON.parse(socket.sent[0]!)).toMatchObject({ type: "error" }); + expect(socket.closes[0]?.[0]).toBe(1009); + }); + + test("aborts the process when the socket closes", async () => { + const backend = new FakeBackend(); + backend.process.stdout = (async function* () { + yield "data"; + })(); + const socket = new FakeSocket(); + const session = new WireExecSession( + socket, + createExecService(backend), + connection(), + ); + await session.receive( + JSON.stringify({ + type: "start", + version: 1, + deployment: "api", + command: ["sh"], + tty: false, + }), + ); + await Bun.sleep(1); + session.close(); + }); +}); + +describe("execProblem", () => { + test("formats HTTP errors as problem+json without leaking internals", async () => { + const response = execProblem( + new (class extends Error { + readonly status = 401; + readonly title = "Unauthorized"; + readonly code = "UNAUTHORIZED"; + })("boom"), + ); + expect(response.status).toBe(500); + expect((await response.json()) as { code: string }).toMatchObject({ + code: "INTERNAL_ERROR", + }); + }); +}); diff --git a/tests/server/kubernetes-state.test.ts b/tests/server/kubernetes-state.test.ts new file mode 100644 index 0000000..996e5b2 --- /dev/null +++ b/tests/server/kubernetes-state.test.ts @@ -0,0 +1,836 @@ +import { createHash } from "node:crypto"; +import { describe, expect, test } from "bun:test"; +import type { + KubernetesObject, + KubernetesObjectApi, + V1Lease, + V1LeaseSpec, +} from "@kubernetes/client-node"; +import { + createKubernetesManagementDependencies, + KubernetesOperationPersistence, + KubernetesWorkspaceAdoptionService, + KubernetesWorkspaceLeaseProvider, + KubernetesWorkspacePersistence, + type LeaseObjects, +} from "../../server/kubernetes-state"; +import type { Operation } from "../../server/operation-store"; +import type { + Workspace, + WorkspaceRevision, +} from "../../server/workspace-store"; +import { WORKSPACE_UID_LABEL } from "../../server/management"; + +type DataObject = KubernetesObject & { + data?: Record; + stringData?: Record; +}; + +class FakeObjects { + readonly objects = new Map(); + readonly patches: KubernetesObject[] = []; + + key(value: KubernetesObject) { + return `${value.kind}:${value.metadata?.namespace ?? ""}:${value.metadata?.name}`; + } + + async create(value: DataObject) { + const key = this.key(value); + if (this.objects.has(key)) throw { code: 409 }; + const stored = structuredClone(value); + if (stored.stringData) { + stored.data = Object.fromEntries( + Object.entries(stored.stringData).map(([name, item]) => [ + name, + Buffer.from(item).toString("base64"), + ]), + ); + delete stored.stringData; + } + stored.metadata = { ...stored.metadata, resourceVersion: "cluster-1" }; + this.objects.set(key, stored); + return stored; + } + + async read(value: KubernetesObject) { + const found = this.objects.get(this.key(value)); + if (!found) throw { code: 404 }; + return structuredClone(found); + } + + async replace(value: DataObject) { + const key = this.key(value); + if (!this.objects.has(key)) throw { code: 404 }; + const stored = structuredClone(value); + if (stored.stringData) { + stored.data = Object.fromEntries( + Object.entries(stored.stringData).map(([name, item]) => [ + name, + Buffer.from(item).toString("base64"), + ]), + ); + delete stored.stringData; + } + stored.metadata = { ...stored.metadata, resourceVersion: "cluster-2" }; + this.objects.set(key, stored); + return stored; + } + + async delete(value: KubernetesObject) { + if (!this.objects.delete(this.key(value))) throw { code: 404 }; + } + + async list( + apiVersion: string, + kind: string, + namespace: string, + _pretty?: string, + _exact?: boolean, + _export?: boolean, + _fieldSelector?: string, + labelSelector?: string, + ) { + const [label, expected] = labelSelector?.split("=") ?? []; + return { + items: [...this.objects.values()].filter( + (item) => + item.apiVersion === apiVersion && + item.kind === kind && + item.metadata?.namespace === namespace && + (!label || item.metadata.labels?.[label] === expected), + ), + }; + } + + async patch(value: KubernetesObject) { + this.patches.push(structuredClone(value)); + return value; + } +} + +const timestamp = "2026-09-02T00:00:00.000Z"; + +function operation(name = "random"): Operation { + return { + apiVersion: "kuber.astrxl.dev/v2", + kind: "Operation", + metadata: { + name, + uid: name, + resourceVersion: "1", + creationTimestamp: timestamp, + }, + spec: { + workspaceId: "demo", + action: "workspace.stop", + idempotencyKey: "same-key", + requestHash: "hash", + }, + status: { state: "pending" }, + }; +} + +function workspace(version: number): Workspace { + return { + apiVersion: "kuber.astrxl.dev/v2", + kind: "Workspace", + metadata: { + name: "demo", + uid: "workspace-uid", + resourceVersion: String(version), + creationTimestamp: timestamp, + }, + spec: { source: { uri: "oci://demo", digest: `sha256:${version}` } }, + status: { latestRevision: version }, + }; +} + +function revision(version: number): WorkspaceRevision { + return { + apiVersion: "kuber.astrxl.dev/v2", + kind: "WorkspaceRevision", + metadata: { + name: `demo-r${version}`, + uid: `revision-${version}`, + workspaceUid: "workspace-uid", + resourceVersion: String(version), + creationTimestamp: timestamp, + }, + spec: { + ...workspace(version).spec, + workspaceId: "demo", + revision: version, + }, + }; +} + +describe("Kubernetes state persistence", () => { + test("stores idempotency as one deterministic, recoverable operation object", async () => { + const fake = new FakeObjects(); + const persistence = new KubernetesOperationPersistence( + fake as unknown as KubernetesObjectApi, + ); + const first = await persistence.createIdempotent(operation("first")); + const second = await persistence.createIdempotent(operation("second")); + expect(second).toEqual(first); + expect(fake.objects.size).toBe(1); + expect(first.metadata.name).toBe( + `operation-${createHash("sha256") + .update("demo\0same-key") + .digest("hex") + .slice(0, 48)}`, + ); + }); + + test("recovers replacement from a matching precreated revision", async () => { + const fake = new FakeObjects(); + const persistence = new KubernetesWorkspacePersistence( + fake as unknown as KubernetesObjectApi, + ); + await persistence.create(workspace(1), revision(1)); + const revisionName = `revision-${createHash("sha256") + .update("demo\0" + "2") + .digest("hex") + .slice(0, 48)}`; + await fake.create({ + apiVersion: "v1", + kind: "Secret", + metadata: { name: revisionName, namespace: "kuber-system" }, + stringData: { payload: JSON.stringify(revision(2)) }, + }); + await persistence.replace(workspace(2), revision(2), "1"); + expect((await persistence.get("demo"))?.status.latestRevision).toBe(2); + }); + + test("adopts only kuber-managed resources and reserves platform adoption for its path", async () => { + const fake = new FakeObjects(); + fake.objects.set("Namespace::demo", { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "demo", + labels: { "app.kubernetes.io/managed-by": "kuber" }, + }, + }); + fake.objects.set("ConfigMap:demo:managed", { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: "managed", + namespace: "demo", + labels: { "app.kubernetes.io/managed-by": "kuber" }, + }, + }); + fake.objects.set("ConfigMap:demo:external", { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { name: "external", namespace: "demo" }, + }); + const adoption = new KubernetesWorkspaceAdoptionService( + fake as unknown as KubernetesObjectApi, + ); + expect(await adoption.adopt("demo", "workspace-uid")).toMatchObject({ + resourcesAdopted: 1, + }); + expect(fake.patches).toHaveLength(2); + expect( + fake.patches.every( + (item) => + item.metadata?.labels?.[WORKSPACE_UID_LABEL] === "workspace-uid", + ), + ).toBe(true); + await expect(adoption.adopt("kuber-system", "uid")).rejects.toThrow( + "reserved", + ); + }); + + test("adoption hydrates apiVersion/kind when list items lack them", async () => { + const fake = new FakeObjects(); + fake.objects.set("Namespace::demo", { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "demo", + labels: { "app.kubernetes.io/managed-by": "kuber" }, + }, + }); + fake.objects.set("Deployment:demo:managed", { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: "managed", + namespace: "demo", + labels: { "app.kubernetes.io/managed-by": "kuber" }, + }, + }); + // Emulate the KubernetesObjectApi behavior of returning list items without + // hydrated apiVersion/kind on the nested objects. + const dehydrating = { + list: async ( + apiVersion: string, + kind: string, + namespace: string, + _pretty?: string, + _exact?: boolean, + _export?: boolean, + _fieldSelector?: string, + labelSelector?: string, + ) => { + const [label, expected] = labelSelector?.split("=") ?? []; + const items = [...fake.objects.values()].filter( + (item) => + item.apiVersion === apiVersion && + item.kind === kind && + item.metadata?.namespace === namespace && + (!label || item.metadata.labels?.[label] === expected), + ); + return { + items: items.map(({ kind: _k, apiVersion: _a, ...item }) => item), + }; + }, + read: (value: KubernetesObject) => fake.read(value), + patch: async (value: KubernetesObject) => { + fake.patches.push(structuredClone(value)); + return value; + }, + }; + const adoption = new KubernetesWorkspaceAdoptionService( + dehydrating as unknown as KubernetesObjectApi, + ); + expect(await adoption.adopt("demo", "workspace-uid")).toMatchObject({ + resourcesAdopted: 1, + }); + expect(fake.patches).toHaveLength(2); + const [namespacePatch, resourcePatch] = fake.patches; + expect(namespacePatch).toMatchObject({ apiVersion: "v1", kind: "Namespace" }); + expect(resourcePatch).toMatchObject({ + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { name: "managed", namespace: "demo" }, + }); + }); +}); + +class FakeLeaseStore implements LeaseObjects { + readonly leases = new Map(); + private rv = 0; + + private key(name: string, namespace: string) { + return `${namespace}/${name}`; + } + + private persisted(value: V1Lease): V1Lease { + const stored = structuredClone(value) as V1Lease; + stored.metadata = { + ...(stored.metadata ?? {}), + name: value.metadata?.name ?? "", + namespace: value.metadata?.namespace ?? "kuber-system", + resourceVersion: String(++this.rv), + }; + return stored; + } + + async create(value: V1Lease) { + const key = this.key( + value.metadata!.name!, + value.metadata!.namespace ?? "kuber-system", + ); + if (this.leases.has(key)) throw { code: 409 }; + const stored = this.persisted(value); + this.leases.set(key, stored); + return structuredClone(stored); + } + + async read(name: string, namespace: string) { + const found = this.leases.get(this.key(name, namespace)); + return found ? structuredClone(found) : undefined; + } + + async replace(value: V1Lease) { + const key = this.key( + value.metadata!.name!, + value.metadata!.namespace ?? "kuber-system", + ); + const current = this.leases.get(key); + if (!current) throw { code: 404 }; + if (current.metadata?.resourceVersion !== value.metadata?.resourceVersion) + throw { code: 409 }; + const stored = this.persisted(value); + this.leases.set(key, stored); + return structuredClone(stored); + } + + async delete(name: string, namespace: string) { + this.leases.delete(this.key(name, namespace)); + } +} + +describe("Kubernetes workspace lease provider", () => { + test("acquire creates a lease and blocks other holders while valid", async () => { + const fake = new FakeLeaseStore(); + const provider = new KubernetesWorkspaceLeaseProvider(fake, "kuber-system"); + const lease = await provider.acquire("demo", "worker-a", 1000); + expect(lease).toBeDefined(); + expect(lease!.workspaceId).toBe("demo"); + expect(lease!.holder).toBe("worker-a"); + expect(fake.leases.size).toBe(1); + expect( + await provider.acquire("demo", "worker-b", 1000), + ).toBeUndefined(); + }); + + test("renews optimistically and refuses after expiry or holder change", async () => { + const fake = new FakeLeaseStore(); + let now = 0; + const provider = new KubernetesWorkspaceLeaseProvider( + fake, + "kuber-system", + () => now, + ); + const lease = await provider.acquire("demo", "worker-a", 1000); + expect(await lease!.renew(1000)).toBe(true); + + const second = await provider.acquire("demo", "worker-b", 1000); + expect(second).toBeUndefined(); + + now = 1500; + expect(await lease!.renew(1000)).toBe(false); + }); + + test("takes over an expired lease from a different holder", async () => { + const fake = new FakeLeaseStore(); + let now = 0; + const provider = new KubernetesWorkspaceLeaseProvider( + fake, + "kuber-system", + () => now, + ); + const first = await provider.acquire("demo", "worker-a", 1000); + expect(first).toBeDefined(); + + now = 1500; + const taken = await provider.acquire("demo", "worker-b", 1000); + expect(taken).toBeDefined(); + expect(taken!.holder).toBe("worker-b"); + const stored = [...fake.leases.values()][0]!; + expect(stored.spec?.holderIdentity).toBe("worker-b"); + expect(stored.spec?.leaseTransitions).toBe(1); + + expect(await first!.renew(1000)).toBe(false); + }); + + test("release only removes a lease still held by the owner", async () => { + const fake = new FakeLeaseStore(); + const provider = new KubernetesWorkspaceLeaseProvider(fake, "kuber-system"); + const lease = await provider.acquire("demo", "worker-a", 1000); + await lease!.release(); + expect(fake.leases.size).toBe(0); + expect(await provider.acquire("demo", "worker-b", 1000)).toBeDefined(); + }); + + test("writes acquireTime and renewTime as microsecond MicroTime strings", async () => { + const fake = new FakeLeaseStore(); + const provider = new KubernetesWorkspaceLeaseProvider( + fake, + "kuber-system", + () => Date.parse("2026-09-03T00:23:00.205Z"), + ); + const lease = await provider.acquire("demo", "worker-a", 1000); + const microRegex = + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/; + for (const stored of fake.leases.values()) { + expect(String(stored.spec?.acquireTime)).toMatch(microRegex); + expect(String(stored.spec?.renewTime)).toMatch(microRegex); + } + await lease!.renew(1000); + for (const stored of fake.leases.values()) { + expect(String(stored.spec?.renewTime)).toMatch(microRegex); + } + }); + + test("retries repeated expired-lease conflicts and eventually acquires", async () => { + const now = Date.parse("2026-09-03T00:00:00.000Z"); + const fake = new TakeoverContentionStore(now, 3); + const provider = new KubernetesWorkspaceLeaseProvider( + fake, + "kuber-system", + () => now, + ); + + const lease = await provider.acquire("demo", "worker-a", 1000); + + expect(lease).toBeDefined(); + expect(lease!.holder).toBe("worker-a"); + expect(fake.conflicts).toBe(3); + }); + + test("exhausts bounded retries and returns undefined under sustained contention", async () => { + const now = Date.parse("2026-09-03T00:00:00.000Z"); + const fake = new SustainedContentionStore(now); + const provider = new KubernetesWorkspaceLeaseProvider( + fake, + "kuber-system", + () => now, + ); + + const lease = await provider.acquire("demo", "worker-a", 1000); + + expect(lease).toBeUndefined(); + expect(fake.replaceAttempts).toBeGreaterThan(0); + expect(fake.replaceAttempts).toBeLessThanOrEqual(6); + }); +}); + +class TakeoverContentionStore implements LeaseObjects { + private rv = 0; + private lease: V1Lease | undefined; + conflicts = 0; + + constructor( + private readonly now: number, + private readonly conflictsBeforeSuccess: number, + ) { + this.lease = this.expiredLease(); + } + + private expiredLease(): V1Lease { + return { + apiVersion: "coordination.k8s.io/v1", + kind: "Lease", + metadata: { + name: "lease-demo", + namespace: "kuber-system", + resourceVersion: String(++this.rv), + }, + spec: { + holderIdentity: "contender", + leaseDurationSeconds: 1, + renewTime: new Date(this.now - 5000).toISOString() as unknown as V1LeaseSpec["renewTime"], + }, + }; + } + + async create(value: V1Lease): Promise { + if (this.lease) throw { code: 409 }; + this.lease = structuredClone(value); + return structuredClone(this.lease); + } + + async read(): Promise { + return this.lease ? structuredClone(this.lease) : undefined; + } + + async replace(value: V1Lease): Promise { + if (!this.lease) throw { code: 404 }; + if (this.conflicts < this.conflictsBeforeSuccess) { + this.conflicts++; + this.lease = this.expiredLease(); + throw { code: 409 }; + } + this.lease = structuredClone(value); + return structuredClone(this.lease); + } + + async delete(): Promise { + this.lease = undefined; + } +} + +class SustainedContentionStore implements LeaseObjects { + replaceAttempts = 0; + + constructor(private readonly now: number) {} + + async create(_value: V1Lease): Promise { + throw { code: 409 }; + } + + async read(): Promise { + return { + apiVersion: "coordination.k8s.io/v1", + kind: "Lease", + metadata: { + name: "lease-demo", + namespace: "kuber-system", + resourceVersion: "1", + }, + spec: { + holderIdentity: "contender", + leaseDurationSeconds: 1, + renewTime: new Date(this.now - 5000).toISOString() as unknown as V1LeaseSpec["renewTime"], + }, + }; + } + + async replace(_value: V1Lease): Promise { + this.replaceAttempts++; + throw { code: 409 }; + } + + async delete(): Promise {} +} + +class DehydratingListFake extends FakeObjects { + override async list( + apiVersion: string, + kind: string, + namespace: string, + _pretty?: string, + _exact?: boolean, + _export?: boolean, + _fieldSelector?: string, + labelSelector?: string, + ) { + const selectors = (labelSelector ?? "") + .split(",") + .map((part) => part.split("=")) + .filter(([label]) => label) as Array<[string, string]>; + const items = [...this.objects.values()].filter( + (item) => + item.apiVersion === apiVersion && + item.kind === kind && + item.metadata?.namespace === namespace && + selectors.every( + ([label, expected]) => item.metadata?.labels?.[label] === expected, + ), + ); + return { + items: items.map(({ apiVersion: _a, kind: _k, ...rest }) => rest), + }; + } +} + +describe("list normalization for dehydrated Kubernetes list items", () => { + test("lists Secret-backed operations despite missing apiVersion/kind on items", async () => { + const fake = new DehydratingListFake(); + const persistence = new KubernetesOperationPersistence( + fake as unknown as KubernetesObjectApi, + ); + const op = operation("op-1"); + await persistence.createIdempotent(op); + + const listed = await persistence.list(); + expect(listed).toHaveLength(1); + expect(listed[0]!.kind).toBe("Operation"); + expect(listed[0]!.spec.workspaceId).toBe("demo"); + }); + + test("lists Secret-backed workspaces despite missing apiVersion/kind on items", async () => { + const fake = new DehydratingListFake(); + const persistence = new KubernetesWorkspacePersistence( + fake as unknown as KubernetesObjectApi, + ); + await persistence.create(workspace(1), revision(1)); + + const listed = await persistence.list(); + expect(listed).toHaveLength(1); + expect(listed[0]!.metadata.name).toBe("demo"); + expect(listed[0]!.kind).toBe("Workspace"); + }); + + test("lists Secret-backed workspace revisions despite missing apiVersion/kind on items", async () => { + const fake = new DehydratingListFake(); + const persistence = new KubernetesWorkspacePersistence( + fake as unknown as KubernetesObjectApi, + ); + await persistence.create(workspace(1), revision(1)); + + const listed = await persistence.listRevisions("demo"); + expect(listed).toHaveLength(1); + expect(listed[0]!.spec.revision).toBe(1); + expect(listed[0]!.kind).toBe("WorkspaceRevision"); + }); + + test("preserves explicit apiVersion/kind when list items already carry them", async () => { + const fake = new FakeObjects(); + const persistence = new KubernetesOperationPersistence( + fake as unknown as KubernetesObjectApi, + ); + const op = operation("op-explicit"); + await persistence.createIdempotent(op); + + const listed = await persistence.list(); + expect(listed).toHaveLength(1); + expect(listed[0]!.apiVersion).toBe("kuber.astrxl.dev/v2"); + expect(listed[0]!.kind).toBe("Operation"); + }); +}); + +function replicaSet( + name: string, + namespace: string, + ownerUid: string, + revisionNumber: number, + image: string, + labels: Record = {}, + hasManagedBy = false, +): Record { + return { + apiVersion: "apps/v1", + kind: "ReplicaSet", + metadata: { + name, + namespace, + ...(hasManagedBy && { labels: { ...labels, "app.kubernetes.io/managed-by": "kuber" } }), + ...(!hasManagedBy && Object.keys(labels).length && { labels }), + annotations: { + "deployment.kubernetes.io/revision": String(revisionNumber), + }, + ...(ownerUid + ? { ownerReferences: [{ kind: "Deployment", name, uid: ownerUid }] } + : {}), + }, + spec: { + template: { + metadata: { labels: { app: name, "pod-template-hash": `hash${revisionNumber}` } }, + spec: { containers: [{ name: "app", image }] }, + }, + }, + }; +} + +function deployment( + name: string, + namespace: string, + uid: string, + image: string, +): Record { + return { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name, + namespace, + uid, + labels: { "app.kubernetes.io/managed-by": "kuber" }, + }, + spec: { + template: { + metadata: { labels: { app: name } }, + spec: { containers: [{ name: "app", image }] }, + }, + }, + }; +} + +class FakeApps { + constructor( + private readonly deployments: KubernetesObject[], + private readonly replicaSets: KubernetesObject[], + ) {} + + async listNamespacedDeployment({ namespace }: { namespace: string }) { + return { + items: this.deployments.filter((d) => d.metadata?.namespace === namespace), + }; + } + + async listNamespacedReplicaSet({ namespace }: { namespace: string }) { + return { + items: this.replicaSets.filter((rs) => rs.metadata?.namespace === namespace), + }; + } +} + +type ManagementClientsType = NonNullable< + Parameters[0] +>; + +const managementClients = ( + deployments: Array>, + replicaSets: Array>, +): ManagementClientsType => { + const apps = new FakeApps(deployments, replicaSets); + const objects = {} as KubernetesObjectApi; + return { + apps: apps as unknown as ManagementClientsType["apps"], + objects: {} as KubernetesObjectApi, + config: {} as ManagementClientsType["config"], + batch: {} as ManagementClientsType["batch"], + core: {} as ManagementClientsType["core"], + coordination: {} as ManagementClientsType["coordination"], + }; +}; + +describe("rollback ReplicaSet discovery", () => { + test("discovers owned revisions without the managed-by selector", async () => { + const depUid = "deployment-uid"; + const deployments = [deployment("web", "demo", depUid, "img:v2")]; + // ReplicaSets inherit only pod-template labels, never the deployment's + // managed-by metadata label. + const replicaSets = [ + replicaSet("web-1", "demo", depUid, 1, "img:v1", { app: "web" }), + replicaSet("web-2", "demo", depUid, 2, "img:v2", { app: "web" }), + ]; + const deps = createKubernetesManagementDependencies( + managementClients(deployments, replicaSets), + ); + const candidates = await deps.planRollback("demo"); + expect(candidates).toHaveLength(1); + expect(candidates[0]).toMatchObject({ + name: "web", + currentRevision: 2, + previousRevision: 1, + image: "img:v1", + }); + }); + + test("excludes ReplicaSets owned by another deployment", async () => { + const depUid = "deployment-uid"; + const otherUid = "other-deployment-uid"; + const deployments = [deployment("web", "demo", depUid, "img:v2")]; + const replicaSets = [ + replicaSet("web-1", "demo", depUid, 1, "img:v1", { app: "web" }), + replicaSet("web-2", "demo", depUid, 2, "img:v2", { app: "web" }), + replicaSet("other-1", "demo", otherUid, 1, "img:other", { app: "other" }), + ]; + const deps = createKubernetesManagementDependencies( + managementClients(deployments, replicaSets), + ); + const candidates = await deps.planRollback("demo"); + expect(candidates).toHaveLength(1); + const candidate = candidates[0]!; + expect(candidate).toMatchObject({ + name: "web", + currentRevision: 2, + previousRevision: 1, + image: "img:v1", + }); + }); + + test("excludes orphan ReplicaSets with no Deployment owner reference", async () => { + const depUid = "deployment-uid"; + const deployments = [deployment("web", "demo", depUid, "img:v1")]; + const replicaSets = [ + replicaSet("orphan-1", "demo", "", 1, "img:orphan", { app: "orphan" }), + ]; + const deps = createKubernetesManagementDependencies( + managementClients(deployments, replicaSets), + ); + const candidates = await deps.planRollback("demo"); + expect(candidates).toHaveLength(0); + }); + + test("excludes unrelated ReplicaSets with an unrelated managed-by label", async () => { + const depUid = "deployment-uid"; + const deployments = [deployment("web", "demo", depUid, "img:v2")]; + const replicaSets = [ + replicaSet("web-1", "demo", depUid, 1, "img:v1", { app: "web" }), + replicaSet("web-2", "demo", depUid, 2, "img:v2", { app: "web" }), + // An unrelated RS that happens to carry the managed-by label (e.g. a + // standalone non-Deployment object) must still be ignored. + replicaSet("standalone", "demo", "other-uid", 1, "img:standalone", { app: "standalone" }, true), + ]; + const deps = createKubernetesManagementDependencies( + managementClients(deployments, replicaSets), + ); + const candidates = await deps.planRollback("demo"); + expect(candidates).toHaveLength(1); + expect(candidates[0]).toMatchObject({ + name: "web", + currentRevision: 2, + previousRevision: 1, + image: "img:v1", + }); + }); +}); diff --git a/tests/server/kubernetes-store.test.ts b/tests/server/kubernetes-store.test.ts new file mode 100644 index 0000000..bc92359 --- /dev/null +++ b/tests/server/kubernetes-store.test.ts @@ -0,0 +1,261 @@ +import { createHash } from "node:crypto"; +import { describe, expect, test } from "bun:test"; +import type { + KubernetesObject, + KubernetesObjectApi, +} from "@kubernetes/client-node"; +import { hashToken } from "../../server/auth"; +import { + KubernetesAuthStore, + KUBER_SYSTEM_NAMESPACE, +} from "../../server/kubernetes-store"; + +type StoredSecret = KubernetesObject & { + data?: Record; + stringData?: Record; + type?: string; +}; + +function objectName(prefix: string, value: string): string { + const digest = createHash("sha256").update(value).digest("hex").slice(0, 48); + return `${prefix}-${digest}`; +} + +function encode(value: string): string { + return Buffer.from(value).toString("base64"); +} + +function secret( + recordType: "user" | "session", + name: string, + values: Record, +): StoredSecret { + return { + apiVersion: "v1", + kind: "Secret", + metadata: { + name, + namespace: KUBER_SYSTEM_NAMESPACE, + labels: { "kuber.astrxl.dev/type": recordType }, + }, + type: "Opaque", + data: Object.fromEntries( + Object.entries(values).map(([key, value]) => [key, encode(value)]), + ), + }; +} + +class FakeObjects { + readonly secrets = new Map(); + readonly patches: StoredSecret[] = []; + readonly deleted: string[] = []; + + async read(value: KubernetesObject): Promise { + const found = this.secrets.get(value.metadata?.name ?? ""); + if (!found) throw { code: 404 }; + return found; + } + + async patch(value: StoredSecret): Promise { + const stored: StoredSecret = { + ...value, + data: Object.fromEntries( + Object.entries(value.stringData ?? {}).map(([key, item]) => [ + key, + encode(item), + ]), + ), + }; + delete stored.stringData; + this.patches.push(value); + this.secrets.set(value.metadata?.name ?? "", stored); + return stored; + } + + async list( + _apiVersion: string, + _kind: string, + _namespace: string, + _pretty?: string, + _exact?: boolean, + _export?: boolean, + _fieldSelector?: string, + labelSelector?: string, + ): Promise<{ items: StoredSecret[] }> { + const type = labelSelector?.split("=")[1]; + return { + items: [...this.secrets.values()].filter( + (item) => item.metadata?.labels?.["kuber.astrxl.dev/type"] === type, + ), + }; + } + + async delete(value: KubernetesObject): Promise { + const name = value.metadata?.name ?? ""; + if (!this.secrets.delete(name)) throw { code: 404 }; + this.deleted.push(name); + } +} + +function setup(): { fake: FakeObjects; store: KubernetesAuthStore } { + const fake = new FakeObjects(); + return { + fake, + store: new KubernetesAuthStore(fake as unknown as KubernetesObjectApi), + }; +} + +describe("KubernetesAuthStore", () => { + test("persists authVersion and not copied roles in new sessions", async () => { + const { fake, store } = setup(); + await store.putUser({ + username: "alice", + passwordHash: "hash", + roles: ["viewer"], + }); + const tokenHash = hashToken("token"); + await store.putSession({ + tokenHash, + username: "alice", + roles: ["admin"], + expiresAt: "2026-09-03T00:00:00.000Z", + }); + + expect(fake.patches[0]?.stringData).toMatchObject({ authVersion: "1" }); + expect(fake.patches[1]?.stringData).toEqual({ + tokenHash, + username: "alice", + authVersion: "1", + expiresAt: "2026-09-03T00:00:00.000Z", + }); + expect(await store.getSession(tokenHash)).toMatchObject({ authVersion: 1 }); + }); + + test("reads legacy records at version one and invalidates them on update", async () => { + const { fake, store } = setup(); + const tokenHash = hashToken("legacy"); + fake.secrets.set( + objectName("user", "alice"), + secret("user", objectName("user", "alice"), { + username: "alice", + passwordHash: "hash", + roles: JSON.stringify(["viewer"]), + disabled: "false", + }), + ); + fake.secrets.set( + objectName("session", tokenHash), + secret("session", objectName("session", tokenHash), { + tokenHash, + username: "alice", + roles: JSON.stringify(["admin"]), + expiresAt: "2026-09-03T00:00:00.000Z", + }), + ); + + expect((await store.getUser("alice"))?.authVersion).toBe(1); + expect((await store.getSession(tokenHash))?.authVersion).toBe(1); + expect( + (await store.updateUser("alice", { roles: ["operator"] }))?.authVersion, + ).toBe(2); + expect(await store.getSession(tokenHash)).toBeUndefined(); + }); + + test("fails closed for malformed, mislabeled, and swapped Secret data", async () => { + const { fake, store } = setup(); + const name = objectName("user", "alice"); + const valid = secret("user", name, { + username: "alice", + passwordHash: "hash", + roles: JSON.stringify(["viewer"]), + authVersion: "1", + disabled: "false", + }); + + fake.secrets.set(name, { ...valid, data: { ...valid.data, roles: "%%%" } }); + expect(await store.getUser("alice")).toBeUndefined(); + fake.secrets.set(name, { ...valid, type: "kubernetes.io/tls" }); + expect(await store.getUser("alice")).toBeUndefined(); + fake.secrets.set(name, { + ...valid, + data: { ...valid.data, unexpected: encode("value") }, + }); + expect(await store.getUser("alice")).toBeUndefined(); + fake.secrets.set( + name, + secret("user", name, { + username: "bob", + passwordHash: "hash", + roles: JSON.stringify(["viewer"]), + authVersion: "1", + disabled: "false", + }), + ); + expect(await store.getUser("alice")).toBeUndefined(); + }); + + test("lists, creates, updates, revokes, and deletes users and sessions", async () => { + const { fake, store } = setup(); + await store.createUser({ + username: "bob", + passwordHash: "b", + roles: ["viewer"], + }); + await store.createUser({ + username: "alice", + passwordHash: "a", + roles: ["operator"], + }); + await expect( + store.createUser({ + username: "alice", + passwordHash: "a", + roles: ["viewer"], + }), + ).rejects.toThrow("already exists"); + expect((await store.listUsers()).map((user) => user.username)).toEqual([ + "alice", + "bob", + ]); + expect( + (await store.updateUser("alice", { disabled: true }))?.authVersion, + ).toBe(2); + + const bobToken = hashToken("bob"); + await store.putSession({ + tokenHash: bobToken, + username: "bob", + authVersion: 1, + expiresAt: "2026-09-03T00:00:00.000Z", + }); + expect(await store.deleteUser("bob")).toBe(true); + expect(fake.secrets.has(objectName("session", bobToken))).toBe(false); + expect(await store.deleteUser("missing")).toBe(false); + }); + + test("lists and deletes expired sessions without a cluster", async () => { + const { fake, store } = setup(); + await store.putUser({ + username: "alice", + passwordHash: "hash", + roles: ["viewer"], + }); + for (const [token, expiration] of [ + ["expired", "2026-09-01T00:00:00.000Z"], + ["active", "2026-09-03T00:00:00.000Z"], + ] as const) { + await store.putSession({ + tokenHash: hashToken(token), + username: "alice", + authVersion: 1, + expiresAt: expiration, + }); + } + const now = Date.parse("2026-09-02T00:00:00.000Z"); + expect(await store.listExpiredSessions(now)).toHaveLength(1); + expect(await store.deleteExpiredSessions(now)).toBe(1); + expect(fake.secrets.has(objectName("session", hashToken("active")))).toBe( + true, + ); + }); +}); diff --git a/tests/server/log-service.test.ts b/tests/server/log-service.test.ts new file mode 100644 index 0000000..495511f --- /dev/null +++ b/tests/server/log-service.test.ts @@ -0,0 +1,410 @@ +import { describe, expect, test } from "bun:test"; +import { + createLogService, + encodeLogEvent, + KubernetesLogError, + MANAGED_BY_SELECTOR, + type ContainerLogRequest, + type KubernetesDeployment, + type KubernetesLogsBackend, + type KubernetesPod, + type KubernetesService, + type LogEvent, +} from "../../server/log-service"; + +class FakeBackend implements KubernetesLogsBackend { + deployments: KubernetesDeployment[] = []; + service: KubernetesService | undefined; + pods: KubernetesPod[] = []; + reads = new Map(); + streams = new Map< + string, + (signal: AbortSignal) => AsyncIterable + >(); + deploymentCalls: Array>> = []; + podSelectors: Array>> = []; + requests: ContainerLogRequest[] = []; + + async listDeployments( + _namespace: string, + labels: Readonly>, + ) { + this.deploymentCalls.push(labels); + return this.deployments; + } + + async getService(_namespace: string, _name: string) { + return this.service; + } + + async listPods( + _namespace: string, + selector: Readonly>, + ) { + this.podSelectors.push(selector); + return this.pods; + } + + async readContainerLogs(request: ContainerLogRequest) { + this.requests.push(request); + const value = this.reads.get(`${request.pod}/${request.container}`) ?? ""; + if (value instanceof Error) throw value; + return value; + } + + streamContainerLogs(request: ContainerLogRequest, signal: AbortSignal) { + this.requests.push(request); + const stream = this.streams.get(`${request.pod}/${request.container}`); + if (stream) return stream(signal); + return openStream(signal); + } +} + +async function* openStream(signal: AbortSignal): AsyncGenerator { + await untilAbort(signal); + yield* [] as string[]; +} + +function untilAbort(signal: AbortSignal): Promise { + if (signal.aborted) return Promise.resolve(); + return new Promise((resolve) => + signal.addEventListener("abort", () => resolve(), { once: true }), + ); +} + +async function* chunks(...values: Array) { + for (const value of values) yield value; +} + +async function nextOfType( + iterator: AsyncIterator, + type: T, + limit = 30, +): Promise> { + for (let index = 0; index < limit; index += 1) { + const result = await iterator.next(); + if (result.done) throw new Error(`Stream ended before ${type}`); + if (result.value.type === type) + return result.value as Extract; + } + throw new Error(`No ${type} event received`); +} + +function setup() { + const backend = new FakeBackend(); + backend.deployments = [{ name: "web", selector: { app: "web" } }]; + backend.pods = [ + { + name: "web-abc", + uid: "uid-1", + phase: "Running", + containers: ["web", "sidecar"], + }, + ]; + const service = createLogService( + backend, + () => new Date("2026-09-02T12:00:00.000Z"), + ); + return { backend, service }; +} + +describe("LogService", () => { + test("enumerates every container in managed deployment pods", async () => { + const { backend, service } = setup(); + expect( + await service.listContainers({ + namespace: "demo", + target: { kind: "managed-deployments" }, + }), + ).toEqual([ + { + namespace: "demo", + targetKind: "deployment", + targetName: "web", + pod: "web-abc", + podUid: "uid-1", + container: "web", + }, + { + namespace: "demo", + targetKind: "deployment", + targetName: "web", + pod: "web-abc", + podUid: "uid-1", + container: "sidecar", + }, + ]); + expect(backend.deploymentCalls).toEqual([MANAGED_BY_SELECTOR]); + expect(backend.podSelectors).toEqual([{ app: "web" }]); + }); + + test("uses a named service selector without listing deployments", async () => { + const { backend, service } = setup(); + backend.service = { name: "frontend", selector: { role: "frontend" } }; + const result = await service.listContainers({ + namespace: "demo", + target: { kind: "service", name: "frontend" }, + }); + expect(result[0]).toMatchObject({ + targetKind: "service", + targetName: "frontend", + }); + expect(backend.deploymentCalls).toHaveLength(0); + expect(backend.podSelectors).toEqual([{ role: "frontend" }]); + }); + + test("rejects missing and selectorless services", async () => { + const { backend, service } = setup(); + const options = { + namespace: "demo", + target: { kind: "service" as const, name: "missing" }, + }; + await expect(service.listContainers(options)).rejects.toMatchObject({ + retryable: false, + }); + backend.service = { name: "missing", selector: {} }; + await expect(service.listContainers(options)).rejects.toThrow( + "no pod selector", + ); + }); + + test("collects timestamped structured lines and passes log options", async () => { + const { backend, service } = setup(); + backend.reads.set( + "web-abc/web", + "2026-09-02T11:59:00.123456Z first\nsecond\n", + ); + backend.reads.set("web-abc/sidecar", "side\n"); + const events = await service.collect({ + namespace: "demo", + target: { kind: "managed-deployments" }, + tailLines: 25, + sinceTime: new Date("2026-09-02T11:00:00Z"), + timestamps: true, + }); + expect(events).toHaveLength(3); + expect(events[0]).toMatchObject({ + type: "log", + timestamp: "2026-09-02T12:00:00.000Z", + logTimestamp: "2026-09-02T11:59:00.123456Z", + message: "first", + pod: "web-abc", + container: "web", + }); + expect(events[1]).toMatchObject({ message: "second" }); + expect(backend.requests[0]).toEqual({ + namespace: "demo", + pod: "web-abc", + container: "web", + tailLines: 25, + sinceSeconds: undefined, + sinceTime: "2026-09-02T11:00:00.000Z", + timestamps: true, + }); + expect(JSON.parse(encodeLogEvent(events[0]!))).toEqual(events[0]); + }); + + test("returns per-container safe errors and continues collection", async () => { + const { backend, service } = setup(); + backend.reads.set( + "web-abc/web", + new KubernetesLogError("container unavailable", false), + ); + backend.reads.set("web-abc/sidecar", "healthy"); + const events = await service.collect({ + namespace: "demo", + target: { kind: "managed-deployments" }, + }); + expect(events[0]).toEqual({ + type: "error", + timestamp: "2026-09-02T12:00:00.000Z", + namespace: "demo", + pod: "web-abc", + container: "web", + message: "container unavailable", + retryable: false, + }); + expect(events[1]).toMatchObject({ type: "log", message: "healthy" }); + expect(JSON.stringify(events)).not.toContain("env"); + expect(JSON.stringify(events)).not.toContain("Secret"); + }); + + test("validates tail and since options before Kubernetes calls", async () => { + const { backend, service } = setup(); + await expect( + service.collect({ + namespace: "demo", + target: { kind: "managed-deployments" }, + tailLines: -1, + }), + ).rejects.toThrow("tailLines"); + await expect( + service.collect({ + namespace: "demo", + target: { kind: "managed-deployments" }, + sinceSeconds: 5, + sinceTime: "2026-09-02T00:00:00Z", + }), + ).rejects.toThrow("mutually exclusive"); + expect(backend.deploymentCalls).toHaveLength(0); + }); + + test("follows chunked lines and stops active streams on abort", async () => { + const { backend, service } = setup(); + let cancelled = false; + backend.streams.set("web-abc/web", (signal) => + (async function* () { + try { + yield new TextEncoder().encode("hel"); + yield "lo\npartial"; + await untilAbort(signal); + } finally { + cancelled = true; + } + })(), + ); + const controller = new AbortController(); + const iterator = service.follow({ + namespace: "demo", + target: { kind: "managed-deployments" }, + signal: controller.signal, + discoveryIntervalMs: 5, + heartbeatIntervalMs: 1_000, + }); + expect(await nextOfType(iterator, "log")).toMatchObject({ + message: "hello", + }); + controller.abort(); + expect((await iterator.next()).done).toBe(true); + expect(cancelled).toBe(true); + }); + + test("discards queued lines immediately on external abort", async () => { + const { backend, service } = setup(); + backend.pods[0] = { ...backend.pods[0]!, containers: ["web"] }; + backend.streams.set("web-abc/web", () => chunks("one\ntwo\nthree\n")); + const controller = new AbortController(); + const iterator = service.follow({ + namespace: "demo", + target: { kind: "managed-deployments" }, + signal: controller.signal, + queueCapacity: 5, + discoveryIntervalMs: 1_000, + heartbeatIntervalMs: 1_000, + }); + expect(await nextOfType(iterator, "log")).toMatchObject({ message: "one" }); + await Bun.sleep(1); + controller.abort(); + expect((await iterator.next()).done).toBe(true); + }); + + test("discovers replacement pod UIDs while following", async () => { + const { backend, service } = setup(); + backend.pods[0] = { ...backend.pods[0]!, containers: ["web"] }; + backend.streams.set("web-abc/web", () => chunks("old\n")); + const controller = new AbortController(); + const iterator = service.follow({ + namespace: "demo", + target: { kind: "managed-deployments" }, + signal: controller.signal, + discoveryIntervalMs: 5, + heartbeatIntervalMs: 1_000, + }); + expect(await nextOfType(iterator, "log")).toMatchObject({ + message: "old", + podUid: "uid-1", + }); + backend.pods = [ + { name: "web-new", uid: "uid-2", containers: ["web"], phase: "Running" }, + ]; + backend.streams.set("web-new/web", () => chunks("new\n")); + expect(await nextOfType(iterator, "log")).toMatchObject({ + message: "new", + pod: "web-new", + podUid: "uid-2", + }); + controller.abort(); + await iterator.next(); + }); + + test("emits heartbeats and retryable stream errors before retrying", async () => { + const { backend, service } = setup(); + backend.pods[0] = { ...backend.pods[0]!, containers: ["web"] }; + let attempts = 0; + backend.streams.set("web-abc/web", () => + (async function* () { + attempts += 1; + if (attempts === 1) throw new Error("temporary disconnect"); + yield "recovered\n"; + })(), + ); + const controller = new AbortController(); + const iterator = service.follow({ + namespace: "demo", + target: { kind: "managed-deployments" }, + signal: controller.signal, + discoveryIntervalMs: 5, + heartbeatIntervalMs: 2, + retryIntervalMs: 7, + }); + expect(await nextOfType(iterator, "error")).toMatchObject({ + message: "temporary disconnect", + retryable: true, + retryAfterMs: 7, + pod: "web-abc", + }); + expect(await nextOfType(iterator, "heartbeat")).toMatchObject({ + timestamp: "2026-09-02T12:00:00.000Z", + }); + expect(await nextOfType(iterator, "log")).toMatchObject({ + message: "recovered", + }); + controller.abort(); + await iterator.next(); + expect(attempts).toBe(2); + }); + + test("emits one terminal target error and closes follow mode", async () => { + const { service } = setup(); + const iterator = service.follow({ + namespace: "demo", + target: { kind: "service", name: "missing" }, + signal: new AbortController().signal, + discoveryIntervalMs: 5, + heartbeatIntervalMs: 5, + }); + expect(await nextOfType(iterator, "error")).toMatchObject({ + message: "Service missing was not found", + retryable: false, + }); + expect((await iterator.next()).done).toBe(true); + }); + + test("applies bounded-queue backpressure to fast producers", async () => { + const { backend, service } = setup(); + backend.pods[0] = { ...backend.pods[0]!, containers: ["web"] }; + let produced = 0; + backend.streams.set("web-abc/web", () => + (async function* () { + for (let index = 0; index < 20; index += 1) { + produced += 1; + yield `${index}\n`; + } + })(), + ); + const controller = new AbortController(); + const iterator = service.follow({ + namespace: "demo", + target: { kind: "managed-deployments" }, + signal: controller.signal, + queueCapacity: 1, + discoveryIntervalMs: 1_000, + heartbeatIntervalMs: 1_000, + }); + expect((await iterator.next()).value).toMatchObject({ message: "0" }); + await Bun.sleep(10); + expect(produced).toBeLessThanOrEqual(3); + controller.abort(); + await iterator.return?.(undefined); + }); +}); diff --git a/tests/server/management.test.ts b/tests/server/management.test.ts new file mode 100644 index 0000000..5f0f65e --- /dev/null +++ b/tests/server/management.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, test } from "bun:test"; +import type { KubernetesObject, V1Deployment } from "@kubernetes/client-node"; +import { + createManagementService, + type ManagementDependencies, + type ResourceIdentity, + WORKSPACE_PROJECT_LABEL, + WORKSPACE_UID_LABEL, +} from "../../server/management"; + +const workspace = { project: "shop", uid: "workspace-1" }; + +function object( + kind: string, + name: string, + uid: string, + labels: Record = { + "app.kubernetes.io/managed-by": "kuber", + [WORKSPACE_UID_LABEL]: workspace.uid, + }, +): KubernetesObject { + return { + apiVersion: kind === "Deployment" ? "apps/v1" : "v1", + kind, + metadata: { name, namespace: workspace.project, uid, labels }, + }; +} + +function dependencies( + overrides: Partial = {}, +): ManagementDependencies { + return { + readNamespace: async () => ({ + uid: "namespace-uid", + labels: { + "app.kubernetes.io/managed-by": "kuber", + [WORKSPACE_UID_LABEL]: workspace.uid, + }, + }), + listDeployments: async () => [], + scaleDeployment: async () => {}, + restartDeployment: async () => {}, + waitForDeployment: async () => {}, + fetchGraphObjects: async () => [], + planRollback: async () => [], + rollbackDeployment: async () => {}, + listProjectResources: async () => [], + listDatabaseResources: async () => [], + listStorageResources: async () => [], + findStaleResources: async () => [], + applyResource: async (resource) => resource, + deleteResource: async () => {}, + reconcileDatabases: async () => ({}), + getDatabaseCredentials: async () => ({ + username: "user", + password: "pass", + }), + reconcileStorage: async () => ({}), + getStorageCredentials: async () => ({}), + ...overrides, + }; +} + +describe("server management service", () => { + test("contains no authorization policy and operates on an explicit workspace", async () => { + const calls: string[] = []; + const service = createManagementService( + dependencies({ + listDeployments: async (project) => { + calls.push(`list:${project}`); + return [object("Deployment", "api", "api-uid") as V1Deployment]; + }, + scaleDeployment: async (project, name, replicas) => { + calls.push(`scale:${project}:${name}:${replicas}`); + }, + }), + ); + + expect(await service.stop(workspace)).toEqual(["api"]); + expect(calls).toEqual(["list:shop", "scale:shop:api:0"]); + }); + + test("targets requested deployments and rejects unmanaged names", async () => { + const restarted: string[] = []; + const service = createManagementService( + dependencies({ + listDeployments: async () => + ["api", "worker"].map( + (name) => object("Deployment", name, `${name}-uid`) as V1Deployment, + ), + restartDeployment: async (_project, name) => { + restarted.push(name); + }, + }), + ); + + expect(await service.restart(workspace, ["worker", "worker"])).toEqual([ + "worker", + ]); + expect(restarted).toEqual(["worker"]); + await expect(service.restart(workspace, ["missing"])).rejects.toThrow( + "No managed deployment named missing", + ); + }); + + test("normal down retains ingress and persistent volume claims", async () => { + const resources = [ + object("Deployment", "api", "deployment-uid"), + object("Ingress", "api", "ingress-uid"), + object("PersistentVolumeClaim", "data", "pvc-uid"), + object("Service", "api", "service-uid"), + ]; + const service = createManagementService( + dependencies({ listProjectResources: async () => resources }), + ); + + const plan = await service.planDown(workspace); + expect(plan.retained.map((item) => item.kind)).toEqual([ + "PersistentVolumeClaim", + "Ingress", + ]); + expect(plan.delete.map((item) => item.kind)).toEqual([ + "Deployment", + "Service", + ]); + }); + + test("uses stale object UIDs as immutable deletion identities", async () => { + const stale = object("Secret", "old-env", "old-secret-uid"); + const deleted: ResourceIdentity[] = []; + const service = createManagementService( + dependencies({ + findStaleResources: async () => [stale], + deleteResource: async (item) => { + deleted.push(item); + }, + }), + ); + + const plan = await service.planResources(workspace, [ + { apiVersion: "v1", kind: "ConfigMap", metadata: { name: "env" } }, + ]); + expect(plan.desired[0]?.metadata?.labels).toMatchObject({ + [WORKSPACE_UID_LABEL]: workspace.uid, + [WORKSPACE_PROJECT_LABEL]: workspace.project, + }); + expect(plan.stale).toEqual([ + { + apiVersion: "v1", + kind: "Secret", + name: "old-env", + namespace: "shop", + uid: "old-secret-uid", + workspaceUid: workspace.uid, + }, + ]); + await service.deleteResources(workspace, plan.stale); + expect(deleted).toEqual(plan.stale); + }); + + test("rejects namespace and resource ownership mismatches", async () => { + const wrongNamespace = createManagementService( + dependencies({ + readNamespace: async () => ({ + uid: "namespace-uid", + labels: { + "app.kubernetes.io/managed-by": "kuber", + [WORKSPACE_UID_LABEL]: "someone-else", + }, + }), + }), + ); + await expect(wrongNamespace.stop(workspace)).rejects.toThrow( + "different-workspace", + ); + + const wrongResource = createManagementService( + dependencies({ + findStaleResources: async () => [ + object("Secret", "foreign", "foreign-uid", { + "app.kubernetes.io/managed-by": "kuber", + [WORKSPACE_UID_LABEL]: "someone-else", + }), + ], + }), + ); + await expect(wrongResource.planResources(workspace, [])).rejects.toThrow( + "is not owned by workspace", + ); + }); + + test("full down requires the namespace UID and includes owned external resources", async () => { + const database = { + ...object("Database", "orders", "database-uid"), + apiVersion: "postgresql.cnpg.io/v1", + metadata: { + ...object("Database", "orders", "database-uid").metadata, + namespace: "database", + }, + }; + const service = createManagementService( + dependencies({ listDatabaseResources: async () => [database] }), + ); + const plan = await service.planDown(workspace, true); + expect(plan.delete.map(({ kind, uid }) => [kind, uid])).toEqual([ + ["Database", "database-uid"], + ["Namespace", "namespace-uid"], + ]); + + const missingUid = createManagementService( + dependencies({ + readNamespace: async () => ({ + labels: { + "app.kubernetes.io/managed-by": "kuber", + [WORKSPACE_UID_LABEL]: workspace.uid, + }, + }), + }), + ); + await expect(missingUid.planDown(workspace, true)).rejects.toThrow( + "no UID deletion precondition", + ); + }); + + test("protects reserved namespaces before consulting Kubernetes", async () => { + let read = false; + const service = createManagementService( + dependencies({ + readNamespace: async () => { + read = true; + return undefined; + }, + }), + ); + await expect( + service.namespaceSafety({ project: "kube-system", uid: "workspace-1" }), + ).rejects.toThrow("reserved"); + expect(read).toBe(false); + }); +}); diff --git a/tests/server/materialize.test.ts b/tests/server/materialize.test.ts new file mode 100644 index 0000000..5104b0c --- /dev/null +++ b/tests/server/materialize.test.ts @@ -0,0 +1,143 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { lstat, mkdtemp, readFile, readlink, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { FilesystemCas } from "../../server/cas"; +import { + materializeWorkspace, + parseWorkspaceManifest, +} from "../../server/materialize"; +import { + BUILD_PROTOCOL_VERSION, + type Sha256Digest, + type WorkspaceManifest, +} from "../../shared/build-protocol"; + +const roots: string[] = []; +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +function digest(data: Uint8Array | string): Sha256Digest { + return `sha256:${createHash("sha256").update(data).digest("hex")}`; +} + +describe("source materialization", () => { + test("materializes files and safe symlinks with declared modes", async () => { + const root = await mkdtemp(join(tmpdir(), "kuber-materialize-")); + roots.push(root); + const cas = new FilesystemCas(join(root, "cas")); + const executable = Buffer.from("#!/bin/sh\necho ok\n"); + const link = Buffer.from("bin/run"); + await cas.put(executable, digest(executable)); + await cas.put(link, digest(link)); + const manifest: WorkspaceManifest = { + version: BUILD_PROTOCOL_VERSION, + files: [ + { + path: "bin/run", + type: "file", + digest: digest(executable), + size: executable.byteLength, + mode: 0o755, + }, + { + path: "run", + type: "symlink", + digest: digest(link), + size: link.byteLength, + mode: 0o777, + }, + ], + }; + const manifestBytes = Buffer.from(JSON.stringify(manifest)); + const workspace = await cas.put(manifestBytes); + const destination = join(root, "workspaces", "build-1"); + + expect(await materializeWorkspace(cas, workspace, destination)).toEqual( + manifest, + ); + expect(await readFile(join(destination, "bin/run"), "utf8")).toContain( + "echo ok", + ); + expect((await lstat(join(destination, "bin/run"))).mode & 0o777).toBe( + 0o755, + ); + expect(await readlink(join(destination, "run"))).toBe("bin/run"); + }); + + test("rejects traversal, path collisions, unsafe links, and size mismatches atomically", async () => { + expect(() => + parseWorkspaceManifest( + Buffer.from( + JSON.stringify({ + version: 1, + files: [ + { + path: "../x", + type: "file", + digest: `sha256:${"a".repeat(64)}`, + size: 0, + mode: 420, + }, + ], + }), + ), + ), + ).toThrow("invalid file"); + expect(() => + parseWorkspaceManifest( + Buffer.from( + JSON.stringify({ + version: 1, + files: [ + { + path: "a", + type: "file", + digest: `sha256:${"a".repeat(64)}`, + size: 0, + mode: 420, + }, + { + path: "a/b", + type: "file", + digest: `sha256:${"b".repeat(64)}`, + size: 0, + mode: 420, + }, + ], + }), + ), + ), + ).toThrow("conflicts"); + + const root = await mkdtemp(join(tmpdir(), "kuber-materialize-")); + roots.push(root); + const cas = new FilesystemCas(join(root, "cas")); + const link = Buffer.from("../../outside"); + const linkDigest = await cas.put(link); + const manifest = Buffer.from( + JSON.stringify({ + version: 1, + files: [ + { + path: "nested/link", + type: "symlink", + digest: linkDigest, + size: link.byteLength, + mode: 0o777, + }, + ], + }), + ); + const workspace = await cas.put(manifest); + const destination = join(root, "workspace"); + await expect( + materializeWorkspace(cas, workspace, destination), + ).rejects.toThrow("Unsafe symlink"); + await expect(lstat(destination)).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); diff --git a/tests/server/operation-store.test.ts b/tests/server/operation-store.test.ts new file mode 100644 index 0000000..e22e3fc --- /dev/null +++ b/tests/server/operation-store.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, test } from "bun:test"; +import { + MemoryOperationStore, + MemoryWorkspaceLeaseProvider, + OperationConflictError, + OperationValidationError, + recoverStaleOperations, + sanitizeOperationResult, +} from "../../server/operation-store"; + +describe("operation store", () => { + test("deduplicates matching requests and rejects key reuse", async () => { + let uid = 0; + const store = new MemoryOperationStore(undefined, () => `uid-${++uid}`); + const input = { + workspaceId: "demo", + action: "deploy", + idempotencyKey: "request-1", + request: { revision: 2 }, + }; + const first = await store.create(input); + expect(first.spec).not.toHaveProperty("request"); + expect((await store.create(input)).metadata.uid).toBe(first.metadata.uid); + expect( + (await store.create({ ...input, request: { revision: 2 } })).metadata.uid, + ).toBe(first.metadata.uid); + expect( + store.create({ ...input, request: { revision: 3 } }), + ).rejects.toBeInstanceOf(OperationConflictError); + }); + + test("hashes object requests canonically and redacts persisted results", async () => { + const store = new MemoryOperationStore(); + const operation = await store.create({ + workspaceId: "demo", + action: "resources.apply", + idempotencyKey: "canonical", + request: { z: 1, nested: { b: 2, a: 1 } }, + }); + expect( + ( + await store.create({ + workspaceId: "demo", + action: "resources.apply", + idempotencyKey: "canonical", + request: { nested: { a: 1, b: 2 }, z: 1 }, + }) + ).metadata.name, + ).toBe(operation.metadata.name); + await store.transition(operation.metadata.name, "running"); + const completed = await store.transition( + operation.metadata.name, + "succeeded", + { + result: { + kind: "Secret", + data: { password: "encoded" }, + metadata: { name: "credentials" }, + }, + }, + ); + expect(completed.status.result).toEqual({ + kind: "Secret", + data: "[REDACTED]", + metadata: { name: "credentials" }, + }); + expect( + sanitizeOperationResult({ DATABASE_URL: "postgres://secret" }), + ).toEqual({ DATABASE_URL: "[REDACTED]" }); + }); + + test("redacts credential-bearing values while preserving ordinary ones", () => { + expect( + sanitizeOperationResult({ + endpoint: "https://alice:s3cret@db.example.com:5432/mydb", + awsKey: "AKIAIOSFODNN7EXAMPLE", + keyMaterial: "-----BEGIN OPENSSH PRIVATE KEY-----", + registry: "https://registry.example.com/v2/app", + note: "used user@example.com for the AKIA lookup", + uid: "abc-123-def", + }), + ).toEqual({ + endpoint: "https://[REDACTED]@db.example.com:5432/mydb", + awsKey: "[REDACTED]", + keyMaterial: "[REDACTED]", + registry: "https://registry.example.com/v2/app", + note: "used user@example.com for the AKIA lookup", + uid: "abc-123-def", + }); + expect( + sanitizeOperationResult([ + "checkout https://git@example.com/org/repo.git", + "AKIAIOSFODNN7EXAMPLE", + ]), + ).toEqual([ + "checkout https://git@example.com/org/repo.git", + "[REDACTED]", + ]); + }); + + test("enforces the operation state machine", async () => { + const store = new MemoryOperationStore(); + const operation = await store.create({ + workspaceId: "demo", + action: "deploy", + idempotencyKey: "request-2", + }); + const running = await store.transition(operation.metadata.name, "running"); + expect(running.status.startedAt).toBeDefined(); + const succeeded = await store.transition( + operation.metadata.name, + "succeeded", + { result: { ready: true } }, + ); + expect(succeeded.metadata.resourceVersion).toBe("3"); + expect( + store.transition(operation.metadata.name, "failed", { + error: { code: "late", message: "late" }, + }), + ).rejects.toBeInstanceOf(OperationConflictError); + + const failed = await store.create({ + workspaceId: "demo", + action: "delete", + idempotencyKey: "request-3", + }); + expect( + store.transition(failed.metadata.name, "failed"), + ).rejects.toBeInstanceOf(OperationValidationError); + }); + + test("provides exclusive, renewable per-workspace leases", async () => { + let now = 0; + const leases = new MemoryWorkspaceLeaseProvider(() => now); + const first = await leases.acquire("demo", "worker-a", 100); + expect(first).toBeDefined(); + expect(await leases.acquire("demo", "worker-b", 100)).toBeUndefined(); + expect(await first!.renew()).toBe(true); + await first!.release(); + expect(await leases.acquire("demo", "worker-b", 100)).toBeDefined(); + now = 101; + }); + + test("startup recovery fails stale pending and running operations", async () => { + const store = new MemoryOperationStore(); + const pending = await store.create({ + workspaceId: "demo", + action: "deploy", + idempotencyKey: "recover-1", + }); + const running = await store.create({ + workspaceId: "demo", + action: "stop", + idempotencyKey: "recover-2", + }); + await store.transition(running.metadata.name, "running"); + const done = await store.create({ + workspaceId: "demo", + action: "restart", + idempotencyKey: "recover-3", + }); + await store.transition(done.metadata.name, "running"); + await store.transition(done.metadata.name, "succeeded", { + result: { ok: true }, + }); + + const recovered = await recoverStaleOperations(store); + expect(recovered).toBe(2); + + const after = await store.list(); + const byId = new Map(after.map((o) => [o.metadata.name, o])); + expect(byId.get(pending.metadata.name)?.status.state).toBe("failed"); + expect(byId.get(running.metadata.name)?.status.state).toBe("failed"); + expect(byId.get(pending.metadata.name)?.status.error?.code).toBe( + "OPERATION_INTERRUPTED", + ); + expect(byId.get(done.metadata.name)?.status.state).toBe("succeeded"); + + expect(await recoverStaleOperations(store)).toBe(0); + }); +}); diff --git a/tests/server/registry.test.ts b/tests/server/registry.test.ts new file mode 100644 index 0000000..b1143a3 --- /dev/null +++ b/tests/server/registry.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { + parseImageReference, + resolveRegistryDigest, +} from "../../server/registry"; + +describe("OCI registry digest resolution", () => { + test("parses tags, ports, defaults, and pinned references", () => { + expect(parseImageReference("localhost:5000/team/image:v1")).toMatchObject({ + registry: "localhost:5000", + repository: "team/image", + reference: "v1", + }); + expect( + parseImageReference("registry.example.com/team/image").reference, + ).toBe("latest"); + const pinned = `sha256:${"a".repeat(64)}` as const; + expect( + parseImageReference(`registry.example.com/team/image@${pinned}`).digest, + ).toBe(pinned); + expect(() => parseImageReference("image:latest")).toThrow("registry host"); + expect(() => + parseImageReference(`registry.example.com/../image@${pinned}`), + ).toThrow("Invalid image reference"); + }); + + test("resolves an anonymous manifest using the advertised digest", async () => { + const expected = `sha256:${"b".repeat(64)}` as const; + const calls: Array<{ url: string; authorization: string | null }> = []; + const fetcher = async ( + input: string | URL | Request, + init?: RequestInit, + ) => { + const headers = new Headers(init?.headers); + calls.push({ + url: String(input), + authorization: headers.get("authorization"), + }); + return new Response("manifest", { + headers: { "docker-content-digest": expected }, + }); + }; + expect( + await resolveRegistryDigest("registry.example.com/team/image:v1", { + fetch: fetcher, + }), + ).toBe(expected); + expect(calls).toEqual([ + { + url: "https://registry.example.com/v2/team/image/manifests/v1", + authorization: null, + }, + ]); + }); + + test("resolves through a trusted internal registry origin", async () => { + const expected = `sha256:${"c".repeat(64)}` as const; + let requested = ""; + await expect( + resolveRegistryDigest("registry.example.com/team/image:v1", { + origin: "http://registry.registry.svc.cluster.local:5000/", + insecure: true, + fetch: async (input) => { + requested = String(input); + return new Response("manifest", { + headers: { "docker-content-digest": expected }, + }); + }, + }), + ).resolves.toBe(expected); + expect(requested).toBe( + "http://registry.registry.svc.cluster.local:5000/v2/team/image/manifests/v1", + ); + }); + + test("follows a standard bearer challenge and hashes a digest-less response", async () => { + const body = '{"schemaVersion":2}'; + const expected = + `sha256:${createHash("sha256").update(body).digest("hex")}` as const; + const calls: Array<{ url: string; authorization: string | null }> = []; + const fetcher = async ( + input: string | URL | Request, + init?: RequestInit, + ) => { + const url = String(input); + const authorization = new Headers(init?.headers).get("authorization"); + calls.push({ url, authorization }); + if (url.startsWith("https://auth.example/token")) { + expect(new URL(url).searchParams.get("scope")).toBe( + "repository:team/image:pull", + ); + expect(authorization).toBe( + `Basic ${Buffer.from("user:pass").toString("base64")}`, + ); + return Response.json({ access_token: "registry-token" }); + } + if (authorization !== "Bearer registry-token") { + return new Response("unauthorized", { + status: 401, + headers: { + "www-authenticate": + 'Bearer realm="https://auth.example/token",service="registry.example.com"', + }, + }); + } + return new Response(body, { + headers: { + "content-type": "application/vnd.oci.image.manifest.v1+json", + }, + }); + }; + + expect( + await resolveRegistryDigest("registry.example.com/team/image:v2", { + fetch: fetcher, + credentials: { username: "user", password: "pass" }, + }), + ).toBe(expected); + expect(calls).toHaveLength(3); + expect(calls[2]?.authorization).toBe("Bearer registry-token"); + }); + + test("rejects unsupported challenges and malformed advertised digests", async () => { + const unauthorized = async () => + new Response("no", { + status: 401, + headers: { "www-authenticate": 'Basic realm="registry"' }, + }); + await expect( + resolveRegistryDigest("registry.example.com/team/image", { + fetch: unauthorized, + }), + ).rejects.toThrow("401"); + + const malformed = async () => + new Response("body", { + headers: { "docker-content-digest": "sha256:bad" }, + }); + await expect( + resolveRegistryDigest("registry.example.com/team/image", { + fetch: malformed, + }), + ).rejects.toThrow("Invalid SHA-256"); + }); +}); diff --git a/tests/server/workspace-store.test.ts b/tests/server/workspace-store.test.ts new file mode 100644 index 0000000..c0e8ac2 --- /dev/null +++ b/tests/server/workspace-store.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "bun:test"; +import { + MAX_WORKSPACE_CONFIG_BYTES, + MemoryWorkspaceStore, + WorkspaceConflictError, + WorkspaceValidationError, + workspaceEtag, +} from "../../server/workspace-store"; + +const source = { uri: "oci://registry.example/app", digest: "sha256:abc" }; + +describe("workspace store", () => { + test("validates IDs and reserved namespaces", async () => { + const store = new MemoryWorkspaceStore(); + for (const id of [ + "Upper", + "has_dot", + "kube-public", + "kuber-system", + "database", + "garage-system", + "routing", + ]) { + expect(store.create({ id, source })).rejects.toBeInstanceOf( + WorkspaceValidationError, + ); + } + }); + + test("creates immutable revisions and uses ETags for replacement", async () => { + let uid = 0; + const store = new MemoryWorkspaceStore({ + now: () => new Date("2026-09-02T00:00:00.000Z"), + uid: () => `uid-${++uid}`, + }); + const created = await store.create({ + id: "demo", + source, + config: { a: 1 }, + }); + expect(created.metadata.uid).toBe("uid-1"); + expect(workspaceEtag(created)).toBe('"1"'); + + const updated = await store.update( + "demo", + { source: { ...source, digest: "sha256:def" }, config: { a: 2 } }, + workspaceEtag(created), + ); + expect(updated.metadata.resourceVersion).toBe("2"); + expect(updated.status.latestRevision).toBe(2); + expect((await store.getRevision("demo", 1))?.spec.config).toEqual({ a: 1 }); + expect((await store.getRevision("demo", 2))?.spec.config).toEqual({ a: 2 }); + expect(store.update("demo", { source }, '"1"')).rejects.toBeInstanceOf( + WorkspaceConflictError, + ); + }); + + test("rejects inline source and oversized config payloads", async () => { + const store = new MemoryWorkspaceStore(); + expect( + store.create({ + id: "inline", + source: { ...source, content: "source" } as typeof source, + }), + ).rejects.toBeInstanceOf(WorkspaceValidationError); + expect( + store.create({ + id: "large", + source, + config: "x".repeat(MAX_WORKSPACE_CONFIG_BYTES), + }), + ).rejects.toBeInstanceOf(WorkspaceValidationError); + }); +}); diff --git a/tests/shared/build-protocol.test.ts b/tests/shared/build-protocol.test.ts new file mode 100644 index 0000000..dc7c169 --- /dev/null +++ b/tests/shared/build-protocol.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, test } from "bun:test"; +import { + assertSha256Digest, + BUILD_PROTOCOL_VERSION, + isSha256Digest, + type BuildRequest, +} from "../../shared/build-protocol"; + +describe("build protocol", () => { + test("exposes a serializable typed request", () => { + const request: BuildRequest = { + version: BUILD_PROTOCOL_VERSION, + id: "build-1", + project: "demo", + service: "web", + spec: { + architecture: "arm64", + image: "registry.example/demo/web:latest", + context: ".", + buildArgs: ["NODE_ENV=production"], + workspace: `sha256:${"a".repeat(64)}`, + }, + }; + expect(JSON.parse(JSON.stringify(request))).toEqual(request); + }); + + test("strictly validates lowercase sha256 digests", () => { + expect(isSha256Digest(`sha256:${"f".repeat(64)}`)).toBe(true); + expect(isSha256Digest(`sha256:${"F".repeat(64)}`)).toBe(false); + expect(() => assertSha256Digest("sha256:short")).toThrow("Invalid SHA-256"); + }); +}); diff --git a/types.d.ts b/types.d.ts index 6263b48..7590aa6 100644 --- a/types.d.ts +++ b/types.d.ts @@ -30,12 +30,6 @@ export interface KuberResource { [key: string]: unknown; } -export interface KuberBuildersConfig { - amd64?: string; - arm64?: string; - remoteRoot?: string; -} - export interface KuberConfig { /** Kubernetes namespace/project name. Overrides the Compose name and working directory. */ project?: string; @@ -43,7 +37,6 @@ export interface KuberConfig { composeFile?: string; /** Container registry hostname or path, without a trailing slash. */ registry?: string; - builders?: KuberBuildersConfig; /** Maximum time to wait for each deployment rollout. */ rolloutTimeoutMs?: number;