From a1705487f2f6b37c4937965f623747cb0daa4913 Mon Sep 17 00:00:00 2001 From: dmgnr Date: Fri, 28 Aug 2026 22:07:40 +0700 Subject: [PATCH] feat: experimental s3 support --- README.md | 38 ++++- command/down.ts | 2 + command/up.ts | 32 ++++- lib/apply.ts | 16 ++- lib/convert.ts | 9 +- lib/render.ts | 8 +- lib/storage.test.ts | 76 ++++++++++ lib/storage.ts | 344 ++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 506 insertions(+), 19 deletions(-) create mode 100644 lib/storage.test.ts create mode 100644 lib/storage.ts diff --git a/README.md b/README.md index a72e754..acc2756 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ It reads a local Compose file, renders Kubernetes resources, applies them to the - ignored `.env*` files - Supports host-based `ports` syntax that renders Kubernetes `Ingress` rules - Supports managed Postgres claims through special pseudo-volumes such as `postgresql:app` +- Supports managed S3 buckets and credentials through pseudo-volumes such as `s3:app` ## Environment Assumptions @@ -63,8 +64,8 @@ bun run index.ts db ls - `start`: same as `up` but skips image builds - `stop`: scale managed deployments to zero - `restart`: roll out a restart across managed deployments -- `down`: delete managed resources while keeping ingress, PVCs, and managed databases -- `down -f`: also delete ingress, PVCs, managed database resources, and the namespace +- `down`: delete managed resources while keeping ingress, PVCs, managed databases, and managed S3 storage +- `down -f`: also delete ingress, PVCs, managed database and S3 resources, and the namespace - `ps`: list deployments in the current project namespace - `logs [deployment]`: print logs for one deployment or all managed deployments - `logs -f [deployment]`: follow logs continuously @@ -129,11 +130,41 @@ services: This creates or reuses the managed CNPG role secret, reconciles the database resource, and injects `DATABASE_URL` into the generated app secret in Kubernetes. +### Managed S3 + +Declare a Garage S3 bucket and access key with a pseudo-volume: + +```yml +services: + app: + volumes: + - s3:app +``` + +This creates `GarageBucket/app` and `GarageKey/app` in `garage-system`. To use different key and bucket names: + +```yml +services: + app: + volumes: + - s3:app-key/shared-assets +``` + +The Garage operator generates the credentials. `kuber` reads its generated Secret and injects these values into the service's `-env` Secret: + +- `AWS_ACCESS_KEY_ID` +- `AWS_SECRET_ACCESS_KEY` +- `AWS_ENDPOINT_URL_S3` +- `AWS_REGION` +- `S3_BUCKET` + +One service can declare both `postgresql:...` and `s3:...`; all generated values are merged into the same service Secret. Managed Garage buckets and keys are retained by normal `down` and deleted by `down -f`. + ### Environment Files `env_file` entries are read locally and turned into a Kubernetes `Secret` named `-env`. Deployments then consume that secret through `envFrom`. -This is also where generated values such as `DATABASE_URL` are injected. +This is also where generated values such as `DATABASE_URL` and the managed S3 environment are injected. ### Volumes @@ -144,6 +175,7 @@ This is also where generated values such as `DATABASE_URL` are injected. - named volumes become PVC-backed mounts - `tmpfs` becomes `emptyDir` with memory backing - `postgresql:...` is treated as a managed database claim, not as a filesystem mount +- `s3:...` is treated as a managed object-storage claim, not as a filesystem mount Named volumes also support kuber-specific storage hints. diff --git a/command/down.ts b/command/down.ts index c31f925..8d0901a 100644 --- a/command/down.ts +++ b/command/down.ts @@ -8,6 +8,7 @@ import { sortResources, } from "../lib/apply"; import { listManagedDatabaseResources } from "../lib/database"; +import { listManagedStorageResources } from "../lib/storage"; export const down = defineCommand({ meta: { @@ -33,6 +34,7 @@ export const down = defineCommand({ if (args.full) { resources.push(...(await listManagedDatabaseResources(project))); + resources.push(...(await listManagedStorageResources(project))); } if (args.full) { diff --git a/command/up.ts b/command/up.ts index 5d15391..8e73bc4 100644 --- a/command/up.ts +++ b/command/up.ts @@ -15,6 +15,7 @@ import { getComposePostgresClaims, reconcilePostgresClaims, } from "../lib/database"; +import { getComposeS3Claims, reconcileS3Claims } from "../lib/storage"; import { restartDeployment, waitForDeploymentRollout } from "../lib/shared"; type UpContext = { @@ -24,6 +25,17 @@ type UpContext = { staleResources?: KubernetesResource[]; }; +function mergeServiceEnv( + current: Record> | undefined, + next: Record>, +): Record> { + const merged = { ...current }; + for (const [service, environment] of Object.entries(next)) { + merged[service] = { ...merged[service], ...environment }; + } + return merged; +} + function getDeploymentNames(resources: KubernetesResource[]): string[] { return resources .filter((resource) => resource.kind === "Deployment") @@ -74,9 +86,23 @@ export async function runUp(build: boolean) { ? false : "No managed postgres volumes", task: async (taskCtx, task) => { - taskCtx.serviceEnv = await reconcilePostgresClaims( - project, - taskCtx.compose!, + taskCtx.serviceEnv = mergeServiceEnv( + taskCtx.serviceEnv, + await reconcilePostgresClaims(project, taskCtx.compose!), + ); + task.output = `${Object.keys(taskCtx.serviceEnv).length} service${Object.keys(taskCtx.serviceEnv).length === 1 ? "" : "s"}`; + }, + }, + { + title: "Reconcile S3 storage", + skip: (taskCtx) => + getComposeS3Claims(taskCtx.compose!).length > 0 + ? false + : "No managed S3 volumes", + task: async (taskCtx, task) => { + taskCtx.serviceEnv = mergeServiceEnv( + taskCtx.serviceEnv, + await reconcileS3Claims(project, taskCtx.compose!), ); task.output = `${Object.keys(taskCtx.serviceEnv).length} service${Object.keys(taskCtx.serviceEnv).length === 1 ? "" : "s"}`; }, diff --git a/lib/apply.ts b/lib/apply.ts index fff84ec..c9e8b30 100644 --- a/lib/apply.ts +++ b/lib/apply.ts @@ -7,13 +7,15 @@ const FIELD_MANAGER = "kuber"; const ResourceOrder = { Namespace: 0, - PersistentVolumeClaim: 1, - Secret: 2, - ConfigMap: 3, - Service: 4, - Deployment: 5, - Ingress: 6, - IngressRoute: 7, + GarageBucket: 1, + GarageKey: 2, + PersistentVolumeClaim: 3, + Secret: 4, + ConfigMap: 5, + Service: 6, + Deployment: 7, + Ingress: 8, + IngressRoute: 9, } as const; const ManagedResources = [ diff --git a/lib/convert.ts b/lib/convert.ts index eee295f..78404ad 100644 --- a/lib/convert.ts +++ b/lib/convert.ts @@ -20,9 +20,10 @@ import type { ComposeSpecification } from "../schema/docker.d"; import type { Service } from "../schema/docker.d"; import { IMAGE_REGISTRY, LABELS } from "../const"; import { getComposeArchPlacement } from "./arch"; -import { getServicePostgresClaim, isPostgresVolumeEntry } from "./database"; +import { isPostgresVolumeEntry } from "./database"; import { toEnvVars } from "./format"; import { deepMerge } from "./shared"; +import { isS3VolumeEntry } from "./storage"; type NormalizedMount = { name: string; @@ -269,7 +270,7 @@ function parseStringMount( index: number, cwd: string, ): NormalizedMount | undefined { - if (isPostgresVolumeEntry(entry)) return; + if (isPostgresVolumeEntry(entry) || isS3VolumeEntry(entry)) return; const parts = entry.split(":"); @@ -841,9 +842,7 @@ export function serviceToDeployment( const mounts = toMounts(service, cwd, volumes); const ports = toPorts(service); const hasEnvSecret = - Boolean(service.env_file) || - Object.keys(extraEnv).length > 0 || - Boolean(getServicePostgresClaim(name, service)); + Boolean(service.env_file) || Object.keys(extraEnv).length > 0; return deepMerge( { diff --git a/lib/render.ts b/lib/render.ts index 2a73583..19564f3 100644 --- a/lib/render.ts +++ b/lib/render.ts @@ -1,12 +1,18 @@ import type { ComposeSpecification } from "../schema/docker.d"; import { composeToKubernetes, type KubernetesResource } from "./convert"; import { reconcilePostgresClaims } from "./database"; +import { reconcileS3Claims } from "./storage"; export async function renderResources( project: string, compose: ComposeSpecification, cwd = process.cwd(), ): Promise { - const serviceEnv = await reconcilePostgresClaims(project, compose); + const postgresEnv = await reconcilePostgresClaims(project, compose); + const s3Env = await reconcileS3Claims(project, compose); + const serviceEnv = { ...postgresEnv }; + for (const [service, environment] of Object.entries(s3Env)) { + serviceEnv[service] = { ...serviceEnv[service], ...environment }; + } return composeToKubernetes(project, compose, cwd, serviceEnv); } diff --git a/lib/storage.test.ts b/lib/storage.test.ts new file mode 100644 index 0000000..690a351 --- /dev/null +++ b/lib/storage.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, test } from "bun:test"; +import type { ComposeSpecification, Service } from "../schema/docker.d"; +import { serviceToDeployment, volumesToPvc } from "./convert"; +import { + getComposeS3Claims, + getServiceS3Claim, + isS3VolumeEntry, +} from "./storage"; + +describe("managed S3 claims", () => { + test("uses the same key and bucket for the short syntax", () => { + expect( + getServiceS3Claim("app", { volumes: ["s3:assets"] } as Service), + ).toEqual({ + service: "app", + key: "assets", + bucket: "assets", + }); + }); + + test("supports explicit key and bucket names", () => { + expect( + getServiceS3Claim("app", { + volumes: ["s3:app-key/shared-assets"], + } as Service), + ).toEqual({ + service: "app", + key: "app-key", + bucket: "shared-assets", + }); + }); + + test("rejects malformed and duplicate declarations", () => { + expect(() => + getServiceS3Claim("app", { volumes: ["s3:"] } as Service), + ).toThrow("Use s3: or s3:/"); + expect(() => + getServiceS3Claim("app", { + volumes: ["s3:one", "s3:two"], + } as Service), + ).toThrow("declares multiple S3 volumes"); + }); + + test("rejects one key targeting different buckets", () => { + const compose = { + services: { + app: { volumes: ["s3:shared/one"] }, + worker: { volumes: ["s3:shared/two"] }, + }, + } as ComposeSpecification; + + expect(() => getComposeS3Claims(compose)).toThrow( + "S3 key shared is claimed for both one and two", + ); + }); + + test("does not render S3 declarations as filesystem volumes", () => { + const service = { image: "example", volumes: ["s3:assets"] } as Service; + + expect(isS3VolumeEntry("s3:assets")).toBe(true); + expect(volumesToPvc("project", service)).toEqual([]); + + const deployment = serviceToDeployment( + "project", + "app", + { services: { app: service } } as ComposeSpecification, + service, + process.cwd(), + { AWS_ACCESS_KEY_ID: "test" }, + ); + expect(deployment.spec?.template.spec?.volumes).toBeUndefined(); + expect(deployment.spec?.template.spec?.containers[0]?.envFrom).toEqual([ + { secretRef: { name: "app-env" } }, + ]); + }); +}); diff --git a/lib/storage.ts b/lib/storage.ts new file mode 100644 index 0000000..91aa4cd --- /dev/null +++ b/lib/storage.ts @@ -0,0 +1,344 @@ +import type { KubernetesObject, V1Secret } from "@kubernetes/client-node"; +import type { ComposeSpecification, Service } from "../schema/docker.d"; +import { LABELS } from "../const"; +import { applyResource } from "./apply"; +import { objectApi } from "./k8s"; + +export const STORAGE_NAMESPACE = "garage-system"; +export const STORAGE_CLUSTER = "garage"; +export const STORAGE_PROJECT_LABEL = "kuber.dev/project"; +export const STORAGE_SERVICE_LABEL = "kuber.dev/service"; + +const GARAGE_API_VERSION = "garage.rajsingh.info/v1beta1"; +const SECRET_WAIT_TIMEOUT_MS = 60_000; +const SECRET_WAIT_INTERVAL_MS = 1_000; + +export type S3Claim = { + service: string; + key: string; + bucket: string; +}; + +type GarageKeyStatus = { + status?: { + phase?: string; + secretRef?: { + name?: string; + namespace?: string; + }; + }; +}; + +function decodeSecretValue(value: string | undefined): string | undefined { + return value ? Buffer.from(value, "base64").toString("utf8") : undefined; +} + +async function readObject( + resource: KubernetesObject, +): Promise { + try { + return (await objectApi.read(resource as never)) as T; + } catch (error) { + if ( + error && + typeof error === "object" && + "code" in error && + error.code === 404 + ) { + return; + } + + throw error; + } +} + +function parseS3VolumeString( + entry: string, +): Omit | undefined { + if (!entry.startsWith("s3:")) return; + + const parts = entry.split(":"); + if (parts.length !== 2) { + throw new Error( + `Invalid S3 volume ${entry}. Use s3: or s3:/.`, + ); + } + + const target = parts[1]?.trim(); + if (!target) { + throw new Error( + `Invalid S3 volume ${entry}. Use s3: or s3:/.`, + ); + } + + const segments = target.split("/"); + if ( + segments.length > 2 || + segments.some((segment) => segment.trim() === "") + ) { + throw new Error( + `Invalid S3 volume ${entry}. Use s3: or s3:/.`, + ); + } + + const key = segments[0]!; + const bucket = segments[1] ?? key; + return { key, bucket }; +} + +export function isS3VolumeEntry( + entry: NonNullable[number], +): boolean { + return typeof entry === "string" && parseS3VolumeString(entry) !== undefined; +} + +export function getServiceS3Claim( + serviceName: string, + service: Service, +): S3Claim | undefined { + const claims = + service.volumes?.flatMap((entry) => { + if (typeof entry !== "string") return []; + + const claim = parseS3VolumeString(entry); + return claim + ? [{ ...claim, service: serviceName } satisfies S3Claim] + : []; + }) ?? []; + + if (claims.length > 1) { + throw new Error( + `Service ${serviceName} declares multiple S3 volumes. Only zero or one s3:<...> entry is allowed per service.`, + ); + } + + return claims[0]; +} + +export function getComposeS3Claims(compose: ComposeSpecification): S3Claim[] { + const claims = Object.entries(compose.services ?? {}).flatMap( + ([serviceName, service]) => { + const claim = getServiceS3Claim(serviceName, service); + return claim ? [claim] : []; + }, + ); + + const bucketsByKey = new Map(); + for (const claim of claims) { + const bucket = bucketsByKey.get(claim.key); + if (bucket && bucket !== claim.bucket) { + throw new Error( + `S3 key ${claim.key} is claimed for both ${bucket} and ${claim.bucket}. A managed key can only target one bucket.`, + ); + } + + bucketsByKey.set(claim.key, claim.bucket); + } + + return claims; +} + +async function reconcileBuckets( + project: string, + claims: S3Claim[], +): Promise { + const buckets = new Map(); + for (const claim of claims) buckets.set(claim.bucket, claim); + + for (const claim of buckets.values()) { + try { + await applyResource({ + apiVersion: GARAGE_API_VERSION, + kind: "GarageBucket", + metadata: { + name: claim.bucket, + namespace: STORAGE_NAMESPACE, + labels: { + ...LABELS, + [STORAGE_PROJECT_LABEL]: project, + [STORAGE_SERVICE_LABEL]: claim.service, + }, + }, + spec: { + clusterRef: { + name: STORAGE_CLUSTER, + }, + globalAlias: claim.bucket, + }, + }); + } catch (error) { + throw new Error( + `Failed to reconcile S3 bucket ${claim.bucket}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } +} + +async function reconcileKeys( + project: string, + claims: S3Claim[], +): Promise { + const keys = new Map(); + for (const claim of claims) keys.set(claim.key, claim); + + for (const claim of keys.values()) { + try { + await applyResource({ + apiVersion: GARAGE_API_VERSION, + kind: "GarageKey", + metadata: { + name: claim.key, + namespace: STORAGE_NAMESPACE, + labels: { + ...LABELS, + [STORAGE_PROJECT_LABEL]: project, + [STORAGE_SERVICE_LABEL]: claim.service, + }, + }, + spec: { + bucketPermissions: [ + { + bucketRef: { + name: claim.bucket, + }, + owner: true, + read: true, + write: true, + }, + ], + clusterRef: { + name: STORAGE_CLUSTER, + }, + name: claim.key, + neverExpires: true, + secretTemplate: { + bucketNameKey: "bucket", + includeBucketName: true, + }, + }, + }); + } catch (error) { + throw new Error( + `Failed to reconcile S3 key ${claim.key}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } +} + +async function readKeyEnvironment( + claim: S3Claim, +): Promise> { + const deadline = Date.now() + SECRET_WAIT_TIMEOUT_MS; + let lastPhase: string | undefined; + + while (Date.now() < deadline) { + const key = await readObject({ + apiVersion: GARAGE_API_VERSION, + kind: "GarageKey", + metadata: { + name: claim.key, + namespace: STORAGE_NAMESPACE, + }, + }); + + lastPhase = key?.status?.phase; + + const secretName = key?.status?.secretRef?.name; + const secretNamespace = + key?.status?.secretRef?.namespace ?? STORAGE_NAMESPACE; + if (secretName) { + const secret = await readObject({ + apiVersion: "v1", + kind: "Secret", + metadata: { + name: secretName, + namespace: secretNamespace, + }, + }); + const accessKeyId = decodeSecretValue(secret?.data?.["access-key-id"]); + const secretAccessKey = decodeSecretValue( + secret?.data?.["secret-access-key"], + ); + const endpoint = decodeSecretValue(secret?.data?.endpoint); + const region = decodeSecretValue(secret?.data?.region); + const bucket = decodeSecretValue(secret?.data?.bucket); + + if (accessKeyId && secretAccessKey && endpoint && region) { + return { + AWS_ACCESS_KEY_ID: accessKeyId, + AWS_SECRET_ACCESS_KEY: secretAccessKey, + AWS_ENDPOINT_URL_S3: endpoint, + AWS_REGION: region, + S3_BUCKET: bucket ?? claim.bucket, + }; + } + } + + await new Promise((resolve) => + setTimeout(resolve, SECRET_WAIT_INTERVAL_MS), + ); + } + + throw new Error( + `Timed out waiting for credentials for S3 key ${claim.key} in namespace ${STORAGE_NAMESPACE}${lastPhase ? ` (last phase: ${lastPhase})` : ""}.`, + ); +} + +export async function reconcileS3Claims( + project: string, + compose: ComposeSpecification, +): Promise>> { + const claims = getComposeS3Claims(compose); + if (claims.length === 0) return {}; + + await reconcileBuckets(project, claims); + await reconcileKeys(project, claims); + + const environmentsByKey = new Map>(); + for (const claim of claims) { + if (environmentsByKey.has(claim.key)) continue; + environmentsByKey.set(claim.key, await readKeyEnvironment(claim)); + } + + return Object.fromEntries( + claims.map((claim) => { + const environment = environmentsByKey.get(claim.key); + if (!environment) throw new Error(`Missing credentials for ${claim.key}`); + return [claim.service, environment]; + }), + ); +} + +export async function listManagedStorageResources( + project: string, +): Promise { + const selector = `${Object.entries(LABELS) + .map(([key, value]) => `${key}=${value}`) + .join(",")},${STORAGE_PROJECT_LABEL}=${project}`; + const resources = await Promise.all( + ["GarageBucket", "GarageKey"].map(async (kind) => { + const result = await objectApi.list( + GARAGE_API_VERSION, + kind, + STORAGE_NAMESPACE, + undefined, + undefined, + undefined, + undefined, + selector, + ); + + return result.items.map((item) => ({ + ...item, + apiVersion: item.apiVersion ?? GARAGE_API_VERSION, + kind: item.kind ?? kind, + metadata: { + ...item.metadata, + namespace: item.metadata?.namespace ?? STORAGE_NAMESPACE, + }, + })); + }), + ); + + return resources.flat(); +}