feat: experimental s3 support

README.md

@@ -18,6 +18,7 @@ It reads a local Compose file, renders Kubernetes resources, applies them to the
   - ignored `.env*` files
 - Supports host-based `ports` syntax that renders Kubernetes `Ingress` rules
 - Supports managed Postgres claims through special pseudo-volumes such as `postgresql:app`
+- Supports managed S3 buckets and credentials through pseudo-volumes such as `s3:app`
 
 ## Environment Assumptions
 
@@ -63,8 +64,8 @@ bun run index.ts db ls
 - `start`: same as `up` but skips image builds
 - `stop`: scale managed deployments to zero
 - `restart`: roll out a restart across managed deployments
-- `down`: delete managed resources while keeping ingress, PVCs, and managed databases
-- `down -f`: also delete ingress, PVCs, managed database resources, and the namespace
+- `down`: delete managed resources while keeping ingress, PVCs, managed databases, and managed S3 storage
+- `down -f`: also delete ingress, PVCs, managed database and S3 resources, and the namespace
 - `ps`: list deployments in the current project namespace
 - `logs [deployment]`: print logs for one deployment or all managed deployments
 - `logs -f [deployment]`: follow logs continuously
@@ -129,11 +130,41 @@ services:
 
 This creates or reuses the managed CNPG role secret, reconciles the database resource, and injects `DATABASE_URL` into the generated app secret in Kubernetes.
 
+### Managed S3
+
+Declare a Garage S3 bucket and access key with a pseudo-volume:
+
+```yml
+services:
+  app:
+    volumes:
+      - s3:app
+```
+
+This creates `GarageBucket/app` and `GarageKey/app` in `garage-system`. To use different key and bucket names:
+
+```yml
+services:
+  app:
+    volumes:
+      - s3:app-key/shared-assets
+```
+
+The Garage operator generates the credentials. `kuber` reads its generated Secret and injects these values into the service's `<service>-env` Secret:
+
+- `AWS_ACCESS_KEY_ID`
+- `AWS_SECRET_ACCESS_KEY`
+- `AWS_ENDPOINT_URL_S3`
+- `AWS_REGION`
+- `S3_BUCKET`
+
+One service can declare both `postgresql:...` and `s3:...`; all generated values are merged into the same service Secret. Managed Garage buckets and keys are retained by normal `down` and deleted by `down -f`.
+
 ### Environment Files
 
 `env_file` entries are read locally and turned into a Kubernetes `Secret` named `<service>-env`. Deployments then consume that secret through `envFrom`.
 
-This is also where generated values such as `DATABASE_URL` are injected.
+This is also where generated values such as `DATABASE_URL` and the managed S3 environment are injected.
 
 ### Volumes
 
@@ -144,6 +175,7 @@ This is also where generated values such as `DATABASE_URL` are injected.
 - named volumes become PVC-backed mounts
 - `tmpfs` becomes `emptyDir` with memory backing
 - `postgresql:...` is treated as a managed database claim, not as a filesystem mount
+- `s3:...` is treated as a managed object-storage claim, not as a filesystem mount
 
 Named volumes also support kuber-specific storage hints.
 

command/down.ts

@@ -8,6 +8,7 @@ import {
   sortResources,
 } from "../lib/apply";
 import { listManagedDatabaseResources } from "../lib/database";
+import { listManagedStorageResources } from "../lib/storage";
 
 export const down = defineCommand({
   meta: {
@@ -33,6 +34,7 @@ export const down = defineCommand({
 
     if (args.full) {
       resources.push(...(await listManagedDatabaseResources(project)));
+      resources.push(...(await listManagedStorageResources(project)));
     }
 
     if (args.full) {

command/up.ts

@@ -15,6 +15,7 @@ import {
   getComposePostgresClaims,
   reconcilePostgresClaims,
 } from "../lib/database";
+import { getComposeS3Claims, reconcileS3Claims } from "../lib/storage";
 import { restartDeployment, waitForDeploymentRollout } from "../lib/shared";
 
 type UpContext = {
@@ -24,6 +25,17 @@ type UpContext = {
   staleResources?: KubernetesResource[];
 };
 
+function mergeServiceEnv(
+  current: Record<string, Record<string, string>> | undefined,
+  next: Record<string, Record<string, string>>,
+): Record<string, Record<string, string>> {
+  const merged = { ...current };
+  for (const [service, environment] of Object.entries(next)) {
+    merged[service] = { ...merged[service], ...environment };
+  }
+  return merged;
+}
+
 function getDeploymentNames(resources: KubernetesResource[]): string[] {
   return resources
     .filter((resource) => resource.kind === "Deployment")
@@ -74,9 +86,23 @@ export async function runUp(build: boolean) {
             ? false
             : "No managed postgres volumes",
         task: async (taskCtx, task) => {
-          taskCtx.serviceEnv = await reconcilePostgresClaims(
-            project,
-            taskCtx.compose!,
+          taskCtx.serviceEnv = mergeServiceEnv(
+            taskCtx.serviceEnv,
+            await reconcilePostgresClaims(project, taskCtx.compose!),
+          );
+          task.output = `${Object.keys(taskCtx.serviceEnv).length} service${Object.keys(taskCtx.serviceEnv).length === 1 ? "" : "s"}`;
+        },
+      },
+      {
+        title: "Reconcile S3 storage",
+        skip: (taskCtx) =>
+          getComposeS3Claims(taskCtx.compose!).length > 0
+            ? false
+            : "No managed S3 volumes",
+        task: async (taskCtx, task) => {
+          taskCtx.serviceEnv = mergeServiceEnv(
+            taskCtx.serviceEnv,
+            await reconcileS3Claims(project, taskCtx.compose!),
           );
           task.output = `${Object.keys(taskCtx.serviceEnv).length} service${Object.keys(taskCtx.serviceEnv).length === 1 ? "" : "s"}`;
         },

lib/apply.ts

@@ -7,13 +7,15 @@ const FIELD_MANAGER = "kuber";
 
 const ResourceOrder = {
   Namespace: 0,
-  PersistentVolumeClaim: 1,
-  Secret: 2,
-  ConfigMap: 3,
-  Service: 4,
-  Deployment: 5,
-  Ingress: 6,
-  IngressRoute: 7,
+  GarageBucket: 1,
+  GarageKey: 2,
+  PersistentVolumeClaim: 3,
+  Secret: 4,
+  ConfigMap: 5,
+  Service: 6,
+  Deployment: 7,
+  Ingress: 8,
+  IngressRoute: 9,
 } as const;
 
 const ManagedResources = [

lib/convert.ts

@@ -20,9 +20,10 @@ import type { ComposeSpecification } from "../schema/docker.d";
 import type { Service } from "../schema/docker.d";
 import { IMAGE_REGISTRY, LABELS } from "../const";
 import { getComposeArchPlacement } from "./arch";
-import { getServicePostgresClaim, isPostgresVolumeEntry } from "./database";
+import { isPostgresVolumeEntry } from "./database";
 import { toEnvVars } from "./format";
 import { deepMerge } from "./shared";
+import { isS3VolumeEntry } from "./storage";
 
 type NormalizedMount = {
   name: string;
@@ -269,7 +270,7 @@ function parseStringMount(
   index: number,
   cwd: string,
 ): NormalizedMount | undefined {
-  if (isPostgresVolumeEntry(entry)) return;
+  if (isPostgresVolumeEntry(entry) || isS3VolumeEntry(entry)) return;
 
   const parts = entry.split(":");
 
@@ -841,9 +842,7 @@ export function serviceToDeployment(
   const mounts = toMounts(service, cwd, volumes);
   const ports = toPorts(service);
   const hasEnvSecret =
-    Boolean(service.env_file) ||
-    Object.keys(extraEnv).length > 0 ||
-    Boolean(getServicePostgresClaim(name, service));
+    Boolean(service.env_file) || Object.keys(extraEnv).length > 0;
 
   return deepMerge(
     {

lib/render.ts

@@ -1,12 +1,18 @@
 import type { ComposeSpecification } from "../schema/docker.d";
 import { composeToKubernetes, type KubernetesResource } from "./convert";
 import { reconcilePostgresClaims } from "./database";
+import { reconcileS3Claims } from "./storage";
 
 export async function renderResources(
   project: string,
   compose: ComposeSpecification,
   cwd = process.cwd(),
 ): Promise<KubernetesResource[]> {
-  const serviceEnv = await reconcilePostgresClaims(project, compose);
+  const postgresEnv = await reconcilePostgresClaims(project, compose);
+  const s3Env = await reconcileS3Claims(project, compose);
+  const serviceEnv = { ...postgresEnv };
+  for (const [service, environment] of Object.entries(s3Env)) {
+    serviceEnv[service] = { ...serviceEnv[service], ...environment };
+  }
   return composeToKubernetes(project, compose, cwd, serviceEnv);
 }

lib/storage.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, test } from "bun:test";
+import type { ComposeSpecification, Service } from "../schema/docker.d";
+import { serviceToDeployment, volumesToPvc } from "./convert";
+import {
+  getComposeS3Claims,
+  getServiceS3Claim,
+  isS3VolumeEntry,
+} from "./storage";
+
+describe("managed S3 claims", () => {
+  test("uses the same key and bucket for the short syntax", () => {
+    expect(
+      getServiceS3Claim("app", { volumes: ["s3:assets"] } as Service),
+    ).toEqual({
+      service: "app",
+      key: "assets",
+      bucket: "assets",
+    });
+  });
+
+  test("supports explicit key and bucket names", () => {
+    expect(
+      getServiceS3Claim("app", {
+        volumes: ["s3:app-key/shared-assets"],
+      } as Service),
+    ).toEqual({
+      service: "app",
+      key: "app-key",
+      bucket: "shared-assets",
+    });
+  });
+
+  test("rejects malformed and duplicate declarations", () => {
+    expect(() =>
+      getServiceS3Claim("app", { volumes: ["s3:"] } as Service),
+    ).toThrow("Use s3:<name> or s3:<key>/<bucket>");
+    expect(() =>
+      getServiceS3Claim("app", {
+        volumes: ["s3:one", "s3:two"],
+      } as Service),
+    ).toThrow("declares multiple S3 volumes");
+  });
+
+  test("rejects one key targeting different buckets", () => {
+    const compose = {
+      services: {
+        app: { volumes: ["s3:shared/one"] },
+        worker: { volumes: ["s3:shared/two"] },
+      },
+    } as ComposeSpecification;
+
+    expect(() => getComposeS3Claims(compose)).toThrow(
+      "S3 key shared is claimed for both one and two",
+    );
+  });
+
+  test("does not render S3 declarations as filesystem volumes", () => {
+    const service = { image: "example", volumes: ["s3:assets"] } as Service;
+
+    expect(isS3VolumeEntry("s3:assets")).toBe(true);
+    expect(volumesToPvc("project", service)).toEqual([]);
+
+    const deployment = serviceToDeployment(
+      "project",
+      "app",
+      { services: { app: service } } as ComposeSpecification,
+      service,
+      process.cwd(),
+      { AWS_ACCESS_KEY_ID: "test" },
+    );
+    expect(deployment.spec?.template.spec?.volumes).toBeUndefined();
+    expect(deployment.spec?.template.spec?.containers[0]?.envFrom).toEqual([
+      { secretRef: { name: "app-env" } },
+    ]);
+  });
+});

lib/storage.ts

@@ -0,0 +1,344 @@
+import type { KubernetesObject, V1Secret } from "@kubernetes/client-node";
+import type { ComposeSpecification, Service } from "../schema/docker.d";
+import { LABELS } from "../const";
+import { applyResource } from "./apply";
+import { objectApi } from "./k8s";
+
+export const STORAGE_NAMESPACE = "garage-system";
+export const STORAGE_CLUSTER = "garage";
+export const STORAGE_PROJECT_LABEL = "kuber.dev/project";
+export const STORAGE_SERVICE_LABEL = "kuber.dev/service";
+
+const GARAGE_API_VERSION = "garage.rajsingh.info/v1beta1";
+const SECRET_WAIT_TIMEOUT_MS = 60_000;
+const SECRET_WAIT_INTERVAL_MS = 1_000;
+
+export type S3Claim = {
+  service: string;
+  key: string;
+  bucket: string;
+};
+
+type GarageKeyStatus = {
+  status?: {
+    phase?: string;
+    secretRef?: {
+      name?: string;
+      namespace?: string;
+    };
+  };
+};
+
+function decodeSecretValue(value: string | undefined): string | undefined {
+  return value ? Buffer.from(value, "base64").toString("utf8") : undefined;
+}
+
+async function readObject<T>(
+  resource: KubernetesObject,
+): Promise<T | undefined> {
+  try {
+    return (await objectApi.read(resource as never)) as T;
+  } catch (error) {
+    if (
+      error &&
+      typeof error === "object" &&
+      "code" in error &&
+      error.code === 404
+    ) {
+      return;
+    }
+
+    throw error;
+  }
+}
+
+function parseS3VolumeString(
+  entry: string,
+): Omit<S3Claim, "service"> | undefined {
+  if (!entry.startsWith("s3:")) return;
+
+  const parts = entry.split(":");
+  if (parts.length !== 2) {
+    throw new Error(
+      `Invalid S3 volume ${entry}. Use s3:<name> or s3:<key>/<bucket>.`,
+    );
+  }
+
+  const target = parts[1]?.trim();
+  if (!target) {
+    throw new Error(
+      `Invalid S3 volume ${entry}. Use s3:<name> or s3:<key>/<bucket>.`,
+    );
+  }
+
+  const segments = target.split("/");
+  if (
+    segments.length > 2 ||
+    segments.some((segment) => segment.trim() === "")
+  ) {
+    throw new Error(
+      `Invalid S3 volume ${entry}. Use s3:<name> or s3:<key>/<bucket>.`,
+    );
+  }
+
+  const key = segments[0]!;
+  const bucket = segments[1] ?? key;
+  return { key, bucket };
+}
+
+export function isS3VolumeEntry(
+  entry: NonNullable<Service["volumes"]>[number],
+): boolean {
+  return typeof entry === "string" && parseS3VolumeString(entry) !== undefined;
+}
+
+export function getServiceS3Claim(
+  serviceName: string,
+  service: Service,
+): S3Claim | undefined {
+  const claims =
+    service.volumes?.flatMap((entry) => {
+      if (typeof entry !== "string") return [];
+
+      const claim = parseS3VolumeString(entry);
+      return claim
+        ? [{ ...claim, service: serviceName } satisfies S3Claim]
+        : [];
+    }) ?? [];
+
+  if (claims.length > 1) {
+    throw new Error(
+      `Service ${serviceName} declares multiple S3 volumes. Only zero or one s3:<...> entry is allowed per service.`,
+    );
+  }
+
+  return claims[0];
+}
+
+export function getComposeS3Claims(compose: ComposeSpecification): S3Claim[] {
+  const claims = Object.entries(compose.services ?? {}).flatMap(
+    ([serviceName, service]) => {
+      const claim = getServiceS3Claim(serviceName, service);
+      return claim ? [claim] : [];
+    },
+  );
+
+  const bucketsByKey = new Map<string, string>();
+  for (const claim of claims) {
+    const bucket = bucketsByKey.get(claim.key);
+    if (bucket && bucket !== claim.bucket) {
+      throw new Error(
+        `S3 key ${claim.key} is claimed for both ${bucket} and ${claim.bucket}. A managed key can only target one bucket.`,
+      );
+    }
+
+    bucketsByKey.set(claim.key, claim.bucket);
+  }
+
+  return claims;
+}
+
+async function reconcileBuckets(
+  project: string,
+  claims: S3Claim[],
+): Promise<void> {
+  const buckets = new Map<string, S3Claim>();
+  for (const claim of claims) buckets.set(claim.bucket, claim);
+
+  for (const claim of buckets.values()) {
+    try {
+      await applyResource({
+        apiVersion: GARAGE_API_VERSION,
+        kind: "GarageBucket",
+        metadata: {
+          name: claim.bucket,
+          namespace: STORAGE_NAMESPACE,
+          labels: {
+            ...LABELS,
+            [STORAGE_PROJECT_LABEL]: project,
+            [STORAGE_SERVICE_LABEL]: claim.service,
+          },
+        },
+        spec: {
+          clusterRef: {
+            name: STORAGE_CLUSTER,
+          },
+          globalAlias: claim.bucket,
+        },
+      });
+    } catch (error) {
+      throw new Error(
+        `Failed to reconcile S3 bucket ${claim.bucket}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+}
+
+async function reconcileKeys(
+  project: string,
+  claims: S3Claim[],
+): Promise<void> {
+  const keys = new Map<string, S3Claim>();
+  for (const claim of claims) keys.set(claim.key, claim);
+
+  for (const claim of keys.values()) {
+    try {
+      await applyResource({
+        apiVersion: GARAGE_API_VERSION,
+        kind: "GarageKey",
+        metadata: {
+          name: claim.key,
+          namespace: STORAGE_NAMESPACE,
+          labels: {
+            ...LABELS,
+            [STORAGE_PROJECT_LABEL]: project,
+            [STORAGE_SERVICE_LABEL]: claim.service,
+          },
+        },
+        spec: {
+          bucketPermissions: [
+            {
+              bucketRef: {
+                name: claim.bucket,
+              },
+              owner: true,
+              read: true,
+              write: true,
+            },
+          ],
+          clusterRef: {
+            name: STORAGE_CLUSTER,
+          },
+          name: claim.key,
+          neverExpires: true,
+          secretTemplate: {
+            bucketNameKey: "bucket",
+            includeBucketName: true,
+          },
+        },
+      });
+    } catch (error) {
+      throw new Error(
+        `Failed to reconcile S3 key ${claim.key}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+}
+
+async function readKeyEnvironment(
+  claim: S3Claim,
+): Promise<Record<string, string>> {
+  const deadline = Date.now() + SECRET_WAIT_TIMEOUT_MS;
+  let lastPhase: string | undefined;
+
+  while (Date.now() < deadline) {
+    const key = await readObject<GarageKeyStatus>({
+      apiVersion: GARAGE_API_VERSION,
+      kind: "GarageKey",
+      metadata: {
+        name: claim.key,
+        namespace: STORAGE_NAMESPACE,
+      },
+    });
+
+    lastPhase = key?.status?.phase;
+
+    const secretName = key?.status?.secretRef?.name;
+    const secretNamespace =
+      key?.status?.secretRef?.namespace ?? STORAGE_NAMESPACE;
+    if (secretName) {
+      const secret = await readObject<V1Secret>({
+        apiVersion: "v1",
+        kind: "Secret",
+        metadata: {
+          name: secretName,
+          namespace: secretNamespace,
+        },
+      });
+      const accessKeyId = decodeSecretValue(secret?.data?.["access-key-id"]);
+      const secretAccessKey = decodeSecretValue(
+        secret?.data?.["secret-access-key"],
+      );
+      const endpoint = decodeSecretValue(secret?.data?.endpoint);
+      const region = decodeSecretValue(secret?.data?.region);
+      const bucket = decodeSecretValue(secret?.data?.bucket);
+
+      if (accessKeyId && secretAccessKey && endpoint && region) {
+        return {
+          AWS_ACCESS_KEY_ID: accessKeyId,
+          AWS_SECRET_ACCESS_KEY: secretAccessKey,
+          AWS_ENDPOINT_URL_S3: endpoint,
+          AWS_REGION: region,
+          S3_BUCKET: bucket ?? claim.bucket,
+        };
+      }
+    }
+
+    await new Promise((resolve) =>
+      setTimeout(resolve, SECRET_WAIT_INTERVAL_MS),
+    );
+  }
+
+  throw new Error(
+    `Timed out waiting for credentials for S3 key ${claim.key} in namespace ${STORAGE_NAMESPACE}${lastPhase ? ` (last phase: ${lastPhase})` : ""}.`,
+  );
+}
+
+export async function reconcileS3Claims(
+  project: string,
+  compose: ComposeSpecification,
+): Promise<Record<string, Record<string, string>>> {
+  const claims = getComposeS3Claims(compose);
+  if (claims.length === 0) return {};
+
+  await reconcileBuckets(project, claims);
+  await reconcileKeys(project, claims);
+
+  const environmentsByKey = new Map<string, Record<string, string>>();
+  for (const claim of claims) {
+    if (environmentsByKey.has(claim.key)) continue;
+    environmentsByKey.set(claim.key, await readKeyEnvironment(claim));
+  }
+
+  return Object.fromEntries(
+    claims.map((claim) => {
+      const environment = environmentsByKey.get(claim.key);
+      if (!environment) throw new Error(`Missing credentials for ${claim.key}`);
+      return [claim.service, environment];
+    }),
+  );
+}
+
+export async function listManagedStorageResources(
+  project: string,
+): Promise<KubernetesObject[]> {
+  const selector = `${Object.entries(LABELS)
+    .map(([key, value]) => `${key}=${value}`)
+    .join(",")},${STORAGE_PROJECT_LABEL}=${project}`;
+  const resources = await Promise.all(
+    ["GarageBucket", "GarageKey"].map(async (kind) => {
+      const result = await objectApi.list(
+        GARAGE_API_VERSION,
+        kind,
+        STORAGE_NAMESPACE,
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        selector,
+      );
+
+      return result.items.map((item) => ({
+        ...item,
+        apiVersion: item.apiVersion ?? GARAGE_API_VERSION,
+        kind: item.kind ?? kind,
+        metadata: {
+          ...item.metadata,
+          namespace: item.metadata?.namespace ?? STORAGE_NAMESPACE,
+        },
+      }));
+    }),
+  );
+
+  return resources.flat();
+}