kuber/server/build-job.ts

import type { BuildArchitecture, BuildSpec } from "../shared/build-protocol";

export type BuildJobOptions = {
  name: string;
  namespace: string;
  spec: BuildSpec;
  workspaceClaimName: string;
  workspaceSubPath?: string;
  cacheImage: string;
  pushImage?: string;
  pushImages?: string[];
  pushRegistryInsecure?: boolean;
  cacheRegistryInsecure?: boolean;
  buildkitImage?: string;
  serviceAccountName?: string;
  registrySecretName?: string;
  labels?: Record<string, string>;
  nodeSelector?: Record<string, string>;
  tolerations?: Array<Record<string, unknown>>;
  ttlSecondsAfterFinished?: number;
  backoffLimit?: number;
};

export type KubernetesJob = {
  apiVersion: "batch/v1";
  kind: "Job";
  metadata: {
    name: string;
    namespace: string;
    labels: Record<string, string>;
    annotations: Record<string, string>;
  };
  spec: Record<string, unknown>;
};

function relativeBuildPath(path: string, name: string): string {
  const normalized = path === "." ? "" : path.replace(/^\.\//, "");
  if (
    !path ||
    path.startsWith("/") ||
    path.includes("\\") ||
    path.includes("\0") ||
    (normalized !== "" &&
      normalized
        .split("/")
        .some((part) => !part || part === ".." || part === "."))
  )
    throw new Error(`${name} must be a safe workspace-relative path`);
  return normalized;
}

function platform(architecture: BuildArchitecture): string {
  return `linux/${architecture}`;
}

export function createBuildJob(options: BuildJobOptions): KubernetesJob {
  const contextPath = relativeBuildPath(options.spec.context, "Build context");
  const dockerfilePath = options.spec.dockerfile
    ? relativeBuildPath(options.spec.dockerfile, "Dockerfile")
    : undefined;
  const workspaceSubPath = options.workspaceSubPath
    ? relativeBuildPath(options.workspaceSubPath, "Workspace subPath")
    : undefined;
  if (
    !/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(options.name) ||
    options.name.length > 63
  )
    throw new Error("Job name must be a valid DNS label");

  const workspace = "/workspace";
  const context = contextPath ? `${workspace}/${contextPath}` : workspace;
  const dockerfile = dockerfilePath
    ? `${workspace}/${dockerfilePath}`
    : `${context}/Dockerfile`;
  const outputImage = options.pushImage ?? options.spec.image;
  const outputImages = [outputImage, ...(options.pushImages ?? [])];
  if (outputImages.some((image) => !image || /[,"\r\n]/.test(image)))
    throw new Error("Invalid BuildKit output image name");
  const importCacheInsecure = options.cacheRegistryInsecure
    ? ",registry.insecure=true"
    : "";
  const exportCacheInsecure = options.cacheRegistryInsecure
    ? ",registry.insecure=true"
    : "";
  const outputInsecure = options.pushRegistryInsecure
    ? ",registry.insecure=true"
    : "";
  const args = [
    "build",
    "--frontend=dockerfile.v0",
    `--local=context=${context}`,
    `--local=dockerfile=${dockerfile.slice(0, dockerfile.lastIndexOf("/"))}`,
    `--opt=filename=${dockerfile.slice(dockerfile.lastIndexOf("/") + 1)}`,
    `--opt=platform=${platform(options.spec.architecture)}`,
    ...(options.spec.target ? [`--opt=target=${options.spec.target}`] : []),
    ...options.spec.buildArgs.map((arg) => `--opt=build-arg:${arg}`),
    `--import-cache=type=registry,ref=${options.cacheImage}${importCacheInsecure}`,
    `--export-cache=type=registry,ref=${options.cacheImage},mode=max${exportCacheInsecure}`,
    `--output=type=image,${outputImages.length === 1 ? `name=${outputImage}` : `"name=${outputImages.join(",")}"`},push=true${outputInsecure}`,
  ];
  const labels = {
    "app.kubernetes.io/name": "kuber-buildkit",
    "app.kubernetes.io/managed-by": "kuber",
    "kuber.astrxl.dev/build": options.name,
    ...options.labels,
  };
  const amd64Toleration = {
    key: "arch",
    operator: "Equal",
    value: "amd64",
    effect: "NoExecute",
  };
  const tolerations = [...(options.tolerations ?? [])];
  if (
    options.spec.architecture === "amd64" &&
    !tolerations.some(
      (toleration) =>
        toleration.key === amd64Toleration.key &&
        toleration.operator === amd64Toleration.operator &&
        toleration.value === amd64Toleration.value &&
        toleration.effect === amd64Toleration.effect &&
        Object.keys(toleration).length === Object.keys(amd64Toleration).length,
    )
  )
    tolerations.push(amd64Toleration);

  return {
    apiVersion: "batch/v1",
    kind: "Job",
    metadata: {
      name: options.name,
      namespace: options.namespace,
      labels,
      annotations: {
        "container.apparmor.security.beta.kubernetes.io/buildkit": "unconfined",
        "kuber.astrxl.dev/workspace": options.spec.workspace,
      },
    },
    spec: {
      backoffLimit: options.backoffLimit ?? 0,
      ttlSecondsAfterFinished: options.ttlSecondsAfterFinished ?? 3600,
      template: {
        metadata: {
          labels,
          annotations: {
            "container.apparmor.security.beta.kubernetes.io/buildkit":
              "unconfined",
          },
        },
        spec: {
          restartPolicy: "Never",
          ...(options.serviceAccountName
            ? { serviceAccountName: options.serviceAccountName }
            : {}),
          automountServiceAccountToken: false,
          nodeSelector: {
            ...options.nodeSelector,
            "kubernetes.io/arch": options.spec.architecture,
          },
          ...(tolerations.length ? { tolerations } : {}),
          securityContext: {
            runAsNonRoot: true,
            runAsUser: 1000,
            runAsGroup: 1000,
            fsGroup: 1000,
            seccompProfile: { type: "Unconfined" },
          },
          ...(options.registrySecretName
            ? { imagePullSecrets: [{ name: options.registrySecretName }] }
            : {}),
          containers: [
            {
              name: "buildkit",
              image: options.buildkitImage ?? "moby/buildkit:rootless",
              imagePullPolicy: "IfNotPresent",
              command: ["buildctl-daemonless.sh"],
              args,
              env: [
                {
                  name: "BUILDKITD_FLAGS",
                  value: "--oci-worker-no-process-sandbox",
                },
                ...(options.registrySecretName
                  ? [{ name: "DOCKER_CONFIG", value: "/docker-config" }]
                  : []),
              ],
              securityContext: {
                runAsNonRoot: true,
                runAsUser: 1000,
                allowPrivilegeEscalation: true,
                seccompProfile: { type: "Unconfined" },
                appArmorProfile: { type: "Unconfined" },
              },
              volumeMounts: [
                {
                  name: "workspace",
                  mountPath: workspace,
                  readOnly: true,
                  ...(workspaceSubPath ? { subPath: workspaceSubPath } : {}),
                },
                {
                  name: "buildkit-state",
                  mountPath: "/home/user/.local/share/buildkit",
                },
                ...(options.registrySecretName
                  ? [
                      {
                        name: "registry-auth",
                        mountPath: "/docker-config",
                        readOnly: true,
                      },
                    ]
                  : []),
              ],
            },
          ],
          volumes: [
            {
              name: "workspace",
              persistentVolumeClaim: { claimName: options.workspaceClaimName },
            },
            { name: "buildkit-state", emptyDir: {} },
            ...(options.registrySecretName
              ? [
                  {
                    name: "registry-auth",
                    secret: {
                      secretName: options.registrySecretName,
                      items: [
                        { key: ".dockerconfigjson", path: "config.json" },
                      ],
                    },
                  },
                ]
              : []),
          ],
        },
      },
    },
  };
}