feat: v2
This commit is contained in:
+257
@@ -0,0 +1,257 @@
|
||||
import { createApp, cleanupExpiredSessions } from "./app";
|
||||
import { RedactingAuditStore } from "./audit-store";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { IMAGE_REGISTRY } from "../const";
|
||||
import {
|
||||
BuildController,
|
||||
buildImageName,
|
||||
} from "./build-controller";
|
||||
import {
|
||||
KubernetesBuildOperations,
|
||||
KubernetesBuildStore,
|
||||
buildObjectApi,
|
||||
} from "./build-kubernetes";
|
||||
import { FilesystemCas } from "./cas";
|
||||
import { KubernetesAuthStore } from "./kubernetes-store";
|
||||
import {
|
||||
createKubernetesClients,
|
||||
createKubernetesLeaseObjects,
|
||||
createKubernetesManagementDependencies,
|
||||
KubernetesAuditPersistence,
|
||||
KubernetesOperationPersistence,
|
||||
KubernetesWorkspaceLeaseProvider,
|
||||
KubernetesWorkspacePersistence,
|
||||
KubernetesWorkspaceStore,
|
||||
} from "./kubernetes-state";
|
||||
import {
|
||||
execUpgradeMatch,
|
||||
handleExecUpgrade,
|
||||
WireExecSession,
|
||||
type ExecConnection,
|
||||
} from "./app";
|
||||
import { KubernetesExec } from "./kubernetes-exec";
|
||||
import { createExecService } from "./exec-service";
|
||||
import { createManagementService } from "./management";
|
||||
import {
|
||||
PersistentOperationStore,
|
||||
recoverStaleOperations,
|
||||
} from "./operation-store";
|
||||
import { KubernetesLogs } from "./kubernetes-logs";
|
||||
import { createLogService } from "./log-service";
|
||||
import { resolveRegistryDigest, type RegistryCredentials } from "./registry";
|
||||
|
||||
const clients = createKubernetesClients();
|
||||
const store = new KubernetesAuthStore(clients.objects);
|
||||
const workspaceStore = new KubernetesWorkspaceStore(
|
||||
new KubernetesWorkspacePersistence(clients.objects),
|
||||
);
|
||||
const operationStore = new PersistentOperationStore(
|
||||
new KubernetesOperationPersistence(clients.objects),
|
||||
);
|
||||
const auditStore = new RedactingAuditStore(
|
||||
new KubernetesAuditPersistence(clients.objects),
|
||||
);
|
||||
const management = createManagementService(
|
||||
createKubernetesManagementDependencies(clients),
|
||||
);
|
||||
const namespace = process.env.KUBER_SYSTEM_NAMESPACE?.trim() || "kuber-system";
|
||||
const dataRoot = process.env.KUBER_DATA_ROOT?.trim() || "/data";
|
||||
const registry = (process.env.KUBER_BUILD_REGISTRY?.trim() || IMAGE_REGISTRY).replace(
|
||||
/\/+$/,
|
||||
"",
|
||||
);
|
||||
const registryConfigPath =
|
||||
process.env.KUBER_REGISTRY_CONFIG?.trim() || "/etc/kuber/registry/config.json";
|
||||
const registryResolveOrigin = process.env.KUBER_REGISTRY_RESOLVE_ORIGIN?.trim();
|
||||
const internalRegistryHost = process.env.KUBER_INTERNAL_REGISTRY_HOST?.trim();
|
||||
const internalRegistryInsecure =
|
||||
process.env.KUBER_INTERNAL_REGISTRY_INSECURE?.trim() === "true";
|
||||
const pushImagePrefix =
|
||||
process.env.KUBER_PUSH_IMAGE_PREFIX?.trim() || "kuber/";
|
||||
if (!process.env.KUBER_REGISTRY_SECRET?.trim()) {
|
||||
console.warn(
|
||||
"KUBER_REGISTRY_SECRET is unset; BuildKit will use anonymous registry access",
|
||||
);
|
||||
}
|
||||
|
||||
async function registryCredentials(): Promise<RegistryCredentials | undefined> {
|
||||
let config: { auths?: Record<string, { auth?: string }> };
|
||||
try {
|
||||
config = JSON.parse(await readFile(registryConfigPath, "utf8"));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
||||
throw error;
|
||||
}
|
||||
const host = registry.split("/")[0]!;
|
||||
const entry =
|
||||
config.auths?.[host] ??
|
||||
config.auths?.[`https://${host}`] ??
|
||||
config.auths?.[`https://${host}/v1/`];
|
||||
if (!entry?.auth) return;
|
||||
const separator = Buffer.from(entry.auth, "base64").toString("utf8").indexOf(":");
|
||||
if (separator < 0) return;
|
||||
const value = Buffer.from(entry.auth, "base64").toString("utf8");
|
||||
return { username: value.slice(0, separator), password: value.slice(separator + 1) };
|
||||
}
|
||||
|
||||
const buildStore = new KubernetesBuildStore(
|
||||
buildObjectApi(clients.objects),
|
||||
namespace,
|
||||
`${dataRoot}/uploads`,
|
||||
);
|
||||
const builds = new BuildController({
|
||||
cas: new FilesystemCas(`${dataRoot}/cas`),
|
||||
store: buildStore,
|
||||
kubernetes: new KubernetesBuildOperations(clients.batch, clients.core),
|
||||
namespace,
|
||||
workspaceRoot: `${dataRoot}/workspaces`,
|
||||
workspaceClaimName: process.env.KUBER_BUILD_DATA_CLAIM?.trim() || "kuber-build-data",
|
||||
cacheImage: (request) =>
|
||||
`${(internalRegistryHost ?? registry)}/kuber/cache-${request.project}-${request.service}`,
|
||||
imageName: (request) => buildImageName(registry, request.project, request.service),
|
||||
pushImage: internalRegistryHost
|
||||
? (request) =>
|
||||
`${internalRegistryHost}/${pushImagePrefix}${request.project}-${request.service}:latest`
|
||||
: undefined,
|
||||
pushRegistryInsecure: internalRegistryHost
|
||||
? internalRegistryInsecure
|
||||
: undefined,
|
||||
buildkitImage: process.env.KUBER_BUILDKIT_IMAGE?.trim() || undefined,
|
||||
registrySecretName: process.env.KUBER_REGISTRY_SECRET?.trim() || undefined,
|
||||
maxLogBytes: Number(process.env.KUBER_BUILD_LOG_BYTES ?? 512 * 1024),
|
||||
resolveDigest: async (image) =>
|
||||
resolveRegistryDigest(image, {
|
||||
credentials: await registryCredentials(),
|
||||
origin: registryResolveOrigin,
|
||||
insecure: registryResolveOrigin?.startsWith("http://"),
|
||||
}),
|
||||
});
|
||||
const logs = createLogService(
|
||||
new KubernetesLogs(clients.config, clients.apps, clients.core),
|
||||
);
|
||||
const execService = createExecService(new KubernetesExec(clients.config));
|
||||
const bootstrapUsername = process.env.KUBER_BOOTSTRAP_USERNAME?.trim();
|
||||
const bootstrapPassword = process.env.KUBER_BOOTSTRAP_PASSWORD;
|
||||
|
||||
if (bootstrapUsername && bootstrapPassword) {
|
||||
const existing = await store.getUser(bootstrapUsername);
|
||||
if (!existing) {
|
||||
await store.putUser({
|
||||
username: bootstrapUsername,
|
||||
passwordHash: await Bun.password.hash(bootstrapPassword, {
|
||||
algorithm: "argon2id",
|
||||
}),
|
||||
roles: ["admin"],
|
||||
});
|
||||
console.log(`Created bootstrap user ${bootstrapUsername}`);
|
||||
}
|
||||
}
|
||||
|
||||
const leases = new KubernetesWorkspaceLeaseProvider(
|
||||
createKubernetesLeaseObjects(clients.coordination),
|
||||
);
|
||||
|
||||
try {
|
||||
const recovered = await recoverStaleOperations(operationStore);
|
||||
if (recovered > 0)
|
||||
console.log(`Marked ${recovered} stale operation(s) as failed`);
|
||||
} catch (error) {
|
||||
console.error("Startup operation recovery failed", error);
|
||||
}
|
||||
|
||||
const sessionCleanupIntervalMs = Number(
|
||||
process.env.KUBER_SESSION_CLEANUP_MS ?? 10 * 60 * 1000,
|
||||
);
|
||||
const sessionCleanupTimer = setInterval(async () => {
|
||||
try {
|
||||
await cleanupExpiredSessions(store);
|
||||
} catch (error) {
|
||||
console.error("Expired session cleanup failed", error);
|
||||
}
|
||||
}, Number.isFinite(sessionCleanupIntervalMs) && sessionCleanupIntervalMs > 0
|
||||
? sessionCleanupIntervalMs
|
||||
: 10 * 60 * 1000);
|
||||
sessionCleanupTimer.unref?.();
|
||||
|
||||
const app = createApp({
|
||||
store,
|
||||
workspaceStore,
|
||||
operationStore,
|
||||
auditStore,
|
||||
management,
|
||||
builds,
|
||||
logs,
|
||||
execService,
|
||||
leases,
|
||||
resolveImage: async (project, service) => {
|
||||
const image = buildImageName(registry, project, service);
|
||||
const digest = await resolveRegistryDigest(image, {
|
||||
credentials: await registryCredentials(),
|
||||
origin: registryResolveOrigin,
|
||||
insecure: registryResolveOrigin?.startsWith("http://"),
|
||||
});
|
||||
return { image: image.replace(/:latest$/, ""), digest, reference: `${image.replace(/:latest$/, "")}@${digest}` };
|
||||
},
|
||||
allowedOrigins: (
|
||||
process.env.KUBER_ALLOWED_ORIGINS ?? "https://kuber.astrxl.dev"
|
||||
)
|
||||
.split(",")
|
||||
.map((origin) => origin.trim())
|
||||
.filter(Boolean),
|
||||
});
|
||||
|
||||
type ExecConnectionData = {
|
||||
connection: ExecConnection;
|
||||
};
|
||||
|
||||
const server = Bun.serve<ExecConnectionData>({
|
||||
port: Number(process.env.PORT ?? 3000),
|
||||
fetch(request, server) {
|
||||
const url = new URL(request.url);
|
||||
const workspaceId = execUpgradeMatch(url);
|
||||
if (workspaceId && request.method === "GET") {
|
||||
return handleExecUpgrade(
|
||||
{
|
||||
store,
|
||||
workspaceStore,
|
||||
auditStore,
|
||||
},
|
||||
request,
|
||||
workspaceId,
|
||||
(upgradeRequest, connection) =>
|
||||
server.upgrade(upgradeRequest, { data: { connection } }),
|
||||
);
|
||||
}
|
||||
return app(request);
|
||||
},
|
||||
websocket: {
|
||||
open(ws) {
|
||||
const { connection } = ws.data;
|
||||
const session = new WireExecSession(
|
||||
{
|
||||
sendText: (data) => ws.sendText(data),
|
||||
close: (code, reason) => ws.close(code, reason),
|
||||
},
|
||||
execService,
|
||||
connection,
|
||||
);
|
||||
ws.data = { connection, session } as unknown as ExecConnectionData;
|
||||
},
|
||||
message(ws, message) {
|
||||
const state = ws.data as unknown as {
|
||||
session?: WireExecSession;
|
||||
};
|
||||
state.session?.receive(
|
||||
typeof message === "string" ? message : new TextDecoder().decode(message),
|
||||
);
|
||||
},
|
||||
close(ws) {
|
||||
const state = ws.data as unknown as {
|
||||
session?: WireExecSession;
|
||||
};
|
||||
state.session?.close();
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`kuber server listening on ${server.url}`);
|
||||
Reference in New Issue
Block a user