511 lines
16 KiB
TypeScript
511 lines
16 KiB
TypeScript
import { execFile, spawn } from "node:child_process";
|
|
import { createHash } from "node:crypto";
|
|
import { constants } from "node:fs";
|
|
import { access } from "node:fs/promises";
|
|
import {
|
|
chmod,
|
|
lstat,
|
|
mkdir,
|
|
open,
|
|
readdir,
|
|
readlink,
|
|
realpath,
|
|
symlink,
|
|
readFile,
|
|
} from "node:fs/promises";
|
|
import type { Stats } from "node:fs";
|
|
import { delimiter, dirname, isAbsolute, relative, resolve, sep } from "node:path";
|
|
import { promisify } from "node:util";
|
|
import {
|
|
BUILD_PROTOCOL_VERSION,
|
|
assertSha256Digest,
|
|
type Sha256Digest,
|
|
type WorkspaceFile,
|
|
type WorkspaceManifest,
|
|
} from "../shared/build-protocol";
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
|
|
export async function gitAvailable(): Promise<boolean> {
|
|
for (const directory of (process.env.PATH ?? "").split(delimiter)) {
|
|
if (!directory) continue;
|
|
try {
|
|
await access(resolve(directory, process.platform === "win32" ? "git.exe" : "git"), constants.X_OK);
|
|
return true;
|
|
} catch { /* Continue searching PATH. */ }
|
|
}
|
|
return false;
|
|
}
|
|
|
|
export type WorkspaceBlob = {
|
|
digest: Sha256Digest;
|
|
data: Uint8Array;
|
|
};
|
|
|
|
export type WorkspaceSnapshot = {
|
|
manifest: WorkspaceManifest;
|
|
digest: Sha256Digest;
|
|
blobs: WorkspaceBlob[];
|
|
};
|
|
|
|
export type BlobReader =
|
|
| ((digest: Sha256Digest) => Promise<Uint8Array>)
|
|
| { get(digest: Sha256Digest): Promise<Uint8Array> };
|
|
|
|
function digest(data: Uint8Array | string): Sha256Digest {
|
|
return `sha256:${createHash("sha256").update(data).digest("hex")}`;
|
|
}
|
|
|
|
export function validateWorkspacePath(path: string): void {
|
|
if (
|
|
!path ||
|
|
path.includes("\0") ||
|
|
path.includes("\\") ||
|
|
isAbsolute(path) ||
|
|
path.split("/").some((part) => !part || part === "." || part === "..")
|
|
) {
|
|
throw new Error(`Unsafe workspace path: ${JSON.stringify(path)}`);
|
|
}
|
|
}
|
|
|
|
function isWithin(root: string, candidate: string): boolean {
|
|
const path = relative(root, candidate);
|
|
return (
|
|
path === "" ||
|
|
(!path.startsWith(`..${sep}`) && path !== ".." && !isAbsolute(path))
|
|
);
|
|
}
|
|
|
|
async function gitFiles(root: string, args: string[]): Promise<string[]> {
|
|
const { stdout } = await execFileAsync(
|
|
"git",
|
|
["-C", root, "ls-files", "-z", ...args],
|
|
{
|
|
encoding: "buffer",
|
|
maxBuffer: 64 * 1024 * 1024,
|
|
},
|
|
);
|
|
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
const files: string[] = [];
|
|
let start = 0;
|
|
for (
|
|
let end = stdout.indexOf(0);
|
|
end !== -1;
|
|
end = stdout.indexOf(0, start)
|
|
) {
|
|
if (end > start) files.push(decoder.decode(stdout.subarray(start, end)));
|
|
start = end + 1;
|
|
}
|
|
return files;
|
|
}
|
|
|
|
async function selectedFiles(root: string): Promise<string[]> {
|
|
const [normal, dotenv] = await Promise.all([
|
|
gitFiles(root, ["--cached", "--others", "--exclude-standard"]),
|
|
gitFiles(root, [
|
|
"--others",
|
|
"--ignored",
|
|
"--exclude-standard",
|
|
"--",
|
|
".env*",
|
|
"**/.env*",
|
|
]),
|
|
]);
|
|
return [...new Set([...normal, ...dotenv])].sort((a, b) =>
|
|
Buffer.from(a).compare(Buffer.from(b)),
|
|
);
|
|
}
|
|
|
|
async function filesystemFiles(root: string): Promise<string[]> {
|
|
const files: string[] = [];
|
|
const ignoredDirectories = new Set([
|
|
".git", ".hg", ".svn", "node_modules", "vendor", "bower_components",
|
|
".venv", "venv", "__pycache__", ".tox", ".mypy_cache", ".pytest_cache",
|
|
".next", ".nuxt", ".svelte-kit", ".cache", ".turbo", "dist", "build", "coverage",
|
|
"target", "out", "tmp", "temp",
|
|
]);
|
|
const sensitiveDirectory = /(?:^|[-_.])(?:secrets?|credentials?|configs?)(?:$|[-_.])/i;
|
|
const ignoreRules: Array<{ base: string; pattern: string; directory: boolean }> = [];
|
|
const loadIgnore = async (directory: string): Promise<void> => {
|
|
try {
|
|
const content = await readFile(resolve(directory, ".gitignore"), "utf8");
|
|
for (const raw of content.split(/\r?\n/)) {
|
|
const line = raw.trim();
|
|
if (!line || line.startsWith("#")) continue;
|
|
// Negations are skipped: safely re-including descendants requires
|
|
// Git's parent-directory semantics, so fallback stays fail-closed.
|
|
if (line.startsWith("!")) continue;
|
|
const rule = line.replace(/^\//, "");
|
|
if (rule) ignoreRules.push({ base: relative(root, directory).split(sep).join("/"), pattern: rule.replace(/\/$/, ""), directory: line.endsWith("/") });
|
|
}
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
}
|
|
};
|
|
const ignoredByRules = (path: string, isDirectory: boolean): boolean => {
|
|
let ignored = false;
|
|
for (const rule of ignoreRules) {
|
|
const prefix = rule.base ? `${rule.base}/` : "";
|
|
if (rule.base && path !== rule.base && !path.startsWith(prefix)) continue;
|
|
const local = rule.base && path.startsWith(prefix) ? path.slice(prefix.length) : path;
|
|
const glob = rule.pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*\*/g, "__DOUBLESTAR__").replace(/\*/g, "[^/]*").replace(/\?/g, "[^/]").replace(/__DOUBLESTAR__/g, ".*");
|
|
const matcher = new RegExp(`^(?:${glob})(?:/.*)?$`);
|
|
const basenameMatcher = new RegExp(`^(?:${glob})$`);
|
|
if ((matcher.test(local) || local.split("/").some((part) => basenameMatcher.test(part))) && (!rule.directory || isDirectory || local.includes("/"))) ignored = true;
|
|
}
|
|
return ignored;
|
|
};
|
|
const visit = async (directory: string): Promise<void> => {
|
|
await loadIgnore(directory);
|
|
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
|
const source = resolve(directory, entry.name);
|
|
const path = relative(root, source).split(sep).join("/");
|
|
if (entry.isDirectory()) {
|
|
if (ignoredDirectories.has(entry.name) || sensitiveDirectory.test(entry.name) || ignoredByRules(path, true)) continue;
|
|
await visit(source);
|
|
}
|
|
else if (entry.isFile() || entry.isSymbolicLink()) {
|
|
if (ignoredByRules(path, false)) continue;
|
|
if (/^\.env/i.test(entry.name) || /(?:secret|credential|password|token|private[-_.]?key)/i.test(entry.name) || /^(?:id_rsa|id_ed25519|known_hosts|config\.json|\.npmrc|\.pypirc|\.netrc)$/i.test(entry.name)) continue;
|
|
files.push(path);
|
|
} else {
|
|
throw new Error(`Special files are not allowed in workspaces: ${relative(root, source).split(sep).join("/")}`);
|
|
}
|
|
}
|
|
};
|
|
await visit(root);
|
|
return files.sort((a, b) => Buffer.from(a).compare(Buffer.from(b)));
|
|
}
|
|
|
|
async function isGitRepository(root: string): Promise<boolean> {
|
|
if (!(await gitAvailable())) return false;
|
|
try {
|
|
await execFileAsync("git", ["-C", root, "rev-parse", "--show-toplevel"]);
|
|
return true;
|
|
} catch (error) {
|
|
const code = (error as { code?: unknown }).code;
|
|
const exitCode = (error as { exitCode?: number }).exitCode;
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
if (code === "ENOENT") return false;
|
|
if (exitCode === 128 || code === 128 || /not a git repository/i.test(message)) return false;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function rejectSelectedSpecialFiles(root: string): Promise<void> {
|
|
// Let Git identify ignored directories using its own ignore engine. With
|
|
// --directory, ignored trees are returned as directory entries, so the
|
|
// filesystem walk below can prune them without visiting their contents.
|
|
const ignoredResult = await execFileAsync(
|
|
"git",
|
|
[
|
|
"-C",
|
|
root,
|
|
"ls-files",
|
|
"--others",
|
|
"--ignored",
|
|
"--exclude-standard",
|
|
"--directory",
|
|
"-z",
|
|
],
|
|
{ encoding: "buffer", maxBuffer: 64 * 1024 * 1024 },
|
|
);
|
|
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
const ignoredDirectories = new Set<string>();
|
|
let start = 0;
|
|
for (
|
|
let end = ignoredResult.stdout.indexOf(0);
|
|
end !== -1;
|
|
end = ignoredResult.stdout.indexOf(0, start)
|
|
) {
|
|
if (end > start) {
|
|
const path = decoder.decode(ignoredResult.stdout.subarray(start, end));
|
|
if (path.endsWith("/")) ignoredDirectories.add(path.slice(0, -1));
|
|
}
|
|
start = end + 1;
|
|
}
|
|
|
|
const specialPaths: string[] = [];
|
|
const visit = async (directory: string): Promise<void> => {
|
|
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
|
if (directory === root && entry.name === ".git") continue;
|
|
const source = resolve(directory, entry.name);
|
|
const path = relative(root, source).split(sep).join("/");
|
|
if (entry.isDirectory()) {
|
|
if (!ignoredDirectories.has(path)) await visit(source);
|
|
} else if (!entry.isFile() && !entry.isSymbolicLink()) {
|
|
specialPaths.push(path);
|
|
}
|
|
}
|
|
};
|
|
await visit(root);
|
|
|
|
if (specialPaths.length === 0) return;
|
|
const input = Buffer.from(`${specialPaths.join("\0")}\0`);
|
|
const child = spawn("git", ["-C", root, "check-ignore", "--stdin", "-z"]);
|
|
const output: Buffer[] = [];
|
|
const errors: Buffer[] = [];
|
|
child.stdout.on("data", (chunk: Buffer) => output.push(chunk));
|
|
child.stderr.on("data", (chunk: Buffer) => errors.push(chunk));
|
|
const completed = new Promise<void>((resolveExit, rejectExit) => {
|
|
child.once("error", rejectExit);
|
|
child.once("close", (code) => {
|
|
if (code === 0 || code === 1) resolveExit();
|
|
else
|
|
rejectExit(
|
|
new Error(
|
|
Buffer.concat(errors).toString("utf8") ||
|
|
`git check-ignore exited with ${code}`,
|
|
),
|
|
);
|
|
});
|
|
});
|
|
child.stdin.end(input);
|
|
await completed;
|
|
const stdout = Buffer.concat(output);
|
|
const ignored = new Set<string>();
|
|
start = 0;
|
|
for (
|
|
let end = stdout.indexOf(0);
|
|
end !== -1;
|
|
end = stdout.indexOf(0, start)
|
|
) {
|
|
if (end > start) ignored.add(decoder.decode(stdout.subarray(start, end)));
|
|
start = end + 1;
|
|
}
|
|
for (const path of specialPaths) {
|
|
const name = path.slice(path.lastIndexOf("/") + 1);
|
|
if (!ignored.has(path) || name.startsWith(".env"))
|
|
throw new Error(`Special files are not allowed in workspaces: ${path}`);
|
|
}
|
|
}
|
|
|
|
function canonicalManifest(manifest: WorkspaceManifest): string {
|
|
return JSON.stringify({
|
|
version: manifest.version,
|
|
files: manifest.files.map((file) => ({
|
|
path: file.path,
|
|
type: file.type,
|
|
digest: file.digest,
|
|
size: file.size,
|
|
mode: file.mode,
|
|
})),
|
|
});
|
|
}
|
|
|
|
export function serializeWorkspaceManifest(
|
|
manifest: WorkspaceManifest,
|
|
): Uint8Array {
|
|
validateWorkspaceManifest(manifest);
|
|
return Buffer.from(canonicalManifest(manifest));
|
|
}
|
|
|
|
export function workspaceManifestDigest(
|
|
manifest: WorkspaceManifest,
|
|
): Sha256Digest {
|
|
return digest(serializeWorkspaceManifest(manifest));
|
|
}
|
|
|
|
export function validateWorkspaceManifest(manifest: WorkspaceManifest): void {
|
|
if (
|
|
manifest.version !== BUILD_PROTOCOL_VERSION ||
|
|
!Array.isArray(manifest.files)
|
|
) {
|
|
throw new Error("Unsupported workspace manifest");
|
|
}
|
|
|
|
let previous = "";
|
|
const seen = new Set<string>();
|
|
for (const file of manifest.files) {
|
|
validateWorkspacePath(file.path);
|
|
assertSha256Digest(file.digest);
|
|
if (!Number.isSafeInteger(file.size) || file.size < 0)
|
|
throw new Error(`Invalid size for ${file.path}`);
|
|
if (
|
|
(file.type === "file" && file.mode !== 0o644 && file.mode !== 0o755) ||
|
|
(file.type === "symlink" && file.mode !== 0o777)
|
|
) {
|
|
throw new Error(`Invalid mode for ${file.path}`);
|
|
}
|
|
if (file.type !== "file" && file.type !== "symlink")
|
|
throw new Error(`Invalid entry type for ${file.path}`);
|
|
if (seen.has(file.path))
|
|
throw new Error(`Duplicate workspace path: ${file.path}`);
|
|
for (const parent of file.path
|
|
.split("/")
|
|
.slice(0, -1)
|
|
.map((_, index, parts) => parts.slice(0, index + 1).join("/"))) {
|
|
if (seen.has(parent))
|
|
throw new Error(`Workspace entry is used as a directory: ${parent}`);
|
|
}
|
|
if (previous && Buffer.from(previous).compare(Buffer.from(file.path)) >= 0)
|
|
throw new Error("Workspace files must be bytewise sorted");
|
|
seen.add(file.path);
|
|
previous = file.path;
|
|
}
|
|
}
|
|
|
|
export async function enumerateWorkspace(
|
|
root: string,
|
|
): Promise<WorkspaceSnapshot> {
|
|
const repository = await realpath(root);
|
|
const gitBacked = await isGitRepository(repository);
|
|
const paths = gitBacked
|
|
? await (async () => {
|
|
await rejectSelectedSpecialFiles(repository);
|
|
return selectedFiles(repository);
|
|
})()
|
|
: await filesystemFiles(repository);
|
|
const files: WorkspaceFile[] = [];
|
|
const blobs = new Map<Sha256Digest, Uint8Array>();
|
|
|
|
for (const path of paths) {
|
|
validateWorkspacePath(path);
|
|
const source = resolve(repository, path);
|
|
if (!isWithin(repository, source))
|
|
throw new Error(`Workspace path escapes root: ${path}`);
|
|
|
|
let stat: Stats;
|
|
try {
|
|
stat = await lstat(source);
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code === "ENOENT") continue;
|
|
throw error;
|
|
}
|
|
|
|
let data: Uint8Array;
|
|
let entry: WorkspaceFile;
|
|
if (stat.isSymbolicLink()) {
|
|
const target = await readlink(source);
|
|
if (
|
|
isAbsolute(target) ||
|
|
!isWithin(repository, resolve(dirname(source), target))
|
|
)
|
|
throw new Error(`Symlink escapes workspace: ${path} -> ${target}`);
|
|
data = Buffer.from(target);
|
|
entry = {
|
|
path,
|
|
type: "symlink",
|
|
digest: digest(data),
|
|
size: data.byteLength,
|
|
mode: 0o777,
|
|
};
|
|
} else if (stat.isFile()) {
|
|
let mode: 0o644 | 0o755;
|
|
const handle = await open(
|
|
source,
|
|
constants.O_RDONLY | constants.O_NOFOLLOW,
|
|
);
|
|
try {
|
|
const opened = await handle.stat();
|
|
if (!opened.isFile()) throw new Error(`Not a regular file: ${path}`);
|
|
mode = opened.mode & 0o111 ? 0o755 : 0o644;
|
|
data = await handle.readFile();
|
|
} finally {
|
|
await handle.close();
|
|
}
|
|
entry = {
|
|
path,
|
|
type: "file",
|
|
digest: digest(data),
|
|
size: data.byteLength,
|
|
mode,
|
|
};
|
|
} else {
|
|
throw new Error(`Special files are not allowed in workspaces: ${path}`);
|
|
}
|
|
files.push(entry);
|
|
blobs.set(entry.digest, data);
|
|
}
|
|
|
|
const manifest = {
|
|
version: BUILD_PROTOCOL_VERSION,
|
|
files,
|
|
} satisfies WorkspaceManifest;
|
|
return {
|
|
manifest,
|
|
digest: workspaceManifestDigest(manifest),
|
|
blobs: [...blobs].map(([blobDigest, data]) => ({
|
|
digest: blobDigest,
|
|
data,
|
|
})),
|
|
};
|
|
}
|
|
|
|
async function ensureParentDirectories(
|
|
root: string,
|
|
path: string,
|
|
): Promise<void> {
|
|
let current = root;
|
|
for (const part of path.split("/").slice(0, -1)) {
|
|
current = resolve(current, part);
|
|
try {
|
|
const stat = await lstat(current);
|
|
if (!stat.isDirectory())
|
|
throw new Error(`Workspace parent is not a directory: ${path}`);
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
|
await mkdir(current, { mode: 0o755 });
|
|
}
|
|
}
|
|
}
|
|
|
|
async function readBlob(
|
|
reader: BlobReader,
|
|
blobDigest: Sha256Digest,
|
|
): Promise<Uint8Array> {
|
|
return typeof reader === "function"
|
|
? reader(blobDigest)
|
|
: reader.get(blobDigest);
|
|
}
|
|
|
|
export async function materializeWorkspace(
|
|
destination: string,
|
|
manifest: WorkspaceManifest,
|
|
reader: BlobReader,
|
|
): Promise<void> {
|
|
validateWorkspaceManifest(manifest);
|
|
await mkdir(destination, { recursive: false, mode: 0o755 });
|
|
const root = await realpath(destination);
|
|
|
|
for (const file of manifest.files.filter((entry) => entry.type === "file")) {
|
|
await ensureParentDirectories(root, file.path);
|
|
const data = await readBlob(reader, file.digest);
|
|
if (data.byteLength !== file.size || digest(data) !== file.digest)
|
|
throw new Error(`Blob verification failed for ${file.path}`);
|
|
const target = resolve(root, file.path);
|
|
const handle = await open(
|
|
target,
|
|
constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,
|
|
file.mode,
|
|
);
|
|
try {
|
|
await handle.writeFile(data);
|
|
await handle.sync();
|
|
} finally {
|
|
await handle.close();
|
|
}
|
|
await chmod(target, file.mode);
|
|
}
|
|
|
|
for (const file of manifest.files.filter(
|
|
(entry) => entry.type === "symlink",
|
|
)) {
|
|
await ensureParentDirectories(root, file.path);
|
|
const data = await readBlob(reader, file.digest);
|
|
if (data.byteLength !== file.size || digest(data) !== file.digest)
|
|
throw new Error(`Blob verification failed for ${file.path}`);
|
|
const linkTarget = Buffer.from(data).toString("utf8");
|
|
const target = resolve(root, file.path);
|
|
if (
|
|
linkTarget.includes("\0") ||
|
|
isAbsolute(linkTarget) ||
|
|
!isWithin(root, resolve(dirname(target), linkTarget))
|
|
)
|
|
throw new Error(
|
|
`Symlink escapes workspace: ${file.path} -> ${linkTarget}`,
|
|
);
|
|
await symlink(linkTarget, target);
|
|
}
|
|
}
|