662 lines
21 KiB
TypeScript
662 lines
21 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
|
import { Listr } from "listr2";
|
|
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { ApiRequestInit, ApiRequestOptions } from "../../lib/api";
|
|
import { KuberApiError } from "../../lib/api";
|
|
import type { ApiRequester } from "../../lib/build";
|
|
import {
|
|
ensureWorkspace,
|
|
reconcileResources,
|
|
runLiveResourceOperation,
|
|
runUp,
|
|
workspaceAdoptionRoute,
|
|
} from "../../command/up";
|
|
import { provideContext } from "../../lib/context";
|
|
import { resolveTrustIdentity, updateTrust } from "../../lib/trust";
|
|
import { workspaceManifestDigest } from "../../lib/workspace";
|
|
|
|
const manifest = { version: 1 as const, files: [] };
|
|
const snapshot = {
|
|
manifest,
|
|
digest: workspaceManifestDigest(manifest),
|
|
blobs: [],
|
|
};
|
|
|
|
describe("up API pipeline", () => {
|
|
test("forwards the build timeout through the trusted requester", async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "kuber-up-api-"));
|
|
const previousCwd = process.cwd();
|
|
const previousConfigHome = process.env.XDG_CONFIG_HOME;
|
|
const configHome = join(root, "config");
|
|
const calls: Array<{
|
|
path: string;
|
|
init?: ApiRequestInit;
|
|
options?: ApiRequestOptions;
|
|
}> = [];
|
|
|
|
try {
|
|
await writeFile(
|
|
join(root, "compose.yml"),
|
|
"services:\n web:\n build: .\n",
|
|
);
|
|
await writeFile(
|
|
join(root, ".kuberrc.ts"),
|
|
'export default { project: "shop" };\n',
|
|
);
|
|
const git = Bun.spawn(["git", "init", "-q", root]);
|
|
expect(await git.exited).toBe(0);
|
|
|
|
process.chdir(root);
|
|
process.env.XDG_CONFIG_HOME = configHome;
|
|
const identity = await resolveTrustIdentity("shop", root);
|
|
await updateTrust((records) => [...records, identity]);
|
|
|
|
const request: ApiRequester = async <T>(
|
|
path: string,
|
|
init?: ApiRequestInit,
|
|
options?: ApiRequestOptions,
|
|
) => {
|
|
calls.push({ path, init, options });
|
|
if (path === "/snapshots/negotiate")
|
|
return { workspace: snapshot.digest, missing: [], ready: true } as T;
|
|
if (path === "/builds") {
|
|
const buildRequest = init?.json as { id?: unknown } | undefined;
|
|
if (typeof buildRequest?.id !== "string")
|
|
throw new Error("Expected build request ID");
|
|
return {
|
|
version: 1,
|
|
id: buildRequest.id,
|
|
state: "succeeded",
|
|
createdAt: "2026-01-01T00:00:00Z",
|
|
} as T;
|
|
}
|
|
if (path.includes("/events")) return [] as T;
|
|
if (path.endsWith("/result"))
|
|
return {
|
|
image: "registry.server/kuber/shop-web",
|
|
digest: `sha256:${"a".repeat(64)}`,
|
|
reference: `registry.server/kuber/shop-web@sha256:${"a".repeat(64)}`,
|
|
} as T;
|
|
if (path === "/workspaces/shop")
|
|
throw new KuberApiError("missing", 404);
|
|
if (path === "/workspaces")
|
|
return {
|
|
metadata: { name: "shop", uid: "workspace", resourceVersion: "1" },
|
|
} as T;
|
|
if (path.endsWith("/adopt")) return { resourcesAdopted: 0 } as T;
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
return {} as T;
|
|
};
|
|
|
|
await provideContext(() => runUp(true, request));
|
|
|
|
const build = calls.find(({ path }) => path === "/builds");
|
|
if (!build) throw new Error("Expected build submission");
|
|
expect(build.options).toEqual({ timeoutMs: 300_000 });
|
|
expect(
|
|
new Headers(build.init?.headers).get("x-kuber-trust-project"),
|
|
).toBe("shop");
|
|
} finally {
|
|
process.chdir(previousCwd);
|
|
if (previousConfigHome === undefined) delete process.env.XDG_CONFIG_HOME;
|
|
else process.env.XDG_CONFIG_HOME = previousConfigHome;
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("creates workspace metadata without embedding source blobs", async () => {
|
|
const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
|
|
const request: ApiRequester = async <T>(
|
|
path: string,
|
|
init?: ApiRequestInit,
|
|
) => {
|
|
calls.push({ path, init });
|
|
if (!init) throw new KuberApiError("missing", 404);
|
|
return {
|
|
metadata: { name: "shop", uid: "uid", resourceVersion: "1" },
|
|
} as T;
|
|
};
|
|
await ensureWorkspace(
|
|
"shop",
|
|
{ services: { web: { image: "nginx" } } },
|
|
snapshot,
|
|
request,
|
|
);
|
|
|
|
expect(calls.map(({ path }) => path)).toEqual([
|
|
"/workspaces/shop",
|
|
"/workspaces",
|
|
]);
|
|
const body = calls[1]!.init?.json as Record<string, unknown>;
|
|
expect(body).toMatchObject({
|
|
id: "shop",
|
|
source: { uri: `cas://${snapshot.digest}`, digest: snapshot.digest },
|
|
});
|
|
expect(JSON.stringify(body)).not.toContain('"blob"');
|
|
expect(JSON.stringify(body)).not.toContain('"files"');
|
|
});
|
|
|
|
test("updates workspace metadata with the current resource-version ETag", async () => {
|
|
const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
|
|
const request: ApiRequester = async <T>(
|
|
path: string,
|
|
init?: ApiRequestInit,
|
|
) => {
|
|
calls.push({ path, init });
|
|
return {
|
|
metadata: {
|
|
name: "shop",
|
|
uid: "uid",
|
|
resourceVersion: init ? "8" : "7",
|
|
},
|
|
} as T;
|
|
};
|
|
await ensureWorkspace("shop", { services: {} }, snapshot, request);
|
|
expect(calls[1]?.init?.method).toBe("PUT");
|
|
expect(new Headers(calls[1]?.init?.headers).get("if-match")).toBe('"7"');
|
|
});
|
|
|
|
test("plans, applies, runs the hook, waits, then deletes stale identities", async () => {
|
|
const order: string[] = [];
|
|
const desired = [
|
|
{ apiVersion: "apps/v1", kind: "Deployment", metadata: { name: "web" } },
|
|
];
|
|
const stale = [
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
name: "old",
|
|
uid: "secret-uid",
|
|
workspaceUid: "workspace-uid",
|
|
},
|
|
];
|
|
const request: ApiRequester = async <T>(path: string) => {
|
|
order.push(path.split("/").at(-1)!);
|
|
if (path.endsWith("/plan")) return { desired, stale } as T;
|
|
return {} as T;
|
|
};
|
|
|
|
await reconcileResources(
|
|
"shop",
|
|
desired,
|
|
42_000,
|
|
() => {
|
|
order.push("hook");
|
|
},
|
|
request,
|
|
);
|
|
expect(order).toEqual(["plan", "apply", "hook", "wait", "delete"]);
|
|
});
|
|
|
|
test("resumes an interrupted reconcile operation by its persisted ID", async () => {
|
|
const calls: string[] = [];
|
|
const applyIdempotencyKeys: Array<string | null> = [];
|
|
let applyAttempts = 0;
|
|
let operationPolls = 0;
|
|
const request: ApiRequester = async <T>(
|
|
path: string,
|
|
init?: ApiRequestInit,
|
|
) => {
|
|
calls.push(path);
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
if (path.endsWith("/apply")) {
|
|
applyIdempotencyKeys.push(
|
|
new Headers(init?.headers).get("idempotency-key"),
|
|
);
|
|
applyAttempts++;
|
|
if (applyAttempts === 1) throw new TypeError("fetch failed");
|
|
return {
|
|
operationId: "operation-apply",
|
|
operation: { status: { state: "running" } },
|
|
} as T;
|
|
}
|
|
if (path === "/operations/operation-apply") {
|
|
operationPolls++;
|
|
if (operationPolls === 1) throw new TypeError("connection reset");
|
|
return { status: { state: "succeeded" } } as T;
|
|
}
|
|
throw new Error(`Unexpected request: ${path}`);
|
|
};
|
|
|
|
await reconcileResources("shop", [], 1, undefined, request, {
|
|
sleep: async () => {},
|
|
});
|
|
|
|
expect(calls).toEqual([
|
|
"/workspaces/shop/resources/plan",
|
|
"/workspaces/shop/resources/apply",
|
|
"/workspaces/shop/resources/apply",
|
|
"/operations/operation-apply/events?after=0",
|
|
"/operations/operation-apply",
|
|
"/operations/operation-apply",
|
|
"/operations/operation-apply/events?after=0",
|
|
]);
|
|
expect(applyIdempotencyKeys[0]).toBeTruthy();
|
|
expect(applyIdempotencyKeys[1]).toBe(applyIdempotencyKeys[0]);
|
|
});
|
|
|
|
test("restarts after an interrupted apply is persisted before its ID is received", async () => {
|
|
const applyRequests: Array<{ key: string | null; json: unknown }> = [];
|
|
const request: ApiRequester = async <T>(
|
|
path: string,
|
|
init?: ApiRequestInit,
|
|
) => {
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
if (path.endsWith("/apply")) {
|
|
applyRequests.push({
|
|
key: new Headers(init?.headers).get("idempotency-key"),
|
|
json: init?.json,
|
|
});
|
|
if (applyRequests.length === 1) throw new TypeError("fetch failed");
|
|
if (applyRequests.length === 2)
|
|
throw new KuberApiError("operation interrupted", 500, {
|
|
title: "Operation interrupted",
|
|
status: 500,
|
|
code: "OPERATION_INTERRUPTED",
|
|
});
|
|
return {
|
|
operationId: "operation-apply",
|
|
operation: { status: { state: "succeeded" } },
|
|
} as T;
|
|
}
|
|
throw new Error(`Unexpected request: ${path}`);
|
|
};
|
|
|
|
await reconcileResources("shop", [], 1, undefined, request, {
|
|
sleep: async () => {},
|
|
});
|
|
|
|
expect(applyRequests).toHaveLength(3);
|
|
expect(applyRequests[0]?.key).toBeTruthy();
|
|
expect(applyRequests[1]?.key).toBe(applyRequests[0]?.key);
|
|
expect(applyRequests[2]?.key).toBeTruthy();
|
|
expect(applyRequests[2]?.key).not.toBe(applyRequests[0]?.key);
|
|
expect(applyRequests[2]?.json).toEqual(applyRequests[0]?.json);
|
|
});
|
|
|
|
test("polls persisted resource progress without duplicating events after reconnect", async () => {
|
|
const progress: string[] = [];
|
|
let poll = 0;
|
|
const request: ApiRequester = async <T>(path: string) => {
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
if (path.endsWith("/apply"))
|
|
return {
|
|
operationId: "operation-apply",
|
|
operation: { status: { state: "running" } },
|
|
} as T;
|
|
if (path.includes("/events")) {
|
|
poll++;
|
|
return {
|
|
items:
|
|
poll === 1
|
|
? [
|
|
{
|
|
sequence: 1,
|
|
data: null,
|
|
},
|
|
{
|
|
sequence: 2,
|
|
data: {
|
|
resource: {
|
|
apiVersion: "v1",
|
|
kind: "Service",
|
|
name: "web",
|
|
},
|
|
phase: "apply",
|
|
state: "started",
|
|
},
|
|
},
|
|
]
|
|
: [
|
|
{
|
|
sequence: 2,
|
|
data: {
|
|
resource: {
|
|
apiVersion: "v1",
|
|
kind: "Service",
|
|
name: "web",
|
|
},
|
|
phase: "apply",
|
|
state: "started",
|
|
},
|
|
},
|
|
{
|
|
sequence: 3,
|
|
data: {
|
|
resource: {
|
|
apiVersion: "v1",
|
|
kind: "Service",
|
|
name: "web",
|
|
},
|
|
phase: "apply",
|
|
state: "succeeded",
|
|
},
|
|
},
|
|
],
|
|
} as T;
|
|
}
|
|
if (path === "/operations/operation-apply") {
|
|
return { status: { state: poll > 1 ? "succeeded" : "running" } } as T;
|
|
}
|
|
throw new Error(`Unexpected request: ${path}`);
|
|
};
|
|
|
|
await reconcileResources("shop", [], 1, undefined, request, {
|
|
sleep: async () => {},
|
|
onEvent: (event) =>
|
|
progress.push(`${event.sequence}:${event.data.state}`),
|
|
});
|
|
expect(progress).toEqual(["2:started", "3:succeeded"]);
|
|
});
|
|
|
|
test("fails visibly when retained operation progress has a cursor gap", async () => {
|
|
let applyAttempts = 0;
|
|
let operationPolls = 0;
|
|
const request: ApiRequester = async <T>(path: string) => {
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
if (path.endsWith("/apply")) {
|
|
applyAttempts += 1;
|
|
return {
|
|
operationId: "operation-apply",
|
|
operation: { status: { state: "running" } },
|
|
} as T;
|
|
}
|
|
if (path.includes("/events"))
|
|
return {
|
|
retainedFirstSequence: 3,
|
|
cursorGap: true,
|
|
items: [],
|
|
} as T;
|
|
if (path === "/operations/operation-apply") {
|
|
operationPolls += 1;
|
|
return { status: { state: "succeeded" } } as T;
|
|
}
|
|
throw new Error(`Unexpected request: ${path}`);
|
|
};
|
|
|
|
await expect(
|
|
reconcileResources("shop", [], 1, undefined, request, {
|
|
sleep: async () => {},
|
|
}),
|
|
).rejects.toThrow("progress history was truncated");
|
|
expect(applyAttempts).toBe(1);
|
|
expect(operationPolls).toBe(0);
|
|
});
|
|
|
|
test("starts live resource subtasks before the operation and updates them from progress", async () => {
|
|
let finish!: () => void;
|
|
let child: { title: string; output: string } | undefined;
|
|
const completed = new Promise<void>((resolve) => (finish = resolve));
|
|
const listr = new Listr([
|
|
{
|
|
title: "Apply resources",
|
|
task: async (_ctx, task) =>
|
|
runLiveResourceOperation(
|
|
task,
|
|
"apply",
|
|
[
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Service",
|
|
name: "web",
|
|
},
|
|
],
|
|
async (onEvent) => {
|
|
expect(child?.title).toBe("Apply Service/web");
|
|
onEvent({
|
|
sequence: 1,
|
|
data: {
|
|
resource: {
|
|
apiVersion: "v1",
|
|
kind: "Service",
|
|
name: "web",
|
|
},
|
|
phase: "apply",
|
|
state: "started",
|
|
},
|
|
});
|
|
expect(child?.output).toBe("started");
|
|
await completed;
|
|
},
|
|
{ onTaskStarted: (_target, activeTask) => (child = activeTask) },
|
|
),
|
|
},
|
|
]);
|
|
|
|
const run = listr.run();
|
|
await Bun.sleep(0);
|
|
finish();
|
|
await run;
|
|
});
|
|
|
|
test("resubmits with a fresh key after server restart interruption", async () => {
|
|
const applyRequests: Array<{ key: string | null; json: unknown }> = [];
|
|
let operationPolls = 0;
|
|
const request: ApiRequester = async <T>(
|
|
path: string,
|
|
init?: ApiRequestInit,
|
|
) => {
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
if (path.endsWith("/apply")) {
|
|
applyRequests.push({
|
|
key: new Headers(init?.headers).get("idempotency-key"),
|
|
json: init?.json,
|
|
});
|
|
return {
|
|
operationId: `operation-apply-${applyRequests.length}`,
|
|
operation: { status: { state: "running" } },
|
|
} as T;
|
|
}
|
|
if (path === "/operations/operation-apply-1") {
|
|
operationPolls++;
|
|
if (operationPolls === 1)
|
|
return {
|
|
status: {
|
|
state: "failed",
|
|
error: {
|
|
code: "OPERATION_INTERRUPTED",
|
|
message: "server restarted",
|
|
},
|
|
},
|
|
} as T;
|
|
return { status: { state: "succeeded" } } as T;
|
|
}
|
|
if (path === "/operations/operation-apply-2")
|
|
return { status: { state: "succeeded" } } as T;
|
|
throw new Error(`Unexpected request: ${path}`);
|
|
};
|
|
|
|
await reconcileResources("shop", [], 1, undefined, request, {
|
|
sleep: async () => {},
|
|
});
|
|
|
|
expect(applyRequests).toHaveLength(2);
|
|
expect(applyRequests[0]?.key).toBeTruthy();
|
|
expect(applyRequests[1]?.key).toBeTruthy();
|
|
expect(applyRequests[1]?.key).not.toBe(applyRequests[0]?.key);
|
|
expect(applyRequests[1]?.json).toEqual(applyRequests[0]?.json);
|
|
});
|
|
|
|
test.each([
|
|
["failed", "OPERATION_FAILED"],
|
|
["cancelled", "OPERATION_CANCELLED"],
|
|
])("does not retry arbitrary %s operations", async (state, code) => {
|
|
let applyAttempts = 0;
|
|
let operationPolls = 0;
|
|
const request: ApiRequester = async <T>(path: string) => {
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
if (path.endsWith("/apply")) {
|
|
applyAttempts++;
|
|
return {
|
|
operationId: "operation-apply",
|
|
operation: { status: { state: "running" } },
|
|
} as T;
|
|
}
|
|
if (path === "/operations/operation-apply") {
|
|
operationPolls++;
|
|
return {
|
|
status: { state, error: { code, message: "terminal" } },
|
|
} as T;
|
|
}
|
|
throw new Error(`Unexpected request: ${path}`);
|
|
};
|
|
|
|
await expect(
|
|
reconcileResources("shop", [], 1, undefined, request, {
|
|
sleep: async () => {},
|
|
}),
|
|
).rejects.toMatchObject({ code });
|
|
expect(applyAttempts).toBe(1);
|
|
expect(operationPolls).toBe(1);
|
|
});
|
|
|
|
test("bounds restart recovery sleeps by the resume deadline", async () => {
|
|
let currentTime = 0;
|
|
const sleeps: Array<{ startedAt: number; milliseconds: number }> = [];
|
|
let operationNumber = 0;
|
|
const request: ApiRequester = async <T>(path: string) => {
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
if (path.endsWith("/apply")) {
|
|
operationNumber++;
|
|
return {
|
|
operationId: `operation-${operationNumber}`,
|
|
operation: { status: { state: "running" } },
|
|
} as T;
|
|
}
|
|
return {
|
|
status: {
|
|
state: "failed",
|
|
error: { code: "OPERATION_INTERRUPTED", message: "restarted" },
|
|
},
|
|
} as T;
|
|
};
|
|
|
|
await expect(
|
|
reconcileResources("shop", [], 0, undefined, request, {
|
|
now: () => currentTime,
|
|
sleep: async (milliseconds) => {
|
|
sleeps.push({ startedAt: currentTime, milliseconds });
|
|
currentTime += milliseconds;
|
|
},
|
|
}),
|
|
).rejects.toThrow("Timed out while reconnecting to resume the operation");
|
|
|
|
expect(sleeps).not.toHaveLength(0);
|
|
expect(
|
|
sleeps.every(
|
|
({ startedAt, milliseconds }) => startedAt + milliseconds <= 60_000,
|
|
),
|
|
).toBe(true);
|
|
expect(currentTime).toBe(60_000);
|
|
});
|
|
|
|
test("does not restart after an apply interruption reaches the resume deadline", async () => {
|
|
let currentTime = 0;
|
|
let applyAttempts = 0;
|
|
const request: ApiRequester = async <T>(path: string) => {
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
applyAttempts++;
|
|
if (applyAttempts === 1) throw new TypeError("connection reset");
|
|
throw new KuberApiError("operation interrupted", 500, {
|
|
title: "Operation interrupted",
|
|
status: 500,
|
|
code: "OPERATION_INTERRUPTED",
|
|
});
|
|
};
|
|
|
|
await expect(
|
|
reconcileResources("shop", [], 0, undefined, request, {
|
|
now: () => currentTime,
|
|
sleep: async (milliseconds) => {
|
|
currentTime += milliseconds;
|
|
if (currentTime < 60_000) currentTime = 60_000;
|
|
},
|
|
}),
|
|
).rejects.toMatchObject({
|
|
code: "OPERATION_INTERRUPTED",
|
|
status: 500,
|
|
});
|
|
expect(applyAttempts).toBe(2);
|
|
expect(currentTime).toBe(60_000);
|
|
});
|
|
|
|
test("fails immediately for a typed 503 operation-store error", async () => {
|
|
const unavailable = new KuberApiError(
|
|
"Operation storage is not configured",
|
|
503,
|
|
{
|
|
title: "Service unavailable",
|
|
status: 503,
|
|
code: "OPERATION_STORE_UNAVAILABLE",
|
|
},
|
|
);
|
|
const calls: string[] = [];
|
|
const sleeps: number[] = [];
|
|
const request: ApiRequester = async <T>(path: string) => {
|
|
calls.push(path);
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
throw unavailable;
|
|
};
|
|
|
|
await expect(
|
|
reconcileResources("shop", [], 1, undefined, request, {
|
|
sleep: async (milliseconds) => {
|
|
sleeps.push(milliseconds);
|
|
},
|
|
}),
|
|
).rejects.toBe(unavailable);
|
|
|
|
expect(calls).toEqual([
|
|
"/workspaces/shop/resources/plan",
|
|
"/workspaces/shop/resources/apply",
|
|
]);
|
|
expect(sleeps).toEqual([]);
|
|
});
|
|
|
|
test("never sleeps past the operation resume deadline", async () => {
|
|
let currentTime = 0;
|
|
const sleeps: Array<{ startedAt: number; milliseconds: number }> = [];
|
|
const request: ApiRequester = async <T>(path: string) => {
|
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
|
throw new TypeError("connection reset");
|
|
};
|
|
|
|
await expect(
|
|
reconcileResources("shop", [], 0, undefined, request, {
|
|
now: () => currentTime,
|
|
sleep: async (milliseconds) => {
|
|
sleeps.push({ startedAt: currentTime, milliseconds });
|
|
currentTime += milliseconds;
|
|
},
|
|
}),
|
|
).rejects.toThrow("connection reset");
|
|
|
|
expect(sleeps).not.toHaveLength(0);
|
|
expect(
|
|
sleeps.every(
|
|
({ startedAt, milliseconds }) => startedAt + milliseconds <= 60_000,
|
|
),
|
|
).toBe(true);
|
|
expect(sleeps.at(-1)).toEqual({
|
|
startedAt: 57_750,
|
|
milliseconds: 2_250,
|
|
});
|
|
expect(currentTime).toBe(60_000);
|
|
});
|
|
|
|
test("fails closed with the precise missing adoption route", async () => {
|
|
const request: ApiRequester = async <T>(path: string) => {
|
|
if (path.endsWith("/plan"))
|
|
throw new Error(
|
|
"Namespace shop is external; refusing workspace mutation",
|
|
);
|
|
return {} as T;
|
|
};
|
|
await expect(
|
|
reconcileResources("shop", [], 1, undefined, request),
|
|
).rejects.toThrow(`POST ${workspaceAdoptionRoute("shop")}`);
|
|
});
|
|
});
|