import { asc, eq } from "drizzle-orm"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; import * as z from "zod"; import { getDb } from "@/db/client"; import { catalogArtifacts, catalogCharacters, catalogSyncJobs, catalogVersions, catalogWeapons, } from "@/db/schema"; import { avatarIconName, characterDetailSchema, constellationSnapshot, talentSnapshot, } from "@/lib/catalog/lunaris"; import type { CatalogConstellation, CatalogTalent } from "@/lib/catalog/lunaris"; import { customTravelers, isUpstreamTraveler } from "@/lib/catalog/travelers"; import { getMediaStorage } from "@/lib/media/storage"; const VERSION_URL = "https://api.lunaris.moe/data/version.json"; const API_ROOT = "https://api.lunaris.moe/data"; const versionSchema = z.object({ version: z.string().regex(/^\d+(?:\.\d+)+$/u), timestamp: z.number().int().optional(), }); const characterSchema = z.record(z.string(), z.object({ enName: z.string(), CardImg: z.string(), element: z.string().optional(), qualityType: z.string().optional(), weaponType: z.string().optional(), })); const weaponSchema = z.record(z.string(), z.object({ enName: z.string(), weaponIcon: z.string(), qualityType: z.string().optional(), weaponType: z.string().optional(), })); const artifactSchema = z.record(z.string(), z.object({ enName: z.string(), setIcon: z.string(), qualityType: z.string().optional(), })); interface SyncedCharacter { version: string; key: string; name: string; element: string | null; rarity: number | null; weaponType: string | null; imageKey: string; elementIconKey: string | null; sourceImage: string | null; localImage: string | null; constellations: CatalogConstellation[]; talents: CatalogTalent[]; } async function getJson(url: string): Promise { const response = await fetch(url, { signal: AbortSignal.timeout(30_000), cache: "no-store" }); if (!response.ok) throw new Error(`Lunaris returned ${response.status} for ${url}`); return response.json(); } async function getOptionalJson(url: string): Promise { const response = await fetch(url, { signal: AbortSignal.timeout(30_000), cache: "no-store" }); if (response.status === 404) return null; if (!response.ok) throw new Error(`Lunaris returned ${response.status} for ${url}`); return response.json(); } function webpName(value: string): string { return value.endsWith(".webp") ? value : `${value}.webp`; } function rarity(value: string | undefined): number | null { if (!value) return null; return { QUALITY_WHITE: 1, QUALITY_GREEN: 2, QUALITY_BLUE: 3, QUALITY_PURPLE: 4, QUALITY_ORANGE: 5 }[value] ?? null; } async function mapConcurrent(items: T[], task: (item: T) => Promise): Promise { const results = new Array(items.length); let next = 0; async function worker() { while (next < items.length) { const index = next++; results[index] = await task(items[index]); } } await Promise.all(Array.from({ length: Math.min(4, items.length) }, () => worker())); return results; } async function uploadAssets( jobId: string, assets: Array<{ source?: string; body?: Uint8Array; target: string; type?: string }>, ): Promise> { const storage = await getMediaStorage(); let next = 0; let completed = 0; const missing = new Set(); await getDb().update(catalogSyncJobs).set({ total: assets.length, progress: 0 }).where(eq(catalogSyncJobs.id, jobId)); async function worker() { while (next < assets.length) { const asset = assets[next++]; if (asset.body) { await storage.write(asset.target, asset.body, { type: asset.type ?? "image/webp", acl: "public-read" }); } else { const response = await fetch(asset.source!, { signal: AbortSignal.timeout(30_000), cache: "no-store" }); if (response.status === 404) missing.add(asset.target); else if (!response.ok) throw new Error(`Asset returned ${response.status}: ${asset.source}`); else await storage.write(asset.target, response, { type: asset.type ?? "image/webp", acl: "public-read" }); } completed += 1; if (completed % 10 === 0 || completed === assets.length) { await getDb().update(catalogSyncJobs).set({ progress: completed }).where(eq(catalogSyncJobs.id, jobId)); } } } await Promise.all(Array.from({ length: Math.min(4, assets.length) }, () => worker())); return missing; } export async function claimCatalogSyncJob() { const [pending] = await getDb().select().from(catalogSyncJobs).where(eq(catalogSyncJobs.status, "pending")).orderBy(asc(catalogSyncJobs.createdAt)).limit(1); if (!pending) return null; const [claimed] = await getDb().update(catalogSyncJobs).set({ status: "processing", startedAt: new Date(), error: null }).where(eq(catalogSyncJobs.id, pending.id)).returning(); return claimed ?? null; } export async function processCatalogSyncJob(jobId: string): Promise { try { const versionInfo = versionSchema.parse(await getJson(VERSION_URL)); const version = versionInfo.version; await getDb().update(catalogSyncJobs).set({ version }).where(eq(catalogSyncJobs.id, jobId)); const [charactersRaw, weaponsRaw, artifactsRaw] = await Promise.all([ getJson(`${API_ROOT}/${version}/charlist.json`), getJson(`${API_ROOT}/${version}/weaponlist.json`), getJson(`${API_ROOT}/${version}/artifactlist.json`), ]); const characterData = characterSchema.parse(charactersRaw); const weaponData = weaponSchema.parse(weaponsRaw); const artifactData = artifactSchema.parse(artifactsRaw); const remoteCharacters: SyncedCharacter[] = await mapConcurrent( Object.entries(characterData).filter(([, value]) => value.enName && value.enName !== "???" && !isUpstreamTraveler(value.enName) && value.CardImg), async ([key, value]) => { const detailRaw = await getOptionalJson(`${API_ROOT}/${version}/en/char/${encodeURIComponent(key)}.json`); const detail = detailRaw ? characterDetailSchema.parse(detailRaw) : null; const avatarIcon = webpName(avatarIconName(value.CardImg)); const element = detail?.info?.element ?? value.element ?? null; return { version, key, name: value.enName, element, rarity: rarity(value.qualityType), weaponType: detail?.info?.weapon ?? value.weaponType ?? null, imageKey: `catalog/${version}/characters/${avatarIcon}`, elementIconKey: element ? `catalog/${version}/elements/${element.toLowerCase()}.png` : null, sourceImage: `${API_ROOT}/assets/avataricon/${encodeURIComponent(avatarIcon)}`, localImage: null, constellations: detail ? constellationSnapshot(detail, (icon) => `catalog/${version}/constellations/${webpName(icon)}`) : [], talents: detail ? talentSnapshot(detail, (icon) => `catalog/${version}/talents/${webpName(icon)}`) : [], }; }, ); const customCharacters: SyncedCharacter[] = await mapConcurrent([...customTravelers], async (traveler) => { const detail = characterDetailSchema.parse(await getJson(`${API_ROOT}/${version}/en/char/${traveler.detailKey}.json`)); return { version, key: traveler.key, name: traveler.name, element: traveler.element, rarity: traveler.rarity, weaponType: "WEAPON_SWORD_ONE_HAND", imageKey: `catalog/${version}/characters/${traveler.asset}`, elementIconKey: `catalog/${version}/elements/${traveler.element.toLowerCase()}.png`, sourceImage: null, localImage: traveler.asset, constellations: constellationSnapshot(detail, (icon) => `catalog/${version}/constellations/${webpName(icon)}`), talents: talentSnapshot(detail, (icon) => `catalog/${version}/talents/${webpName(icon)}`), }; }); let characters = [...remoteCharacters, ...customCharacters]; let weapons = Object.entries(weaponData).filter(([, value]) => value.enName && value.enName !== "???" && value.weaponIcon).map(([key, value]) => ({ version, key, name: value.enName, rarity: rarity(value.qualityType), weaponType: value.weaponType ?? null, imageKey: `catalog/${version}/weapons/${webpName(value.weaponIcon)}`, sourceImage: `${API_ROOT}/assets/weaponicon/${encodeURIComponent(webpName(value.weaponIcon))}`, })); let artifacts = Object.entries(artifactData).filter(([, value]) => value.enName && value.enName !== "???" && value.setIcon).map(([key, value]) => ({ version, key, name: value.enName, rarity: rarity(value.qualityType), imageKey: `catalog/${version}/artifacts/${webpName(value.setIcon)}`, sourceImage: `${API_ROOT}/assets/artifacts/${encodeURIComponent(webpName(value.setIcon))}`, })); const uniqueAssets = new Map(); for (const character of characters) { uniqueAssets.set(character.imageKey, character.localImage ? { body: await readFile(join(process.cwd(), "lib/catalog/assets", character.localImage)), target: character.imageKey, } : { source: character.sourceImage!, target: character.imageKey }); } for (const item of [...weapons, ...artifacts]) uniqueAssets.set(item.imageKey, { source: item.sourceImage, target: item.imageKey }); for (const character of characters) { if (character.elementIconKey && character.element) uniqueAssets.set(character.elementIconKey, { source: `${API_ROOT}/assets/icons/${encodeURIComponent(character.element.toLowerCase())}.png`, target: character.elementIconKey, type: "image/png", }); } for (const character of characters) { for (const constellation of character.constellations) { if (constellation.iconKey) uniqueAssets.set(constellation.iconKey, { source: `${API_ROOT}/assets/skills/${encodeURIComponent(constellation.iconKey.split("/").at(-1)!)}`, target: constellation.iconKey, }); } for (const talent of character.talents) { if (talent.iconKey) uniqueAssets.set(talent.iconKey, { source: `${API_ROOT}/assets/skills/${encodeURIComponent(talent.iconKey.split("/").at(-1)!)}`, target: talent.iconKey, }); } } const missingAssets = await uploadAssets(jobId, [...uniqueAssets.values()]); characters = characters.filter((item) => !missingAssets.has(item.imageKey)); characters = characters.map((item) => ({ ...item, elementIconKey: item.elementIconKey && !missingAssets.has(item.elementIconKey) ? item.elementIconKey : null, constellations: item.constellations.map((constellation) => ({ ...constellation, iconKey: constellation.iconKey && !missingAssets.has(constellation.iconKey) ? constellation.iconKey : null })), talents: item.talents.map((talent) => ({ ...talent, iconKey: talent.iconKey && !missingAssets.has(talent.iconKey) ? talent.iconKey : null })) })); weapons = weapons.filter((item) => !missingAssets.has(item.imageKey)); artifacts = artifacts.filter((item) => !missingAssets.has(item.imageKey)); const storage = await getMediaStorage(); await Promise.all([ storage.write(`catalog/${version}/characters.json`, JSON.stringify(characters.map((item) => ({ version: item.version, key: item.key, name: item.name, element: item.element, rarity: item.rarity, weaponType: item.weaponType, imageKey: item.imageKey, elementIconKey: item.elementIconKey, constellations: item.constellations, talents: item.talents }))), { type: "application/json", acl: "public-read" }), storage.write(`catalog/${version}/weapons.json`, JSON.stringify(weapons.map((item) => ({ version: item.version, key: item.key, name: item.name, rarity: item.rarity, weaponType: item.weaponType, imageKey: item.imageKey }))), { type: "application/json", acl: "public-read" }), storage.write(`catalog/${version}/artifacts.json`, JSON.stringify(artifacts.map((item) => ({ version: item.version, key: item.key, name: item.name, rarity: item.rarity, imageKey: item.imageKey }))), { type: "application/json", acl: "public-read" }), ]); await getDb().transaction(async (tx) => { await tx.insert(catalogVersions).values({ version, sourceTimestamp: versionInfo.timestamp ?? null }).onConflictDoUpdate({ target: catalogVersions.version, set: { sourceTimestamp: versionInfo.timestamp ?? null, syncedAt: new Date() } }); await tx.delete(catalogCharacters).where(eq(catalogCharacters.version, version)); await tx.delete(catalogWeapons).where(eq(catalogWeapons.version, version)); await tx.delete(catalogArtifacts).where(eq(catalogArtifacts.version, version)); if (characters.length) await tx.insert(catalogCharacters).values(characters.map((item) => ({ version: item.version, key: item.key, name: item.name, element: item.element, rarity: item.rarity, weaponType: item.weaponType, imageKey: item.imageKey, elementIconKey: item.elementIconKey, constellations: item.constellations, talents: item.talents }))); if (weapons.length) await tx.insert(catalogWeapons).values(weapons.map((item) => ({ version: item.version, key: item.key, name: item.name, rarity: item.rarity, weaponType: item.weaponType, imageKey: item.imageKey }))); if (artifacts.length) await tx.insert(catalogArtifacts).values(artifacts.map((item) => ({ version: item.version, key: item.key, name: item.name, rarity: item.rarity, imageKey: item.imageKey }))); await tx.update(catalogVersions).set({ active: false }); await tx.update(catalogVersions).set({ active: true, syncedAt: new Date() }).where(eq(catalogVersions.version, version)); await tx.update(catalogSyncJobs).set({ status: "completed", completedAt: new Date(), progress: uniqueAssets.size, total: uniqueAssets.size }).where(eq(catalogSyncJobs.id, jobId)); }); await storage.write("catalog/current.json", JSON.stringify({ version }), { type: "application/json", acl: "public-read" }); } catch (cause) { await getDb().update(catalogSyncJobs).set({ status: "failed", completedAt: new Date(), error: cause instanceof Error ? cause.message.slice(0, 4_000) : "Unknown sync failure" }).where(eq(catalogSyncJobs.id, jobId)); throw cause; } }