import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { sql } from "drizzle-orm"; import * as z from "zod"; import { getDb } from "@/db/client"; import { catalogArtifacts, catalogCharacters, catalogMaterials, catalogWeapons, } from "@/db/schema"; import { ascensionSnapshot, levelingSnapshot, avatarIconName, baseStatsSnapshot, characterDetailSchema, constellationSnapshot, nameCardImageName, splashImageName, talentSnapshot, } from "@/lib/catalog/lunaris"; import type { CatalogAscensionGroup, CatalogLevelingGroup, CatalogBaseStat, CatalogConstellation, CatalogMaterial, CatalogTalent, } from "@/lib/catalog/lunaris"; import { customTravelers, isUpstreamTraveler } from "@/lib/catalog/travelers"; import { withCatalogSyncLock } from "@/lib/catalog/sync-lock"; import { getRedisClient, redisEventPrefix } from "@/lib/redis/client"; import { compareLunarisVersions, lunarisVersionSchema, } from "@/lib/catalog/version"; import { getMediaStorage } from "@/lib/media/storage"; const VERSION_URL = "https://api.lunaris.moe/data/version.json"; const API_ROOT = "https://api.lunaris.moe/data"; const PROGRESS_INTERVAL_MS = 400; function syncConcurrency(): number { const configured = Number(process.env.CATALOG_SYNC_CONCURRENCY ?? 12); return Number.isFinite(configured) ? Math.max(1, Math.min(24, Math.floor(configured))) : 12; } const versionSchema = z.object({ version: lunarisVersionSchema, timestamp: z.number().int().optional(), }); const currentVersionSchema = z.object({ version: lunarisVersionSchema }); const characterSchema = z.record( z.string(), z.object({ enName: z.string(), CardImg: z.string(), element: z.string().optional(), qualityType: z.string().optional(), weaponType: z.string().optional(), }), ); const weaponSchema = z.record( z.string(), z.object({ enName: z.string(), weaponIcon: z.string(), qualityType: z.string().optional(), weaponType: z.string().optional(), }), ); const artifactSchema = z.record( z.string(), z.object({ enName: z.string(), setIcon: z.string(), qualityType: z.string().optional(), }), ); const materialSchema = z.record( z.string(), z.object({ enName: z.string(), enDescription: z.string().optional(), icon: z.string(), qualityType: z.string().optional(), type: z.string().optional(), }), ); interface SyncedCharacter { key: string; name: string; element: string | null; rarity: number | null; weaponType: string | null; imageKey: string; elementIconKey: string | null; splashImageKey: string | null; nameCardImageKey: string | null; sourceImage: string | null; splashSourceImage: string | null; nameCardSourceImage: string | null; localImage: string | null; constellations: CatalogConstellation[]; talents: CatalogTalent[]; ascension: CatalogAscensionGroup[]; leveling: CatalogLevelingGroup[]; baseStats: CatalogBaseStat[]; } type SyncControl = { signal?: AbortSignal; checkCancelled: () => Promise; report: (phase: string, completed: number, total: number, message: string) => Promise }; async function retryTransient(label: string, task: () => Promise, signal?: AbortSignal): Promise { for (let attempt = 1; ; attempt += 1) { signal?.throwIfAborted(); try { return await task(); } catch (cause) { signal?.throwIfAborted(); const message = cause instanceof Error ? cause.message : String(cause); if (attempt >= 3 || !/timed out|timeout|unexpected error has occurred/i.test(message)) { throw new Error(`${label}: ${message}`, { cause }); } await new Promise((resolve) => setTimeout(resolve, attempt * 1_000)); } } } async function getJson(url: string, signal?: AbortSignal): Promise { return retryTransient(`Lunaris request ${url}`, async () => { const response = await fetch(url, { signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(60_000)]) : AbortSignal.timeout(60_000), cache: "no-store", }); if (!response.ok) throw new Error(`Lunaris returned ${response.status} for ${url}`); return response.json(); }, signal); } async function getOptionalJson(url: string, signal?: AbortSignal): Promise { return retryTransient(`Lunaris request ${url}`, async () => { const response = await fetch(url, { signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(60_000)]) : AbortSignal.timeout(60_000), cache: "no-store", }); if (response.status === 404) return null; if (!response.ok) throw new Error(`Lunaris returned ${response.status} for ${url}`); return response.json(); }, signal); } function webpName(value: string): string { return value.endsWith(".webp") ? value : `${value}.webp`; } function pngName(value: string): string { return value.endsWith(".png") ? value : `${value}.png`; } function rarity(value: string | undefined): number | null { if (!value) return null; return ( { QUALITY_WHITE: 1, QUALITY_GREEN: 2, QUALITY_BLUE: 3, QUALITY_PURPLE: 4, QUALITY_ORANGE: 5, }[value] ?? null ); } export async function getLatestLunarisVersion( signal?: AbortSignal, ): Promise { return versionSchema.parse(await getJson(VERSION_URL, signal)).version; } async function mapConcurrent( items: T[], task: (item: T) => Promise, control?: SyncControl, ): Promise { const results = new Array(items.length); let next = 0; let completed = 0; let lastProgressAt = 0; async function reportProgress() { if (!control) return; const now = Date.now(); if (completed < items.length && now - lastProgressAt < PROGRESS_INTERVAL_MS) return; lastProgressAt = now; await control.report("characters", completed, items.length, `ดึงข้อมูลตัวละคร ${completed}/${items.length}`); } async function worker() { while (true) { const index = next++; if (index >= items.length) return; await control?.checkCancelled(); results[index] = await task(items[index]); completed += 1; await reportProgress(); } } await Promise.all( Array.from({ length: Math.min(syncConcurrency(), items.length) }, () => worker()), ); return results; } async function uploadAssets( assets: Array<{ source?: string; body?: Uint8Array; target: string; type?: string; }>, control?: SyncControl, ): Promise> { const storage = await getMediaStorage(); let next = 0; let completed = 0; let lastProgressAt = 0; const missing = new Set(); async function reportProgress(currentAsset?: string) { if (!control) return; const now = Date.now(); if (completed < assets.length && now - lastProgressAt < PROGRESS_INTERVAL_MS) return; lastProgressAt = now; const message = currentAsset ? `กำลังดาวน์โหลดรูปภาพ ${Math.min(completed + 1, assets.length)}/${assets.length}\n${currentAsset.split("/").at(-1)}` : `ดาวน์โหลดรูปภาพ ${completed}/${assets.length}`; await control.report("assets", completed, assets.length, message); } async function worker() { while (true) { const index = next++; if (index >= assets.length) return; await control?.checkCancelled(); const asset = assets[index]; await reportProgress(asset.target); await retryTransient(`Asset ${asset.target}`, async () => { if (!(await storage.file(asset.target).exists())) { if (asset.body) { await storage.write(asset.target, asset.body, { type: asset.type ?? "image/webp", acl: "public-read", }); } else { const response = await fetch(asset.source!, { signal: control?.signal ? AbortSignal.any([control.signal, AbortSignal.timeout(60_000)]) : AbortSignal.timeout(60_000), cache: "no-store", }); if (response.status === 404) missing.add(asset.target); else if (!response.ok) throw new Error(`Asset returned ${response.status}: ${asset.source}`); else await storage.write(asset.target, response, { type: asset.type ?? "image/webp", acl: "public-read", }); } } }, control?.signal); completed += 1; await reportProgress(); } } await Promise.all( Array.from({ length: Math.min(syncConcurrency(), assets.length) }, () => worker()), ); return missing; } async function syncCatalogVersion(version: string, control?: SyncControl): Promise { await control?.report("lists", 0, 4, "กำลังดึงรายการ Catalog จาก Lunaris"); const [charactersRaw, weaponsRaw, artifactsRaw, materialsRaw] = await Promise.all([ getJson(`${API_ROOT}/${version}/charlist.json`, control?.signal), getJson(`${API_ROOT}/${version}/weaponlist.json`, control?.signal), getJson(`${API_ROOT}/${version}/artifactlist.json`, control?.signal), getJson(`${API_ROOT}/${version}/materiallist.json`, control?.signal), ]); await control?.report("lists", 4, 4, "ได้รับรายการ Catalog แล้ว"); const characterData = characterSchema.parse(charactersRaw); const weaponData = weaponSchema.parse(weaponsRaw); const artifactData = artifactSchema.parse(artifactsRaw); const materialData = materialSchema.parse(materialsRaw); const materials: CatalogMaterial[] = Object.entries(materialData).map( ([key, value]) => ({ key, name: value.enName, description: value.enDescription ?? null, rarity: rarity(value.qualityType), type: value.type ?? null, imageKey: `catalog/materials/${pngName(value.icon)}`, }), ); const remoteCharacters: SyncedCharacter[] = await mapConcurrent( Object.entries(characterData).filter( ([, value]) => value.enName && value.enName !== "???" && !isUpstreamTraveler(value.enName) && value.CardImg, ), async ([key, value]) => { const detailRaw = await getOptionalJson( `${API_ROOT}/${version}/en/char/${encodeURIComponent(key)}.json`, control?.signal, ); const detail = detailRaw ? characterDetailSchema.parse(detailRaw) : null; const avatarIcon = webpName(avatarIconName(value.CardImg)); const splashImage = splashImageName(value.CardImg); const nameCardImage = nameCardImageName(value.CardImg); const element = detail?.info?.element ?? value.element ?? null; return { key, name: value.enName, element, rarity: rarity(value.qualityType), weaponType: detail?.info?.weapon ?? value.weaponType ?? null, imageKey: `catalog/characters/avatar/${avatarIcon}`, elementIconKey: element ? `catalog/elements/${element.toLowerCase()}.png` : null, splashImageKey: `catalog/characters/splash/${splashImage}`, nameCardImageKey: `catalog/characters/namecard/${nameCardImage}`, sourceImage: `${API_ROOT}/assets/avataricon/${encodeURIComponent(avatarIcon)}`, splashSourceImage: `${API_ROOT}/assets/gachaimg/${encodeURIComponent(splashImage)}`, nameCardSourceImage: `${API_ROOT}/assets/namecardpic/${encodeURIComponent(nameCardImage)}`, localImage: null, constellations: detail ? constellationSnapshot( detail, (icon) => `catalog/characters/constellations/${webpName(icon)}`, ) : [], talents: detail ? talentSnapshot( detail, (icon) => `catalog/characters/talents/${webpName(icon)}`, ) : [], ascension: detail ? ascensionSnapshot(detail) : [], leveling: detail ? levelingSnapshot(detail) : [], baseStats: detail ? baseStatsSnapshot(detail) : [], }; }, control, ); const customCharacters: SyncedCharacter[] = await mapConcurrent( [...customTravelers], async (traveler) => { const detail = characterDetailSchema.parse( await getJson( `${API_ROOT}/${version}/en/char/${traveler.detailKey}.json`, control?.signal, ), ); return { key: traveler.key, name: traveler.name, element: traveler.element, rarity: traveler.rarity, weaponType: "WEAPON_SWORD_ONE_HAND", imageKey: `catalog/characters/avatar/${traveler.asset}`, elementIconKey: `catalog/elements/${traveler.element.toLowerCase()}.png`, splashImageKey: null, nameCardImageKey: null, sourceImage: null, splashSourceImage: null, nameCardSourceImage: null, localImage: traveler.asset, constellations: constellationSnapshot( detail, (icon) => `catalog/characters/constellations/${webpName(icon)}`, ), talents: talentSnapshot( detail, (icon) => `catalog/characters/talents/${webpName(icon)}`, ), ascension: ascensionSnapshot(detail), leveling: levelingSnapshot(detail), baseStats: baseStatsSnapshot(detail), }; }, control, ); let characters = [...remoteCharacters, ...customCharacters]; let weapons = Object.entries(weaponData) .filter( ([, value]) => value.enName && value.enName !== "???" && value.weaponIcon, ) .map(([key, value]) => ({ key, name: value.enName, rarity: rarity(value.qualityType), weaponType: value.weaponType ?? null, imageKey: `catalog/weapons/${webpName(value.weaponIcon)}`, sourceImage: `${API_ROOT}/assets/weaponicon/${encodeURIComponent(webpName(value.weaponIcon))}`, })); let artifacts = Object.entries(artifactData) .filter( ([, value]) => value.enName && value.enName !== "???" && value.setIcon, ) .map(([key, value]) => ({ key, name: value.enName, rarity: rarity(value.qualityType), imageKey: `catalog/artifacts/${webpName(value.setIcon)}`, sourceImage: `${API_ROOT}/assets/artifacts/${encodeURIComponent(webpName(value.setIcon))}`, })); const uniqueAssets = new Map< string, { source?: string; body?: Uint8Array; target: string; type?: string } >(); for (const character of characters) { uniqueAssets.set( character.imageKey, character.localImage ? { body: await readFile( join(process.cwd(), "lib/catalog/assets", character.localImage), ), target: character.imageKey, } : { source: character.sourceImage!, target: character.imageKey }, ); if (character.splashImageKey && character.splashSourceImage) { uniqueAssets.set(character.splashImageKey, { source: character.splashSourceImage, target: character.splashImageKey, type: "image/png", }); } if (character.nameCardImageKey && character.nameCardSourceImage) { uniqueAssets.set(character.nameCardImageKey, { source: character.nameCardSourceImage, target: character.nameCardImageKey, type: "image/png", }); } } for (const item of [...weapons, ...artifacts]) uniqueAssets.set(item.imageKey, { source: item.sourceImage, target: item.imageKey, }); for (const material of materials) { const source = materialData[material.key].icon; uniqueAssets.set(material.imageKey, { source: `${API_ROOT}/assets/items/${encodeURIComponent(pngName(source))}`, target: material.imageKey, type: "image/png", }); } for (const character of characters) { if (character.elementIconKey && character.element) uniqueAssets.set(character.elementIconKey, { source: `${API_ROOT}/assets/icons/${encodeURIComponent(character.element.toLowerCase())}.png`, target: character.elementIconKey, type: "image/png", }); } for (const character of characters) { for (const constellation of character.constellations) { if (constellation.iconKey) uniqueAssets.set(constellation.iconKey, { source: `${API_ROOT}/assets/skills/${encodeURIComponent(constellation.iconKey.split("/").at(-1)!)}`, target: constellation.iconKey, }); } for (const talent of character.talents) { if (talent.iconKey) uniqueAssets.set(talent.iconKey, { source: `${API_ROOT}/assets/skills/${encodeURIComponent(talent.iconKey.split("/").at(-1)!)}`, target: talent.iconKey, }); } } await control?.report("assets", 0, uniqueAssets.size, "กำลังดาวน์โหลดรูปภาพ Catalog"); const missingAssets = await uploadAssets([...uniqueAssets.values()], control); characters = characters.filter((item) => !missingAssets.has(item.imageKey)); characters = characters.map((item) => ({ ...item, elementIconKey: item.elementIconKey && !missingAssets.has(item.elementIconKey) ? item.elementIconKey : null, splashImageKey: item.splashImageKey && !missingAssets.has(item.splashImageKey) ? item.splashImageKey : null, nameCardImageKey: item.nameCardImageKey && !missingAssets.has(item.nameCardImageKey) ? item.nameCardImageKey : null, constellations: item.constellations.map((constellation) => ({ ...constellation, iconKey: constellation.iconKey && !missingAssets.has(constellation.iconKey) ? constellation.iconKey : null, })), talents: item.talents.map((talent) => ({ ...talent, iconKey: talent.iconKey && !missingAssets.has(talent.iconKey) ? talent.iconKey : null, })), })); weapons = weapons.filter((item) => !missingAssets.has(item.imageKey)); artifacts = artifacts.filter((item) => !missingAssets.has(item.imageKey)); await control?.checkCancelled(); const storage = await getMediaStorage(); await Promise.all([ storage.write( "catalog/characters.json", JSON.stringify( characters.map((item) => ({ key: item.key, name: item.name, element: item.element, rarity: item.rarity, weaponType: item.weaponType, imageKey: item.imageKey, elementIconKey: item.elementIconKey, splashImageKey: item.splashImageKey, nameCardImageKey: item.nameCardImageKey, constellations: item.constellations, talents: item.talents, ascension: item.ascension, leveling: item.leveling, baseStats: item.baseStats, })), ), { type: "application/json", acl: "public-read" }, ), storage.write( "catalog/weapons.json", JSON.stringify( weapons.map((item) => ({ key: item.key, name: item.name, rarity: item.rarity, weaponType: item.weaponType, imageKey: item.imageKey, })), ), { type: "application/json", acl: "public-read" }, ), storage.write( "catalog/artifacts.json", JSON.stringify( artifacts.map((item) => ({ key: item.key, name: item.name, rarity: item.rarity, imageKey: item.imageKey, })), ), { type: "application/json", acl: "public-read" }, ), storage.write( "catalog/materials.json", JSON.stringify(materials), { type: "application/json", acl: "public-read" }, ), ]); await control?.checkCancelled(); await control?.report("database", 0, 1, "กำลังบันทึก Catalog"); await getDb().transaction(async (tx) => { await tx.delete(catalogCharacters); await tx.delete(catalogMaterials); await tx.delete(catalogWeapons); await tx.delete(catalogArtifacts); if (characters.length) await tx .insert(catalogCharacters) .values( characters.map((item) => ({ key: item.key, name: item.name, element: item.element, rarity: item.rarity, weaponType: item.weaponType, imageKey: item.imageKey, elementIconKey: item.elementIconKey, coopImageKey: item.splashImageKey, nameCardImageKey: item.nameCardImageKey, constellations: item.constellations, talents: item.talents, ascension: item.ascension, leveling: item.leveling, baseStats: item.baseStats, })), ); if (materials.length) await tx.insert(catalogMaterials).values(materials); if (weapons.length) await tx .insert(catalogWeapons) .values( weapons.map((item) => ({ key: item.key, name: item.name, rarity: item.rarity, weaponType: item.weaponType, imageKey: item.imageKey, })), ); if (artifacts.length) await tx .insert(catalogArtifacts) .values( artifacts.map((item) => ({ key: item.key, name: item.name, rarity: item.rarity, imageKey: item.imageKey, })), ); }); await control?.report("database", 1, 1, "บันทึก Catalog แล้ว"); await storage.write("catalog/current.json", JSON.stringify({ version }), { type: "application/json", acl: "public-read", }); } export type CatalogSyncResult = "synced" | "unchanged"; export type CatalogSyncJob = { id: string; status: "queued" | "running" | "completed" | "unchanged" | "cancelled" | "error"; phase: string; completed: number; total: number; message: string; version?: string; error?: string; }; const syncJobKey = (id: string) => `${redisEventPrefix()}:catalog-sync-job:${id}`; async function writeSyncJob(job: CatalogSyncJob) { const redis = await getRedisClient(); await redis.set(syncJobKey(job.id), JSON.stringify(job), "EX", 60 * 60); return job; } export async function getCatalogSyncJob(id: string): Promise { const redis = await getRedisClient(); const value = await redis.get(syncJobKey(id)); return value ? JSON.parse(value) as CatalogSyncJob : null; } export async function cancelCatalogSyncJob(id: string): Promise { const job = await getCatalogSyncJob(id); if (!job || ["completed", "unchanged", "cancelled", "error"].includes(job.status)) return job; return writeSyncJob({ ...job, message: "กำลังยกเลิก Sync…", status: "cancelled" }); } export async function startCatalogSyncJob(): Promise { const id = crypto.randomUUID(); const job = await writeSyncJob({ id, status: "queued", phase: "queued", completed: 0, total: 0, message: "กำลังเตรียม Sync" }); void executeCatalogSyncJob(job).catch(() => undefined); return job; } async function executeCatalogSyncJob(initial: CatalogSyncJob) { let lastCancellationCheck = 0; let cancellationCheck: Promise | null = null; const checkCancelled = async () => { if (Date.now() - lastCancellationCheck < PROGRESS_INTERVAL_MS) return; if (!cancellationCheck) { cancellationCheck = (async () => { const current = await getCatalogSyncJob(initial.id); lastCancellationCheck = Date.now(); if (current?.status === "cancelled") throw new Error("CATALOG_SYNC_CANCELLED"); })().finally(() => { cancellationCheck = null; }); } await cancellationCheck; }; const report = async (phase: string, completed: number, total: number, message: string) => { const current = await getCatalogSyncJob(initial.id); if (current?.status === "cancelled") throw new Error("CATALOG_SYNC_CANCELLED"); await writeSyncJob({ ...initial, status: "running", phase, completed, total, message }); }; try { await writeSyncJob({ ...initial, status: "running", phase: "version", message: "กำลังตรวจสอบเวอร์ชัน Lunaris" }); const result = await processCatalogSync(undefined, { checkCancelled, report }); await writeSyncJob({ ...initial, status: result === "unchanged" ? "unchanged" : "completed", phase: "done", completed: 1, total: 1, message: result === "unchanged" ? "Catalog เป็นเวอร์ชันล่าสุดแล้ว" : "Sync Catalog สำเร็จ" }); } catch (cause) { if (cause instanceof Error && cause.message === "CATALOG_SYNC_CANCELLED") { await writeSyncJob({ ...initial, status: "cancelled", phase: "cancelled", message: "ยกเลิก Sync แล้ว" }); } else { await writeSyncJob({ ...initial, status: "error", phase: "error", message: "Sync ไม่สำเร็จ", error: cause instanceof Error ? cause.message : "Unknown sync error" }); } } } export async function getCurrentCatalogVersion(): Promise { const storage = await getMediaStorage(); const current = storage.file("catalog/current.json"); if (!(await current.exists())) return null; return currentVersionSchema.parse(await current.json()).version; } async function hasCatalogRecords(): Promise { const db = getDb(); const [[character], [material], [weapon], [artifact]] = await Promise.all([ db.select({ key: catalogCharacters.key }).from(catalogCharacters).where(sql`${catalogCharacters.baseStats} <> '[]'::jsonb AND ${catalogCharacters.leveling} <> '[]'::jsonb`).limit(1), db.select({ key: catalogMaterials.key }).from(catalogMaterials).limit(1), db.select({ key: catalogWeapons.key }).from(catalogWeapons).limit(1), db.select({ key: catalogArtifacts.key }).from(catalogArtifacts).limit(1), ]); return Boolean(character && material && weapon && artifact); } export async function processCatalogSync( requestedVersion?: string, control?: SyncControl, ): Promise { const version = lunarisVersionSchema.parse( requestedVersion ?? (await getLatestLunarisVersion()), ); return withCatalogSyncLock(async () => { const currentVersion = await getCurrentCatalogVersion(); if ( currentVersion && compareLunarisVersions(version, currentVersion) <= 0 && await hasCatalogRecords() ) { return "unchanged"; } await control?.checkCancelled(); await syncCatalogVersion(version, control); return "synced"; }); }