Files
buzz-sheet/lib/catalog/sync.ts
T
gunshiz 88afa8e947
CI / Verify (push) Successful in 1m48s
CI / Build immutable images and deploy (push) Successful in 3m21s
feat : let discord fetcher can stop working and accept every message type
2026-09-21 18:29:17 +07:00

755 lines
26 KiB
TypeScript

import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { sql } from "drizzle-orm";
import * as z from "zod";
import { getDb } from "@/db/client";
import {
catalogArtifacts,
catalogCharacters,
catalogMaterials,
catalogWeapons,
} from "@/db/schema";
import {
ascensionSnapshot,
levelingSnapshot,
avatarIconName,
baseStatsSnapshot,
characterDetailSchema,
constellationSnapshot,
nameCardImageName,
splashImageName,
talentSnapshot,
} from "@/lib/catalog/lunaris";
import type {
CatalogAscensionGroup,
CatalogLevelingGroup,
CatalogBaseStat,
CatalogConstellation,
CatalogMaterial,
CatalogTalent,
} from "@/lib/catalog/lunaris";
import { customTravelers, isUpstreamTraveler } from "@/lib/catalog/travelers";
import { withCatalogSyncLock } from "@/lib/catalog/sync-lock";
import { getRedisClient, redisEventPrefix } from "@/lib/redis/client";
import {
compareLunarisVersions,
lunarisVersionSchema,
} from "@/lib/catalog/version";
import { getMediaStorage } from "@/lib/media/storage";
const VERSION_URL = "https://api.lunaris.moe/data/version.json";
const API_ROOT = "https://api.lunaris.moe/data";
const PROGRESS_INTERVAL_MS = 400;
function syncConcurrency(): number {
const configured = Number(process.env.CATALOG_SYNC_CONCURRENCY ?? 12);
return Number.isFinite(configured)
? Math.max(1, Math.min(24, Math.floor(configured)))
: 12;
}
const versionSchema = z.object({
version: lunarisVersionSchema,
timestamp: z.number().int().optional(),
});
const currentVersionSchema = z.object({ version: lunarisVersionSchema });
const characterSchema = z.record(
z.string(),
z.object({
enName: z.string(),
CardImg: z.string(),
element: z.string().optional(),
qualityType: z.string().optional(),
weaponType: z.string().optional(),
}),
);
const weaponSchema = z.record(
z.string(),
z.object({
enName: z.string(),
weaponIcon: z.string(),
qualityType: z.string().optional(),
weaponType: z.string().optional(),
}),
);
const artifactSchema = z.record(
z.string(),
z.object({
enName: z.string(),
setIcon: z.string(),
qualityType: z.string().optional(),
}),
);
const materialSchema = z.record(
z.string(),
z.object({
enName: z.string(),
enDescription: z.string().optional(),
icon: z.string(),
qualityType: z.string().optional(),
type: z.string().optional(),
}),
);
interface SyncedCharacter {
key: string;
name: string;
element: string | null;
rarity: number | null;
weaponType: string | null;
imageKey: string;
elementIconKey: string | null;
splashImageKey: string | null;
nameCardImageKey: string | null;
sourceImage: string | null;
splashSourceImage: string | null;
nameCardSourceImage: string | null;
localImage: string | null;
constellations: CatalogConstellation[];
talents: CatalogTalent[];
ascension: CatalogAscensionGroup[];
leveling: CatalogLevelingGroup[];
baseStats: CatalogBaseStat[];
}
type SyncControl = { signal?: AbortSignal; checkCancelled: () => Promise<void>; report: (phase: string, completed: number, total: number, message: string) => Promise<void> };
async function retryTimeout<T>(label: string, task: () => Promise<T>, signal?: AbortSignal): Promise<T> {
for (let attempt = 1; ; attempt += 1) {
signal?.throwIfAborted();
try {
return await task();
} catch (cause) {
signal?.throwIfAborted();
const message = cause instanceof Error ? cause.message : String(cause);
if (attempt >= 3 || !/timed out|timeout/i.test(message)) {
throw new Error(`${label}: ${message}`, { cause });
}
await new Promise((resolve) => setTimeout(resolve, attempt * 1_000));
}
}
}
async function getJson(url: string, signal?: AbortSignal): Promise<unknown> {
return retryTimeout(`Lunaris request ${url}`, async () => {
const response = await fetch(url, {
signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(60_000)]) : AbortSignal.timeout(60_000),
cache: "no-store",
});
if (!response.ok)
throw new Error(`Lunaris returned ${response.status} for ${url}`);
return response.json();
}, signal);
}
async function getOptionalJson(url: string, signal?: AbortSignal): Promise<unknown | null> {
return retryTimeout(`Lunaris request ${url}`, async () => {
const response = await fetch(url, {
signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(60_000)]) : AbortSignal.timeout(60_000),
cache: "no-store",
});
if (response.status === 404) return null;
if (!response.ok)
throw new Error(`Lunaris returned ${response.status} for ${url}`);
return response.json();
}, signal);
}
function webpName(value: string): string {
return value.endsWith(".webp") ? value : `${value}.webp`;
}
function pngName(value: string): string {
return value.endsWith(".png") ? value : `${value}.png`;
}
function rarity(value: string | undefined): number | null {
if (!value) return null;
return (
{
QUALITY_WHITE: 1,
QUALITY_GREEN: 2,
QUALITY_BLUE: 3,
QUALITY_PURPLE: 4,
QUALITY_ORANGE: 5,
}[value] ?? null
);
}
export async function getLatestLunarisVersion(
signal?: AbortSignal,
): Promise<string> {
return versionSchema.parse(await getJson(VERSION_URL, signal)).version;
}
async function mapConcurrent<T, R>(
items: T[],
task: (item: T) => Promise<R>,
control?: SyncControl,
): Promise<R[]> {
const results = new Array<R>(items.length);
let next = 0;
let completed = 0;
let lastProgressAt = 0;
async function reportProgress() {
if (!control) return;
const now = Date.now();
if (completed < items.length && now - lastProgressAt < PROGRESS_INTERVAL_MS) return;
lastProgressAt = now;
await control.report("characters", completed, items.length, `ดึงข้อมูลตัวละคร ${completed}/${items.length}`);
}
async function worker() {
while (true) {
const index = next++;
if (index >= items.length) return;
await control?.checkCancelled();
results[index] = await task(items[index]);
completed += 1;
await reportProgress();
}
}
await Promise.all(
Array.from({ length: Math.min(syncConcurrency(), items.length) }, () => worker()),
);
return results;
}
async function uploadAssets(
assets: Array<{
source?: string;
body?: Uint8Array;
target: string;
type?: string;
}>,
control?: SyncControl,
): Promise<Set<string>> {
const storage = await getMediaStorage();
let next = 0;
let completed = 0;
let lastProgressAt = 0;
const missing = new Set<string>();
async function reportProgress() {
if (!control) return;
const now = Date.now();
if (completed < assets.length && now - lastProgressAt < PROGRESS_INTERVAL_MS) return;
lastProgressAt = now;
await control.report("assets", completed, assets.length, `ดาวน์โหลดรูปภาพ ${completed}/${assets.length}`);
}
async function worker() {
while (true) {
const index = next++;
if (index >= assets.length) return;
await control?.checkCancelled();
const asset = assets[index];
await retryTimeout(`Asset ${asset.target}`, async () => {
if (!(await storage.file(asset.target).exists())) {
if (asset.body) {
await storage.write(asset.target, asset.body, {
type: asset.type ?? "image/webp",
acl: "public-read",
});
} else {
const response = await fetch(asset.source!, {
signal: control?.signal ? AbortSignal.any([control.signal, AbortSignal.timeout(60_000)]) : AbortSignal.timeout(60_000),
cache: "no-store",
});
if (response.status === 404) missing.add(asset.target);
else if (!response.ok)
throw new Error(`Asset returned ${response.status}: ${asset.source}`);
else
await storage.write(asset.target, response, {
type: asset.type ?? "image/webp",
acl: "public-read",
});
}
}
}, control?.signal);
completed += 1;
await reportProgress();
}
}
await Promise.all(
Array.from({ length: Math.min(syncConcurrency(), assets.length) }, () => worker()),
);
return missing;
}
async function syncCatalogVersion(version: string, control?: SyncControl): Promise<void> {
await control?.report("lists", 0, 4, "กำลังดึงรายการ Catalog จาก Lunaris");
const [charactersRaw, weaponsRaw, artifactsRaw, materialsRaw] = await Promise.all([
getJson(`${API_ROOT}/${version}/charlist.json`, control?.signal),
getJson(`${API_ROOT}/${version}/weaponlist.json`, control?.signal),
getJson(`${API_ROOT}/${version}/artifactlist.json`, control?.signal),
getJson(`${API_ROOT}/${version}/materiallist.json`, control?.signal),
]);
await control?.report("lists", 4, 4, "ได้รับรายการ Catalog แล้ว");
const characterData = characterSchema.parse(charactersRaw);
const weaponData = weaponSchema.parse(weaponsRaw);
const artifactData = artifactSchema.parse(artifactsRaw);
const materialData = materialSchema.parse(materialsRaw);
const materials: CatalogMaterial[] = Object.entries(materialData).map(
([key, value]) => ({
key,
name: value.enName,
description: value.enDescription ?? null,
rarity: rarity(value.qualityType),
type: value.type ?? null,
imageKey: `catalog/materials/${pngName(value.icon)}`,
}),
);
const remoteCharacters: SyncedCharacter[] = await mapConcurrent(
Object.entries(characterData).filter(
([, value]) =>
value.enName &&
value.enName !== "???" &&
!isUpstreamTraveler(value.enName) &&
value.CardImg,
),
async ([key, value]) => {
const detailRaw = await getOptionalJson(
`${API_ROOT}/${version}/en/char/${encodeURIComponent(key)}.json`,
control?.signal,
);
const detail = detailRaw
? characterDetailSchema.parse(detailRaw)
: null;
const avatarIcon = webpName(avatarIconName(value.CardImg));
const splashImage = splashImageName(value.CardImg);
const nameCardImage = nameCardImageName(value.CardImg);
const element = detail?.info?.element ?? value.element ?? null;
return {
key,
name: value.enName,
element,
rarity: rarity(value.qualityType),
weaponType: detail?.info?.weapon ?? value.weaponType ?? null,
imageKey: `catalog/characters/avatar/${avatarIcon}`,
elementIconKey: element
? `catalog/elements/${element.toLowerCase()}.png`
: null,
splashImageKey: `catalog/characters/splash/${splashImage}`,
nameCardImageKey: `catalog/characters/namecard/${nameCardImage}`,
sourceImage: `${API_ROOT}/assets/avataricon/${encodeURIComponent(avatarIcon)}`,
splashSourceImage: `${API_ROOT}/assets/gachaimg/${encodeURIComponent(splashImage)}`,
nameCardSourceImage: `${API_ROOT}/assets/namecardpic/${encodeURIComponent(nameCardImage)}`,
localImage: null,
constellations: detail
? constellationSnapshot(
detail,
(icon) => `catalog/characters/constellations/${webpName(icon)}`,
)
: [],
talents: detail
? talentSnapshot(
detail,
(icon) => `catalog/characters/talents/${webpName(icon)}`,
)
: [],
ascension: detail ? ascensionSnapshot(detail) : [],
leveling: detail ? levelingSnapshot(detail) : [],
baseStats: detail ? baseStatsSnapshot(detail) : [],
};
},
control,
);
const customCharacters: SyncedCharacter[] = await mapConcurrent(
[...customTravelers],
async (traveler) => {
const detail = characterDetailSchema.parse(
await getJson(
`${API_ROOT}/${version}/en/char/${traveler.detailKey}.json`,
control?.signal,
),
);
return {
key: traveler.key,
name: traveler.name,
element: traveler.element,
rarity: traveler.rarity,
weaponType: "WEAPON_SWORD_ONE_HAND",
imageKey: `catalog/characters/avatar/${traveler.asset}`,
elementIconKey: `catalog/elements/${traveler.element.toLowerCase()}.png`,
splashImageKey: null,
nameCardImageKey: null,
sourceImage: null,
splashSourceImage: null,
nameCardSourceImage: null,
localImage: traveler.asset,
constellations: constellationSnapshot(
detail,
(icon) => `catalog/characters/constellations/${webpName(icon)}`,
),
talents: talentSnapshot(
detail,
(icon) => `catalog/characters/talents/${webpName(icon)}`,
),
ascension: ascensionSnapshot(detail),
leveling: levelingSnapshot(detail),
baseStats: baseStatsSnapshot(detail),
};
},
control,
);
let characters = [...remoteCharacters, ...customCharacters];
let weapons = Object.entries(weaponData)
.filter(
([, value]) =>
value.enName && value.enName !== "???" && value.weaponIcon,
)
.map(([key, value]) => ({
key,
name: value.enName,
rarity: rarity(value.qualityType),
weaponType: value.weaponType ?? null,
imageKey: `catalog/weapons/${webpName(value.weaponIcon)}`,
sourceImage: `${API_ROOT}/assets/weaponicon/${encodeURIComponent(webpName(value.weaponIcon))}`,
}));
let artifacts = Object.entries(artifactData)
.filter(
([, value]) => value.enName && value.enName !== "???" && value.setIcon,
)
.map(([key, value]) => ({
key,
name: value.enName,
rarity: rarity(value.qualityType),
imageKey: `catalog/artifacts/${webpName(value.setIcon)}`,
sourceImage: `${API_ROOT}/assets/artifacts/${encodeURIComponent(webpName(value.setIcon))}`,
}));
const uniqueAssets = new Map<
string,
{ source?: string; body?: Uint8Array; target: string; type?: string }
>();
for (const character of characters) {
uniqueAssets.set(
character.imageKey,
character.localImage
? {
body: await readFile(
join(process.cwd(), "lib/catalog/assets", character.localImage),
),
target: character.imageKey,
}
: { source: character.sourceImage!, target: character.imageKey },
);
if (character.splashImageKey && character.splashSourceImage) {
uniqueAssets.set(character.splashImageKey, {
source: character.splashSourceImage,
target: character.splashImageKey,
type: "image/png",
});
}
if (character.nameCardImageKey && character.nameCardSourceImage) {
uniqueAssets.set(character.nameCardImageKey, {
source: character.nameCardSourceImage,
target: character.nameCardImageKey,
type: "image/png",
});
}
}
for (const item of [...weapons, ...artifacts])
uniqueAssets.set(item.imageKey, {
source: item.sourceImage,
target: item.imageKey,
});
for (const material of materials) {
const source = materialData[material.key].icon;
uniqueAssets.set(material.imageKey, {
source: `${API_ROOT}/assets/items/${encodeURIComponent(pngName(source))}`,
target: material.imageKey,
type: "image/png",
});
}
for (const character of characters) {
if (character.elementIconKey && character.element)
uniqueAssets.set(character.elementIconKey, {
source: `${API_ROOT}/assets/icons/${encodeURIComponent(character.element.toLowerCase())}.png`,
target: character.elementIconKey,
type: "image/png",
});
}
for (const character of characters) {
for (const constellation of character.constellations) {
if (constellation.iconKey)
uniqueAssets.set(constellation.iconKey, {
source: `${API_ROOT}/assets/skills/${encodeURIComponent(constellation.iconKey.split("/").at(-1)!)}`,
target: constellation.iconKey,
});
}
for (const talent of character.talents) {
if (talent.iconKey)
uniqueAssets.set(talent.iconKey, {
source: `${API_ROOT}/assets/skills/${encodeURIComponent(talent.iconKey.split("/").at(-1)!)}`,
target: talent.iconKey,
});
}
}
await control?.report("assets", 0, uniqueAssets.size, "กำลังดาวน์โหลดรูปภาพ Catalog");
const missingAssets = await uploadAssets([...uniqueAssets.values()], control);
characters = characters.filter((item) => !missingAssets.has(item.imageKey));
characters = characters.map((item) => ({
...item,
elementIconKey:
item.elementIconKey && !missingAssets.has(item.elementIconKey)
? item.elementIconKey
: null,
splashImageKey:
item.splashImageKey && !missingAssets.has(item.splashImageKey)
? item.splashImageKey
: null,
nameCardImageKey:
item.nameCardImageKey && !missingAssets.has(item.nameCardImageKey)
? item.nameCardImageKey
: null,
constellations: item.constellations.map((constellation) => ({
...constellation,
iconKey:
constellation.iconKey && !missingAssets.has(constellation.iconKey)
? constellation.iconKey
: null,
})),
talents: item.talents.map((talent) => ({
...talent,
iconKey:
talent.iconKey && !missingAssets.has(talent.iconKey)
? talent.iconKey
: null,
})),
}));
weapons = weapons.filter((item) => !missingAssets.has(item.imageKey));
artifacts = artifacts.filter((item) => !missingAssets.has(item.imageKey));
await control?.checkCancelled();
const storage = await getMediaStorage();
await Promise.all([
storage.write(
"catalog/characters.json",
JSON.stringify(
characters.map((item) => ({
key: item.key,
name: item.name,
element: item.element,
rarity: item.rarity,
weaponType: item.weaponType,
imageKey: item.imageKey,
elementIconKey: item.elementIconKey,
splashImageKey: item.splashImageKey,
nameCardImageKey: item.nameCardImageKey,
constellations: item.constellations,
talents: item.talents,
ascension: item.ascension,
leveling: item.leveling,
baseStats: item.baseStats,
})),
),
{ type: "application/json", acl: "public-read" },
),
storage.write(
"catalog/weapons.json",
JSON.stringify(
weapons.map((item) => ({
key: item.key,
name: item.name,
rarity: item.rarity,
weaponType: item.weaponType,
imageKey: item.imageKey,
})),
),
{ type: "application/json", acl: "public-read" },
),
storage.write(
"catalog/artifacts.json",
JSON.stringify(
artifacts.map((item) => ({
key: item.key,
name: item.name,
rarity: item.rarity,
imageKey: item.imageKey,
})),
),
{ type: "application/json", acl: "public-read" },
),
storage.write(
"catalog/materials.json",
JSON.stringify(materials),
{ type: "application/json", acl: "public-read" },
),
]);
await control?.checkCancelled();
await control?.report("database", 0, 1, "กำลังบันทึก Catalog");
await getDb().transaction(async (tx) => {
await tx.delete(catalogCharacters);
await tx.delete(catalogMaterials);
await tx.delete(catalogWeapons);
await tx.delete(catalogArtifacts);
if (characters.length)
await tx
.insert(catalogCharacters)
.values(
characters.map((item) => ({
key: item.key,
name: item.name,
element: item.element,
rarity: item.rarity,
weaponType: item.weaponType,
imageKey: item.imageKey,
elementIconKey: item.elementIconKey,
coopImageKey: item.splashImageKey,
nameCardImageKey: item.nameCardImageKey,
constellations: item.constellations,
talents: item.talents,
ascension: item.ascension,
leveling: item.leveling,
baseStats: item.baseStats,
})),
);
if (materials.length)
await tx.insert(catalogMaterials).values(materials);
if (weapons.length)
await tx
.insert(catalogWeapons)
.values(
weapons.map((item) => ({
key: item.key,
name: item.name,
rarity: item.rarity,
weaponType: item.weaponType,
imageKey: item.imageKey,
})),
);
if (artifacts.length)
await tx
.insert(catalogArtifacts)
.values(
artifacts.map((item) => ({
key: item.key,
name: item.name,
rarity: item.rarity,
imageKey: item.imageKey,
})),
);
});
await control?.report("database", 1, 1, "บันทึก Catalog แล้ว");
await storage.write("catalog/current.json", JSON.stringify({ version }), {
type: "application/json",
acl: "public-read",
});
}
export type CatalogSyncResult = "synced" | "unchanged";
export type CatalogSyncJob = {
id: string;
status: "queued" | "running" | "completed" | "unchanged" | "cancelled" | "error";
phase: string;
completed: number;
total: number;
message: string;
version?: string;
error?: string;
};
const syncJobKey = (id: string) => `${redisEventPrefix()}:catalog-sync-job:${id}`;
async function writeSyncJob(job: CatalogSyncJob) {
const redis = await getRedisClient();
await redis.set(syncJobKey(job.id), JSON.stringify(job), "EX", 60 * 60);
return job;
}
export async function getCatalogSyncJob(id: string): Promise<CatalogSyncJob | null> {
const redis = await getRedisClient();
const value = await redis.get(syncJobKey(id));
return value ? JSON.parse(value) as CatalogSyncJob : null;
}
export async function cancelCatalogSyncJob(id: string): Promise<CatalogSyncJob | null> {
const job = await getCatalogSyncJob(id);
if (!job || ["completed", "unchanged", "cancelled", "error"].includes(job.status)) return job;
return writeSyncJob({ ...job, message: "กำลังยกเลิก Sync…", status: "cancelled" });
}
export async function startCatalogSyncJob(): Promise<CatalogSyncJob> {
const id = crypto.randomUUID();
const job = await writeSyncJob({ id, status: "queued", phase: "queued", completed: 0, total: 0, message: "กำลังเตรียม Sync" });
void executeCatalogSyncJob(job).catch(() => undefined);
return job;
}
async function executeCatalogSyncJob(initial: CatalogSyncJob) {
let lastCancellationCheck = 0;
let cancellationCheck: Promise<void> | null = null;
const checkCancelled = async () => {
if (Date.now() - lastCancellationCheck < PROGRESS_INTERVAL_MS) return;
if (!cancellationCheck) {
cancellationCheck = (async () => {
const current = await getCatalogSyncJob(initial.id);
lastCancellationCheck = Date.now();
if (current?.status === "cancelled") throw new Error("CATALOG_SYNC_CANCELLED");
})().finally(() => {
cancellationCheck = null;
});
}
await cancellationCheck;
};
const report = async (phase: string, completed: number, total: number, message: string) => {
const current = await getCatalogSyncJob(initial.id);
if (current?.status === "cancelled") throw new Error("CATALOG_SYNC_CANCELLED");
await writeSyncJob({ ...initial, status: "running", phase, completed, total, message });
};
try {
await writeSyncJob({ ...initial, status: "running", phase: "version", message: "กำลังตรวจสอบเวอร์ชัน Lunaris" });
const result = await processCatalogSync(undefined, { checkCancelled, report });
await writeSyncJob({ ...initial, status: result === "unchanged" ? "unchanged" : "completed", phase: "done", completed: 1, total: 1, message: result === "unchanged" ? "Catalog เป็นเวอร์ชันล่าสุดแล้ว" : "Sync Catalog สำเร็จ" });
} catch (cause) {
if (cause instanceof Error && cause.message === "CATALOG_SYNC_CANCELLED") {
await writeSyncJob({ ...initial, status: "cancelled", phase: "cancelled", message: "ยกเลิก Sync แล้ว" });
} else {
await writeSyncJob({ ...initial, status: "error", phase: "error", message: "Sync ไม่สำเร็จ", error: cause instanceof Error ? cause.message : "Unknown sync error" });
}
}
}
export async function getCurrentCatalogVersion(): Promise<string | null> {
const storage = await getMediaStorage();
const current = storage.file("catalog/current.json");
if (!(await current.exists())) return null;
return currentVersionSchema.parse(await current.json()).version;
}
async function hasCatalogRecords(): Promise<boolean> {
const db = getDb();
const [[character], [material], [weapon], [artifact]] = await Promise.all([
db.select({ key: catalogCharacters.key }).from(catalogCharacters).where(sql`${catalogCharacters.baseStats} <> '[]'::jsonb AND ${catalogCharacters.leveling} <> '[]'::jsonb`).limit(1),
db.select({ key: catalogMaterials.key }).from(catalogMaterials).limit(1),
db.select({ key: catalogWeapons.key }).from(catalogWeapons).limit(1),
db.select({ key: catalogArtifacts.key }).from(catalogArtifacts).limit(1),
]);
return Boolean(character && material && weapon && artifact);
}
export async function processCatalogSync(
requestedVersion?: string,
control?: SyncControl,
): Promise<CatalogSyncResult> {
const version = lunarisVersionSchema.parse(
requestedVersion ?? (await getLatestLunarisVersion()),
);
return withCatalogSyncLock(async () => {
const currentVersion = await getCurrentCatalogVersion();
if (
currentVersion &&
compareLunarisVersions(version, currentVersion) <= 0 &&
await hasCatalogRecords()
) {
return "unchanged";
}
await control?.checkCancelled();
await syncCatalogVersion(version, control);
return "synced";
});
}