astral/buzz-sheet
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove catalog sync jobs and version history
CI / Verify (push) Successful in 58s
Details
CI / Build immutable images and deploy (push) Successful in 1m19s
Details

Browse Source
This commit is contained in:
gunshiz
2026-08-31 23:21:14 +07:00 Unverified
parent 737b54bff8
commit bb06108899
14 changed files with 6865 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/catalog/sync.ts
+21 -130
View File
@@ -1,4 +1,3 @@
import { asc, eq } from "drizzle-orm";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import * as z from "zod";
@@ -6,10 +5,7 @@ import * as z from "zod";
import { getDb } from "@/db/client";
import {
catalogArtifacts,
catalogCharacterCurrent,
catalogCharacters,
catalogSyncJobs,
catalogVersions,
catalogWeapons,
} from "@/db/schema";
import {
@@ -63,7 +59,6 @@ const artifactSchema = z.record(
);
interface SyncedCharacter {
version: string;
key: string;
name: string;
element: string | null;
@@ -138,7 +133,6 @@ async function mapConcurrent<T, R>(
}
async function uploadAssets(
jobId: string,
assets: Array<{
source?: string;
body?: Uint8Array;
@@ -148,12 +142,7 @@ async function uploadAssets(
): Promise<Set<string>> {
const storage = await getMediaStorage();
let next = 0;
let completed = 0;
const missing = new Set<string>();
await getDb()
.update(catalogSyncJobs)
.set({ total: assets.length, progress: 0 })
.where(eq(catalogSyncJobs.id, jobId));
async function worker() {
while (next < assets.length) {
const asset = assets[next++];
@@ -178,13 +167,6 @@ async function uploadAssets(
});
}
}
completed += 1;
if (completed % 10 === 0 || completed === assets.length) {
await getDb()
.update(catalogSyncJobs)
.set({ progress: completed })
.where(eq(catalogSyncJobs.id, jobId));
}
}
}
await Promise.all(
@@ -193,40 +175,19 @@ async function uploadAssets(
return missing;
}
export async function claimCatalogSyncJob() {
const [pending] = await getDb()
.select()
.from(catalogSyncJobs)
.where(eq(catalogSyncJobs.status, "pending"))
.orderBy(asc(catalogSyncJobs.createdAt))
.limit(1);
if (!pending) return null;
const [claimed] = await getDb()
.update(catalogSyncJobs)
.set({ status: "processing", startedAt: new Date(), error: null })
.where(eq(catalogSyncJobs.id, pending.id))
.returning();
return claimed ?? null;
}
export async function processCatalogSyncJob(jobId: string): Promise<void> {
try {
const versionInfo = versionSchema.parse(await getJson(VERSION_URL));
const version = versionInfo.version;
await getDb()
.update(catalogSyncJobs)
.set({ version })
.where(eq(catalogSyncJobs.id, jobId));
const [charactersRaw, weaponsRaw, artifactsRaw] = await Promise.all([
export async function processCatalogSync(): Promise<void> {
const versionInfo = versionSchema.parse(await getJson(VERSION_URL));
const version = versionInfo.version;
const [charactersRaw, weaponsRaw, artifactsRaw] = await Promise.all([
getJson(`${API_ROOT}/${version}/charlist.json`),
getJson(`${API_ROOT}/${version}/weaponlist.json`),
getJson(`${API_ROOT}/${version}/artifactlist.json`),
]);
const characterData = characterSchema.parse(charactersRaw);
const weaponData = weaponSchema.parse(weaponsRaw);
const artifactData = artifactSchema.parse(artifactsRaw);
const characterData = characterSchema.parse(charactersRaw);
const weaponData = weaponSchema.parse(weaponsRaw);
const artifactData = artifactSchema.parse(artifactsRaw);
const remoteCharacters: SyncedCharacter[] = await mapConcurrent(
const remoteCharacters: SyncedCharacter[] = await mapConcurrent(
Object.entries(characterData).filter(
([, value]) =>
value.enName &&
@@ -246,7 +207,6 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
const nameCardImage = nameCardImageName(value.CardImg);
const element = detail?.info?.element ?? value.element ?? null;
return {
version,
key,
name: value.enName,
element,
@@ -276,7 +236,7 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
: [],
};
},
);
);
const customCharacters: SyncedCharacter[] = await mapConcurrent(
[...customTravelers],
async (traveler) => {
@@ -286,7 +246,6 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
),
);
return {
version,
key: traveler.key,
name: traveler.name,
element: traveler.element,
@@ -318,12 +277,11 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
value.enName && value.enName !== "???" && value.weaponIcon,
)
.map(([key, value]) => ({
version,
key,
name: value.enName,
rarity: rarity(value.qualityType),
weaponType: value.weaponType ?? null,
imageKey: `catalog/${version}/weapons/${webpName(value.weaponIcon)}`,
imageKey: `catalog/weapons/${webpName(value.weaponIcon)}`,
sourceImage: `${API_ROOT}/assets/weaponicon/${encodeURIComponent(webpName(value.weaponIcon))}`,
}));
let artifacts = Object.entries(artifactData)
@@ -331,11 +289,10 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
([, value]) => value.enName && value.enName !== "???" && value.setIcon,
)
.map(([key, value]) => ({
version,
key,
name: value.enName,
rarity: rarity(value.qualityType),
imageKey: `catalog/${version}/artifacts/${webpName(value.setIcon)}`,
imageKey: `catalog/artifacts/${webpName(value.setIcon)}`,
sourceImage: `${API_ROOT}/assets/artifacts/${encodeURIComponent(webpName(value.setIcon))}`,
}));
const uniqueAssets = new Map<
@@ -398,7 +355,7 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
});
}
}
const missingAssets = await uploadAssets(jobId, [...uniqueAssets.values()]);
const missingAssets = await uploadAssets([...uniqueAssets.values()]);
characters = characters.filter((item) => !missingAssets.has(item.imageKey));
characters = characters.map((item) => ({
...item,
@@ -454,10 +411,9 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
{ type: "application/json", acl: "public-read" },
),
storage.write(
`catalog/${version}/weapons.json`,
"catalog/weapons.json",
JSON.stringify(
weapons.map((item) => ({
version: item.version,
key: item.key,
name: item.name,
rarity: item.rarity,
@@ -468,10 +424,9 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
{ type: "application/json", acl: "public-read" },
),
storage.write(
`catalog/${version}/artifacts.json`,
"catalog/artifacts.json",
JSON.stringify(
artifacts.map((item) => ({
version: item.version,
key: item.key,
name: item.name,
rarity: item.rarity,
@@ -483,32 +438,14 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
]);
await getDb().transaction(async (tx) => {
await tx
.insert(catalogVersions)
.values({ version, sourceTimestamp: versionInfo.timestamp ?? null })
.onConflictDoUpdate({
target: catalogVersions.version,
set: {
sourceTimestamp: versionInfo.timestamp ?? null,
syncedAt: new Date(),
},
});
await tx
.delete(catalogCharacters)
.where(eq(catalogCharacters.version, version));
await tx.delete(catalogCharacterCurrent);
await tx
.delete(catalogWeapons)
.where(eq(catalogWeapons.version, version));
await tx
.delete(catalogArtifacts)
.where(eq(catalogArtifacts.version, version));
await tx.delete(catalogCharacters);
await tx.delete(catalogWeapons);
await tx.delete(catalogArtifacts);
if (characters.length)
await tx
.insert(catalogCharacters)
.values(
characters.map((item) => ({
version: item.version,
key: item.key,
name: item.name,
element: item.element,
@@ -522,28 +459,11 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
talents: item.talents,
})),
);
if (characters.length)
await tx.insert(catalogCharacterCurrent).values(
characters.map((item) => ({
key: item.key,
name: item.name,
element: item.element,
rarity: item.rarity,
weaponType: item.weaponType,
imageKey: item.imageKey,
elementIconKey: item.elementIconKey,
splashImageKey: item.splashImageKey,
nameCardImageKey: item.nameCardImageKey,
constellations: item.constellations,
talents: item.talents,
})),
);
if (weapons.length)
await tx
.insert(catalogWeapons)
.values(
weapons.map((item) => ({
version: item.version,
key: item.key,
name: item.name,
rarity: item.rarity,
@@ -556,44 +476,15 @@ export async function processCatalogSyncJob(jobId: string): Promise<void> {
.insert(catalogArtifacts)
.values(
artifacts.map((item) => ({
version: item.version,
key: item.key,
name: item.name,
rarity: item.rarity,
imageKey: item.imageKey,
})),
);
await tx.update(catalogVersions).set({ active: false });
await tx
.update(catalogVersions)
.set({ active: true, syncedAt: new Date() })
.where(eq(catalogVersions.version, version));
await tx
.update(catalogSyncJobs)
.set({
status: "completed",
completedAt: new Date(),
progress: uniqueAssets.size,
total: uniqueAssets.size,
})
.where(eq(catalogSyncJobs.id, jobId));
});
await storage.write("catalog/current.json", JSON.stringify({ version }), {
type: "application/json",
acl: "public-read",
});
} catch (cause) {
await getDb()
.update(catalogSyncJobs)
.set({
status: "failed",
completedAt: new Date(),
error:
cause instanceof Error
? cause.message.slice(0, 4_000)
: "Unknown sync failure",
})
.where(eq(catalogSyncJobs.id, jobId));
throw cause;
}
await storage.write("catalog/current.json", JSON.stringify({ version }), {
type: "application/json",
acl: "public-read",
});
}