feat(assets): AssetManager parallel fetch

- max 40/s, 20 concurrent

src/games/genshin/data/assets.ts

@@ -23,6 +23,7 @@ import { throwNotOk } from "@/lib/util";
 globalThis.devCache ??= new TTLCache();
 export const cache = globalThis.devCache;
 const log = createLogger("ASSET");
+const storeLog = createLogger("STORE");
 
 export class AssetManager {
   private cache = new Map<string, ArrayBuffer>();
@@ -58,27 +59,49 @@ export class AssetManager {
   async getMissingAssets() {
     await mkdir("store").catch(() => {});
     const existing = new Set(await readdir("store"));
+    log`Initializing AssetManager`;
     const missing = new Set<string>();
-    (await this.getAssetList()).map(
+    const list = await this.getAssetList();
-      (n) => existing.has(hash(n).toString(36)) || missing.add(n),
+    list.map((n) => existing.has(hash(n).toString(36)) || missing.add(n));
-    );
+    const listDeduplicated = new Set(list);
+    const queue = [...missing.values()]
+      .map((path) => ({ path, tries: 0 }))
+      .sort(() => Math.random() - 0.5);
+    var running = 0;
+    log.warn`Missing:  ${missing.size}\nIn store: ${existing.size}\nTotal:    ${list.length}/${listDeduplicated.size}`;
 
-    log`${missing.size} assets missing, ${existing.size} assets in store`;
+    const manager = setInterval(() => {
+      if (queue.length === 0 && running === 0) {
+        log`All missing assets fetched and stored`;
+        return clearInterval(manager);
+      }
 
-    for (const name of missing)
+      if (running >= 20) return;
-      await fetch(new URL(name, "https://enka.network"))
+      const q = queue.pop();
+      if (!q) return;
+
+      running++;
+      fetch(new URL(q.path, "https://enka.network"))
         .then((res) => {
           if (res.status === 404) {
-            console.warn(`Asset not found: ${name}`);
+            log.warn`Asset not found: ${q.path}`;
             return new ArrayBuffer(0);
           }
           if (!res.ok)
-            throw new Error(`Failed to fetch ${name}: ${res.statusText}`);
+            throw new Error(`Failed to fetch ${q.path}: ${res.statusText}`);
           return res.arrayBuffer();
         })
-        .then((d) => write(join("store", hash(name).toString(36)), d));
+        .then((d) => write(join("store", hash(q.path).toString(36)), d))
+        .then(() => log`${q.path} stored. ${queue.length + running} left.`)
+        .catch((e) => {
+          log.error`${e}`;
+          q.tries++;
+          queue.push(q);
+        })
+        .finally(() => running--);
+    }, 1000 / 40);
 
-    log`All missing assets fetched and stored`;
+    // for (const name of missing);
   }
 
   async purgeExtraAssets() {
@@ -203,7 +226,7 @@ export async function getEnkaStore<K extends keyof StoreTypeMap>(
   const cached = cache.get(`store.${key}`);
   if (cached) return cached as StoreTypeMap[K];
 
-  log`Fresh: store.${key}`;
+  storeLog`Fresh: store.${key}`;
   const response = await fetch(
     `https://api.enka.network/store/gi/${key}.json`,
   ).then(throwNotOk);