支持大文件

2026-03-15 09:02:02 +08:00
parent 4654f36202
commit 5c278e1925
12 changed files with 410 additions and 177 deletions
--- a/backend/src/dearth/ModCheckService.ts
+++ b/backend/src/dearth/ModCheckService.ts
@@ -550,12 +550,12 @@ export class ModCheckService {
        jarData = fs.readFileSync(file.filename);
      }
      
-      const { Azip } = await import("../utils/ziplib.js");
-      const zipEntries = Azip(jarData);
+      const { yauzl_promise } = await import("../utils/ziplib.js");
+      const zipEntries = await yauzl_promise(jarData);
      
      for (const entry of zipEntries) {
-        if (entry.entryName === iconPath || entry.entryName.endsWith(iconPath)) {
-          const data = await entry.getData();
+        if (entry.fileName === iconPath || entry.fileName.endsWith(iconPath)) {
+          const data = await entry.ReadEntry;
          const ext = iconPath.split('.').pop()?.toLowerCase();
          const mimeType = ext === 'png' ? 'png' : 'jpeg';
          
--- a/backend/src/dearth/utils/FileExtractor.ts
+++ b/backend/src/dearth/utils/FileExtractor.ts
@@ -22,23 +22,18 @@ export class FileExtractor {
      const fullPath = path.join(this.modsPath, jarFilename);

      try {
-        let fileData: Buffer | null = null;
-        try {
-          fileData = fs.readFileSync(fullPath);
-          const mixins = await JarParser.extractMixins(fileData);
-          const infos = await JarParser.extractModInfo(fileData);
+        const hash = await this.calculateFileHash(fullPath);
+        const mixins = await JarParser.extractMixinsFromFile(fullPath);
+        const infos = await JarParser.extractModInfoFromFile(fullPath);

-          files.push({
-            filename: fullPath,
-            hash: crypto.createHash('sha1').update(fileData).digest('hex'),
-            mixins,
-            infos,
-          });
+        files.push({
+          filename: fullPath,
+          hash,
+          mixins,
+          infos,
+        });

-          logger.debug("文件已处理", { 文件名: fullPath, 绝对路径: path.resolve(fullPath), Mixin数量: mixins.length });
-        } finally {
-          fileData = null;
-        }
+        logger.debug("文件已处理", { 文件名: fullPath, 绝对路径: path.resolve(fullPath), Mixin数量: mixins.length });
      } catch (error: any) {
        logger.error("处理文件时出错", { 文件名: fullPath, 错误: error.message });
      }
@@ -48,6 +43,25 @@ export class FileExtractor {
    return files;
  }

+  private async calculateFileHash(filePath: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const hash = crypto.createHash('sha1');
+      const stream = fs.createReadStream(filePath);
+      
+      stream.on('data', (chunk) => {
+        hash.update(chunk);
+      });
+      
+      stream.on('end', () => {
+        resolve(hash.digest('hex'));
+      });
+      
+      stream.on('error', (error) => {
+        reject(error);
+      });
+    });
+  }
+
  private getJarFiles(): string[] {
    if (!fs.existsSync(this.modsPath)) {
      fs.mkdirSync(this.modsPath, { recursive: true });
--- a/backend/src/modloader/forge.ts
+++ b/backend/src/modloader/forge.ts
@@ -2,7 +2,7 @@ import got, { Got } from "got";
 import fs from "node:fs";
 import fse from "fs-extra";
 import { execPromise, fastdownload, version_compare, verifySHA1 } from "../utils/utils.js";
-import { Azip } from "../utils/ziplib.js";
+import { yauzl_promise } from "../utils/ziplib.js";
 import { Config } from "../utils/config.js";
 import { logger } from "../utils/logger.js";

@@ -84,18 +84,20 @@ export class Forge {
  async library() {
    const _downlist: [string, string][] = [];
    const data = await fs.promises.readFile(`${this.path}/forge-${this.minecraft}-${this.loaderVersion}-installer.jar`);
-    const zip = Azip(data);
+    const zip = await yauzl_promise(data);

-    for await (const entry of zip) {
-      if (entry.entryName === "version.json" || entry.entryName === "install_profile.json") {
-        JSON.parse((entry.getData()).toString()).libraries.forEach(async (e: any) => {
+    for (const entry of zip) {
+      if (entry.fileName === "version.json" || entry.fileName === "install_profile.json") {
+        const entryData = await entry.ReadEntry;
+        JSON.parse(entryData.toString()).libraries.forEach(async (e: any) => {
          const t = e.downloads.artifact.path;
          _downlist.push([`https://bmclapi2.bangbang93.com/maven/${t}`, `${this.path}/libraries/${t}`]);
        });
      }

-      if (entry.entryName === "install_profile.json") {
-        const json = JSON.parse((entry.getData()).toString()) as IForge;
+      if (entry.fileName === "install_profile.json") {
+        const entryData = await entry.ReadEntry;
+        const json = JSON.parse(entryData.toString()) as IForge;
        const vjson = await this.got.get(`version/${this.minecraft}/json`).json<IVersion>();
        console.log(`${new URL(vjson.downloads.server_mappings.url).pathname}`);
        const mojpath = this.MTP(json.data.MOJMAPS.server);
--- a/backend/src/modloader/minecraft.ts
+++ b/backend/src/modloader/minecraft.ts
@@ -2,7 +2,7 @@ import fs from "node:fs";
 import { fastdownload, version_compare } from "../utils/utils.js";
 import got from "got";
 import p from "path";
-import { Azip } from "../utils/ziplib.js";
+import { yauzl_promise } from "../utils/ziplib.js";
 import { Config } from "../utils/config.js";

 interface ILInfo {
@@ -54,12 +54,12 @@ export class Minecraft {
    if (version_compare(this.minecraft, "1.18") === 1) {
      const mcpath = `${this.path}/libraries/net/minecraft/server/${this.minecraft}/server-${this.minecraft}.jar`;
      await fastdownload([`https://bmclapi2.bangbang93.com/version/${this.minecraft}/server`, mcpath]);
-      const zip = await Azip(await fs.promises.readFile(mcpath));
-      for await (const entry of zip) {
-        if (entry.entryName.startsWith("META-INF/libraries/") && !entry.entryName.endsWith("/")) {
-          console.log(entry.entryName);
-          const data = entry.getData();
-          const filepath = `${this.path}/libraries/${entry.entryName.replace("META-INF/libraries/", "")}`;
+      const zip = await yauzl_promise(await fs.promises.readFile(mcpath));
+      for (const entry of zip) {
+        if (entry.fileName.startsWith("META-INF/libraries/") && !entry.fileName.endsWith("/")) {
+          console.log(entry.fileName);
+          const data = await entry.ReadEntry;
+          const filepath = `${this.path}/libraries/${entry.fileName.replace("META-INF/libraries/", "")}`;
          const dir = p.dirname(filepath);
          await fs.promises.mkdir(dir, { recursive: true });
          await fs.promises.writeFile(filepath, data);
--- a/backend/src/utils/jar-parser.ts
+++ b/backend/src/utils/jar-parser.ts
@@ -1,20 +1,124 @@
 import { IInfoFile, IMixinFile } from "../dearth/types.js";
-import { Azip } from "./ziplib.js";
+import { yauzl_promise } from "./ziplib.js";
 import toml from "smol-toml";
+import fs from "node:fs";
+import yauzl from "yauzl";

 export class JarParser {
  static async extractModInfo(jarData: Buffer): Promise<IInfoFile[]> {
+    return this.extractModInfoFromBuffer(jarData);
+  }
+
+  static async extractMixins(jarData: Buffer): Promise<IMixinFile[]> {
+    return this.extractMixinsFromBuffer(jarData);
+  }
+
+  static async extractModInfoFromFile(filePath: string): Promise<IInfoFile[]> {
+    return new Promise((resolve, reject) => {
+      const infos: IInfoFile[] = [];
+      
+      yauzl.open(filePath, (err, zipfile) => {
+        if (err) {
+          reject(err);
+          return;
+        }
+        
+        zipfile.on("entry", (entry) => {
+          if (entry.fileName.endsWith("neoforge.mods.toml") || entry.fileName.endsWith("mods.toml") || entry.fileName.endsWith("fabric.mod.json")) {
+            zipfile.openReadStream(entry, (err, stream) => {
+              if (err) {
+                return;
+              }
+              
+              const chunks: Buffer[] = [];
+              stream.on("data", (chunk) => {
+                chunks.push(chunk);
+              });
+              
+              stream.on("end", () => {
+                try {
+                  const data = Buffer.concat(chunks);
+                  if (entry.fileName.endsWith(".toml")) {
+                    infos.push({ name: entry.fileName, data: JSON.stringify(toml.parse(data.toString())) });
+                  } else if (entry.fileName.endsWith(".json")) {
+                    infos.push({ name: entry.fileName, data: data.toString() });
+                  }
+                } catch (error) {
+                  // 忽略解析错误
+                }
+              });
+            });
+          }
+        });
+        
+        zipfile.on("end", () => {
+          resolve(infos);
+        });
+        
+        zipfile.on("error", (err) => {
+          reject(err);
+        });
+      });
+    });
+  }
+
+  static async extractMixinsFromFile(filePath: string): Promise<IMixinFile[]> {
+    return new Promise((resolve, reject) => {
+      const mixins: IMixinFile[] = [];
+      
+      yauzl.open(filePath, (err, zipfile) => {
+        if (err) {
+          reject(err);
+          return;
+        }
+        
+        zipfile.on("entry", (entry) => {
+          if (entry.fileName.endsWith(".mixins.json") && !entry.fileName.includes("/")) {
+            zipfile.openReadStream(entry, (err, stream) => {
+              if (err) {
+                return;
+              }
+              
+              const chunks: Buffer[] = [];
+              stream.on("data", (chunk) => {
+                chunks.push(chunk);
+              });
+              
+              stream.on("end", () => {
+                try {
+                  const data = Buffer.concat(chunks);
+                  mixins.push({ name: entry.fileName, data: data.toString() });
+                } catch (error) {
+                  // 忽略解析错误
+                }
+              });
+            });
+          }
+        });
+        
+        zipfile.on("end", () => {
+          resolve(mixins);
+        });
+        
+        zipfile.on("error", (err) => {
+          reject(err);
+        });
+      });
+    });
+  }
+
+  private static async extractModInfoFromBuffer(jarData: Buffer): Promise<IInfoFile[]> {
    const infos: IInfoFile[] = [];
-    const zipEntries = Azip(jarData);
+    const zipEntries = await yauzl_promise(jarData);
    
    for (const entry of zipEntries) {
      try {
-        if (entry.entryName.endsWith("neoforge.mods.toml") || entry.entryName.endsWith("mods.toml")) {
-          const data = await entry.getData();
-          infos.push({ name: entry.entryName, data: JSON.stringify(toml.parse(data.toString())) });
-        } else if (entry.entryName.endsWith("fabric.mod.json")) {
-          const data = await entry.getData();
-          infos.push({ name: entry.entryName, data: data.toString() });
+        if (entry.fileName.endsWith("neoforge.mods.toml") || entry.fileName.endsWith("mods.toml")) {
+          const data = await entry.ReadEntry;
+          infos.push({ name: entry.fileName, data: JSON.stringify(toml.parse(data.toString())) });
+        } else if (entry.fileName.endsWith("fabric.mod.json")) {
+          const data = await entry.ReadEntry;
+          infos.push({ name: entry.fileName, data: data.toString() });
        }
      } catch (error: any) {
        continue;
@@ -24,15 +128,15 @@ export class JarParser {
    return infos;
  }

-  static async extractMixins(jarData: Buffer): Promise<IMixinFile[]> {
+  private static async extractMixinsFromBuffer(jarData: Buffer): Promise<IMixinFile[]> {
    const mixins: IMixinFile[] = [];
-    const zipEntries = Azip(jarData);
+    const zipEntries = await yauzl_promise(jarData);

    for (const entry of zipEntries) {
-      if (entry.entryName.endsWith(".mixins.json") && !entry.entryName.includes("/")) {
+      if (entry.fileName.endsWith(".mixins.json") && !entry.fileName.includes("/")) {
        try {
-          const data = await entry.getData();
-          mixins.push({ name: entry.entryName, data: data.toString() });
+          const data = await entry.ReadEntry;
+          mixins.push({ name: entry.fileName, data: data.toString() });
        } catch (error: any) {
          continue;
        }
--- a/backend/src/utils/ziplib.ts
+++ b/backend/src/utils/ziplib.ts
@@ -1,4 +1,3 @@
-import admZip from "adm-zip";
 import yauzl from "yauzl";
 import Stream from "node:stream";

@@ -11,7 +10,7 @@ export async function yauzl_promise(buffer: Buffer): Promise<IentryP[]> {
  const zip = await (new Promise((resolve, reject) => {
    yauzl.fromBuffer(
      buffer,
-      /*{lazyEntries:true},*/ (err, zipfile) => {
+      (err, zipfile) => {
        if (err) {
          reject(err);
          return;
@@ -43,6 +42,9 @@ export async function yauzl_promise(buffer: Buffer): Promise<IentryP[]> {
        stream.on("end", () => {
          resolve(Buffer.concat(chunks));
        });
+        stream.on("error", (err) => {
+          reject(err);
+        });
      });
    });
  };
@@ -64,16 +66,15 @@ export async function yauzl_promise(buffer: Buffer): Promise<IentryP[]> {

  return new Promise((resolve, reject) => {
    const entries: IentryP[] = [];
-    zip.on("entry", async (entry: yauzl.Entry) => {
-    const entryP = entry as IentryP;
-    //console.log(entry.fileName);
-    entryP.openReadStream = _openReadStream(zip, entry);
-    entryP.ReadEntry = _ReadEntry(zip, entry);
-    entries.push(entryP);
-      if (zip.entryCount === entries.length) {
-        zip.close();
-        resolve(entries);
-      }
+    zip.on("entry", (entry: yauzl.Entry) => {
+      const entryP = entry as IentryP;
+      entryP.openReadStream = _openReadStream(zip, entry);
+      entryP.ReadEntry = _ReadEntry(zip, entry);
+      entries.push(entryP);
+    });
+    zip.on("end", () => {
+      zip.close();
+      resolve(entries);
    });
    zip.on("error", (err) => {
      reject(err);
@@ -81,8 +82,15 @@ export async function yauzl_promise(buffer: Buffer): Promise<IentryP[]> {
  });
 }

-export function Azip(buffer: Buffer) {
-  const zip = new admZip(buffer);
-  const entries = zip.getEntries();
-  return entries;
+// 从文件路径打开 zip 文件的函数
+export async function yauzl_from_file(filePath: string): Promise<yauzl.ZipFile> {
+  return new Promise((resolve, reject) => {
+    yauzl.open(filePath, { lazyEntries: true }, (err, zipfile) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+      resolve(zipfile);
+    });
+  });
 }