🔧 Refactoring with a new unrar package

models/comicvine.metadata.model.ts

@@ -66,6 +66,8 @@ const ComicVineMetadataSchema = mongoose.Schema({
 	_id: false,
 	aliases: [String],
 	api_detail_url: String,
+	has_staff_review: { type: mongoose.Schema.Types.Mixed },
+
 	cover_date: String,
 	date_added: String,
 	date_last_updated: String,
@@ -84,7 +86,6 @@ const ComicVineMetadataSchema = mongoose.Schema({
 		image_tags: String,
 	},
 
-	has_staff_review: Boolean,
 	id: Number,
 	name: String,
 	resource_type: String,

package-lock.json

@@ -16,6 +16,7 @@
         "@types/mkdirp": "^1.0.0",
         "@types/node": "^13.9.8",
         "@types/string-similarity": "^4.0.0",
+        "@zhangfuxing/unrar": "github:rishighan/unrar",
         "axios": "^0.25.0",
         "axios-retry": "^3.2.4",
         "bree": "^7.1.5",
@@ -45,7 +46,7 @@
         "nats": "^1.3.2",
         "node-7z": "^3.0.0",
         "node-calibre": "^2.1.1",
-        "node-unrar-js": "^1.0.2",
+        "node-unrar-js": "^1.0.5",
         "sharp": "^0.28.3",
         "socket.io": "^4.4.0",
         "threetwo-ui-typings": "^1.0.13",
@@ -2164,6 +2165,11 @@
         "url": "https://opencollective.com/typescript-eslint"
       }
     },
+    "node_modules/@zhangfuxing/unrar": {
+      "version": "0.6.3",
+      "resolved": "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1",
+      "license": "MIT"
+    },
     "node_modules/7zip-bin": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz",
@@ -9453,9 +9459,12 @@
       "dev": true
     },
     "node_modules/node-unrar-js": {
-      "version": "1.0.3",
+      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.5.tgz",
-      "integrity": "sha512-erEhBIhOxKYxxnIhB2CiIU1RQjPN7jjNKbpthWig0VTz3L6AwyORilvTdVzYOkxyTWGY9ZLqHB+EyqmnvvdI5g=="
+      "integrity": "sha512-e2FfnO1rsbXIYPTHkzA/FV8+xDeuEIyRd5X8VQmWJMC5zeZFV9PX1TROTv2bmukX1kBJ2U8Lzu7LZiwUXByaOQ==",
+      "engines": {
+        "node": ">=10.0.0"
+      }
     },
     "node_modules/normalize-path": {
       "version": "3.0.0",
@@ -16657,6 +16666,10 @@
         "eslint-visitor-keys": "^2.0.0"
       }
     },
+    "@zhangfuxing/unrar": {
+      "version": "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1",
+      "from": "@zhangfuxing/unrar@github:rishighan/unrar"
+    },
     "7zip-bin": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz",
@@ -22114,9 +22127,9 @@
       "dev": true
     },
     "node-unrar-js": {
-      "version": "1.0.3",
+      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.5.tgz",
-      "integrity": "sha512-erEhBIhOxKYxxnIhB2CiIU1RQjPN7jjNKbpthWig0VTz3L6AwyORilvTdVzYOkxyTWGY9ZLqHB+EyqmnvvdI5g=="
+      "integrity": "sha512-e2FfnO1rsbXIYPTHkzA/FV8+xDeuEIyRd5X8VQmWJMC5zeZFV9PX1TROTv2bmukX1kBJ2U8Lzu7LZiwUXByaOQ=="
     },
     "normalize-path": {
       "version": "3.0.0",

package.json

@@ -47,6 +47,7 @@
     "@types/mkdirp": "^1.0.0",
     "@types/node": "^13.9.8",
     "@types/string-similarity": "^4.0.0",
+    "@zhangfuxing/unrar": "github:rishighan/unrar",
     "axios": "^0.25.0",
     "axios-retry": "^3.2.4",
     "bree": "^7.1.5",
@@ -76,7 +77,7 @@
     "nats": "^1.3.2",
     "node-7z": "^3.0.0",
     "node-calibre": "^2.1.1",
-    "node-unrar-js": "^1.0.2",
+    "node-unrar-js": "^1.0.5",
     "sharp": "^0.28.3",
     "socket.io": "^4.4.0",
     "threetwo-ui-typings": "^1.0.13",

services/library.service.ts

@@ -53,7 +53,6 @@ import {
 	IExtractedComicBookCoverFile,
 	IExtractionOptions,
 } from "threetwo-ui-typings";
-import { unrarArchive } from "../utils/uncompression.utils";
 const ObjectId = require("mongoose").Types.ObjectId;
 import fsExtra from "fs-extra";
 const through2 = require("through2");
@@ -578,10 +577,6 @@ export default class ImportService extends Service {
 						}>
 					) {
 						console.log(ctx.params);
-						return await unrarArchive(
-							ctx.params.filePath,
-							ctx.params.options
-						);
 					},
 				},
 			},

services/libraryqueue.service.ts

@@ -69,49 +69,51 @@ export default class QueueService extends Service {
 						console.info("New job received!", job.data);
 						console.info(`Processing queue...`);
 						// extract the cover
-						const result = await extractCoverFromFile2(
+						// const result = await extractCoverFromFile2(
-							job.data.fileObject
+						// 	job.data.fileObject
-						);
+						// );
+
+							const {
+							extension,
+							fileNameWithExtension,
+							fileNameWithoutExtension,
+						} = getFileConstituents(job.data.fileObject.filePath);
+						const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`;
+						const foo = await extractFromArchive(job.data.fileObject.filePath, targetDirectory, extension );
 
 						// infer any issue-related metadata from the filename
-						const { inferredIssueDetails } = refineQuery(result.name);
+						// const { inferredIssueDetails } = refineQuery(result.name);
-						console.log("Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2));
+						// console.log("Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2));
-						// const {
+						
-						// 	extension,
+						// // // write to mongo
-						// 	fileNameWithExtension,
+						// console.log("Writing to mongo...")	
-						// 	fileNameWithoutExtension,
+						// const dbImportResult = await this.broker.call(
-						// } = getFileConstituents(job.data.fileObject.filePath);
+						// 	"library.rawImportToDB",
-						// const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`;
+						// 	{
-						// const foo = await extractFromArchive(job.data.fileObject.filePath, targetDirectory, extension );
+						// 		importStatus: {
-						// write to mongo
+						// 			isImported: true,
-						console.log("Writing to mongo...")	
+						// 			tagged: false,
-						const dbImportResult = await this.broker.call(
+						// 			matchedResult: {
-							"library.rawImportToDB",
+						// 				score: "0",
-							{
+						// 			},
-								importStatus: {
+						// 		},
-									isImported: true,
+						// 		rawFileDetails: result,
-									tagged: false,
+						// 		inferredMetadata: {
-									matchedResult: {
+						// 			issue: inferredIssueDetails,
-										score: "0",
+						// 		},
-									},
+						// 		sourcedMetadata: {
-								},
+						// 			comicInfo: {},
-								rawFileDetails: result,
+						// 			comicvine: {},
-								inferredMetadata: {
+						// 		},
-									issue: inferredIssueDetails,
+						// 		// since we already have at least 1 copy
-								},
+						// 		// mark it as not wanted by default
-								sourcedMetadata: {
+						// 		acquisition: {
-									comicInfo: {},
+						// 			wanted: false,
-									comicvine: {},
+						// 		}
-								},
+						// 	}
-								// since we already have at least 1 copy
+						// );
-								// mark it as not wanted by default
-								acquisition: {
-									wanted: false,
-								}
-							}
-						);
 						return Promise.resolve({
-							dbImportResult,
+							// dbImportResult,
 							id: job.id,
 							worker: process.pid,
 						});
@@ -151,6 +153,7 @@ export default class QueueService extends Service {
 							console.error(
 								`An error occured in 'process.import' queue on job id '${job.id}': ${error.message}`
 							);
+							console.error(job.data);
 						}
 					);
 					await this.getQueue("process.import").on(

utils/uncompression.utils.ts

@@ -31,10 +31,10 @@ SOFTWARE.
  *     Initial:        2021/05/04        Rishi Ghan
  */
 
-const fse = require("fs-extra");
 import { promises as fs } from "fs";
+const fse = require("fs-extra");
+const Unrar = require("unrar");
 import path, { parse } from "path";
-
 import {
 	IExtractComicBookCoverErrorResponse,
 	IExtractedComicBookCoverFile,
@@ -49,12 +49,11 @@ import {
 	walkFolder,
 } from "../utils/file.utils";
 import { resizeImage } from "./imagetransformation.utils";
-import { isNil, isUndefined } from "lodash";
+import { each, filter, isNil, isUndefined, remove } from "lodash";
 import { convertXMLToJSON } from "./xml.utils";
 import sevenBin from "7zip-bin";
-import { extract } from "node-7z";
+import { extract, list } from "node-7z";
 const pathTo7zip = sevenBin.path7za;
-const unrar = require("node-unrar-js");
 const { Calibre } = require("node-calibre");
 import { USERDATA_DIRECTORY, COMICS_DIRECTORY } from "../constants/directories";
 
@@ -133,72 +132,52 @@ export const extractCoverFromFile2 = async (
 	}
 };
 
-export const unrarArchive = async (
-	filePath: string,
-	options: IExtractionOptions
-) => {
-	// create directory
-	const directoryOptions = {
-		mode: 0o2775,
-	};
-
-	const fileBuffer = await fse
-		.readFile(filePath)
-		.catch((err) => console.error("Failed to read file", err));
-	try {
-		console.info("Unrar initiating.");
-		await fse.ensureDir(options.targetExtractionFolder, directoryOptions);
-		console.info(`${options.targetExtractionFolder} was created.`);
-
-		const extractor = await unrar.createExtractorFromData({
-			data: fileBuffer,
-		});
-		const files = extractor.extract({});
-		const extractedFiles = [...files.files];
-		for (const file of extractedFiles) {
-			console.info(`Attempting to write ${file.fileHeader.name}`);
-			const fileBuffer = file.extraction;
-			const fileName = explodePath(file.fileHeader.name).fileName;
-			// resize image
-			await resizeImage(
-				fileBuffer,
-				path.resolve(options.targetExtractionFolder + "/" + fileName),
-				200
-			);
-		}
-		// walk the newly created folder and return results
-		return await walkFolder(options.targetExtractionFolder, [
-			".jpg",
-			".png",
-			".jpeg",
-		]);
-	} catch (error) {
-		console.info(`${error}`);
-	}
-};
-
 export const extractComicInfoXMLFromRar = async (
 	filePath: string,
 	fileToExtract: string
 ) => {
 	try {
-		// Read the archive file into a typedArray
+		// Create the target directory
-		const fileBuffer = await fse
+		const directoryOptions = {
-			.readFile(filePath)
+			mode: 0o2775,
-			.catch((err) => console.error("Failed to read file", err));
+		};
-		const extractor = await unrar.createExtractorFromData({
+		const { fileNameWithoutExtension } = getFileConstituents(filePath);
-			data: fileBuffer,
+		const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`;
+		await fse.ensureDir(targetDirectory, directoryOptions);
+		console.info(`%s was created.`, targetDirectory);
+
+		const archive = new Unrar(path.resolve(filePath));
+		// or
+		// var archive = new Unrar({
+		//   path:      protectedArchivePath,
+		//   arguments: ['-pPassword'],
+		//   bin: pathToUnrarBin // Default: unrar
+		// });
+		const loo = [];
+		archive.list(function (err, entries) {
+			remove(entries, ({ type }) => type === "Directory");
+			const comicInfoXML = remove(entries, ({ name }) => name.toLowerCase() === "comicinfo.xml");
+			const foo = entries.sort((a, b) => {
+				if (!isUndefined(a) && !isUndefined(b)) {
+					return a.name
+						.toLowerCase()
+						.localeCompare(b.name.toLowerCase());
+				}
+			});
+			loo.push(foo)
+			console.log(loo[0][0]);
+			var stream = archive.stream(loo[0][0].name); // name of entry
+			stream.on('error', console.error);
+			stream.pipe(require('fs').createWriteStream(`${targetDirectory}/0.jpg`));
+			if(!isUndefined(comicInfoXML[0]) ) {
+				console.log(comicInfoXML);
+				const comicinfoStream = archive.stream(comicInfoXML[0]["name"]);
+			comicinfoStream.on('error', console.error);
+			comicinfoStream.pipe(require('fs').createWriteStream(`${targetDirectory}/dhanda.xml`));
+
+			}
+			
 		});
-		console.info('Unrar initiating.');
-
- 
-
-  const files = extractor.extract({});
-  const extractedFiles = [...files.files];
-  console.log(extractedFiles[0]);
-  for (const file of extractedFiles) {
-    console.info(`Attempting to write ${file.fileHeader.name}`);
-  }
 
 		// const extracted = extractor.extract({
 		// 	files: ({ name }) => name.toLowerCase() === 'comicinfo.xml',
@@ -225,25 +204,88 @@ export const extractComicInfoXMLFromZip = async (
 	filePath: string,
 	outputDirectory: string
 ) => {
-	const foo = extract(path.resolve(filePath), outputDirectory, {
+	try {
-		$cherryPick: ["*.xml"],
+		const listStream = list(path.resolve(filePath), {
-		$bin: pathTo7zip,
+			$cherryPick: ["*.png", "*.jpg", , "*.jpeg", "*.webp", "*.xml"],
-	});
+			$bin: pathTo7zip,
-	for await (const chunk of foo) {
+		});
-		if (chunk.status === "extracted") {
+		const fileList = [];
-			console.log(
+		listStream
-				`comicinfo.xml detected in ${filePath}, attempting extraction...`
+			.on("data", (chunk) => fileList.push(chunk))
-			);
+			.on("end", async () => {
-			const fileContents = await fs.readFile(
+				// Look for ComicInfo.xml
-				path.resolve(`${outputDirectory}/${chunk.file}`),
+				const comicInfoXML = remove(fileList, (item) =>
-				"utf8"
+					!isUndefined(item)
-			);
+						? path.basename(item.file).toLowerCase() ===
-			const parsedJSON = await convertXMLToJSON(
+						  "comicinfo.xml"
-				Buffer.from(fileContents)
+						: undefined
-			);
+				);
-			console.log(parsedJSON);
+				// Sort the file list array naturally
-			return parsedJSON.comicinfo;
+				const sortedFileList = fileList.sort((a, b) =>
-		}
+					a.file.toLowerCase().localeCompare(b.file.toLowerCase())
+				);
+
+				// Create the target directory
+				const directoryOptions = {
+					mode: 0o2775,
+				};
+				const { fileNameWithoutExtension } =
+					getFileConstituents(filePath);
+				const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`;
+				await fse.ensureDir(targetDirectory, directoryOptions);
+				console.info(`%s was created.`, targetDirectory);
+
+				// files to write
+				const filesToWrite = [];
+				if (
+					!isUndefined(sortedFileList[0]) &&
+					!isUndefined(sortedFileList[0].file)
+				) {
+					filesToWrite.push(sortedFileList[0].file);
+				}
+
+				// if ComicInfo.xml present, include it in the file list to be written to disk
+				if (!isUndefined(comicInfoXML[0])) {
+					console.log(`ComicInfo.xml detected in ${filePath}`);
+					filesToWrite.push(comicInfoXML[0].file);
+				}
+
+				// Remove nulls, undefined and empty elements from the file list
+				const filteredFilesToWrite = filesToWrite.filter(
+					(item) => !isUndefined(item)
+				);
+				const extractStream = extract(
+					`${path.resolve(filePath)}`,
+					targetDirectory,
+					{
+						$cherryPick: filteredFilesToWrite,
+						$bin: pathTo7zip,
+					}
+				);
+				extractStream.on("data", (data) => {
+					//do something with the image
+					console.log("ZIP:", data);
+				});
+			});
+
+		// for await (const chunk of foo) {
+		// 	if (chunk.status === "extracted") {
+		// 		console.log(
+		// 			`comicinfo.xml detected in ${filePath}, attempting extraction...`
+		// 		);
+		// 		const fileContents = await fs.readFile(
+		// 			path.resolve(`${outputDirectory}/${chunk.file}`),
+		// 			"utf8"
+		// 		);
+		// 		const parsedJSON = await convertXMLToJSON(
+		// 			Buffer.from(fileContents)
+		// 		);
+		// 		console.log(parsedJSON);
+		// 		return parsedJSON.comicinfo;
+		// 	}
+		// }
+	} catch (error) {
+		throw new Error(error);
 	}
 };
 

yarn.lock

@@ -1102,6 +1102,10 @@
     "@typescript-eslint/types" "4.33.0"
     "eslint-visitor-keys" "^2.0.0"
 
+"@zhangfuxing/unrar@github:rishighan/unrar":
+  "resolved" "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1"
+  "version" "0.6.3"
+
 "7zip-bin@^5.1.1":
   "integrity" "sha512-sAP4LldeWNz0lNzmTird3uWfFDWWTeg6V/MsmyyLR9X1idwKBWIgt/ZvinqQldJm3LecKEs1emkbquO6PCiLVQ=="
   "resolved" "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz"
@@ -5209,10 +5213,10 @@
   "resolved" "https://registry.npmjs.org/node-releases/-/node-releases-2.0.1.tgz"
   "version" "2.0.1"
 
-"node-unrar-js@^1.0.2":
+"node-unrar-js@^1.0.5":
-  "integrity" "sha512-erEhBIhOxKYxxnIhB2CiIU1RQjPN7jjNKbpthWig0VTz3L6AwyORilvTdVzYOkxyTWGY9ZLqHB+EyqmnvvdI5g=="
+  "integrity" "sha512-e2FfnO1rsbXIYPTHkzA/FV8+xDeuEIyRd5X8VQmWJMC5zeZFV9PX1TROTv2bmukX1kBJ2U8Lzu7LZiwUXByaOQ=="
-  "resolved" "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.3.tgz"
+  "resolved" "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.5.tgz"
-  "version" "1.0.3"
+  "version" "1.0.5"
 
 "nopt@*", "nopt@^5.0.0":
   "version" "5.0.0"