Merge pull request #3 from rishighan/mimetype-check

➕ MIMEtype check for comic book archives

.eslintrc.js

@@ -7,8 +7,10 @@ module.exports = {
 	ignorePatterns: [ "test/*"],
 	parser: "@typescript-eslint/parser",
 	parserOptions: {
+		ecmaVersion: "esnext",
 		project: "tsconfig.json",
-		sourceType: "module"
+		sourceType: "module",
+		ecmaVersion: "latest",
 	},
 	plugins: ["prefer-arrow", "import", "@typescript-eslint"],
 	rules: {

mixins/db.mixin.ts

@@ -1,13 +1,14 @@
 const path = require("path");
 const mkdir = require("mkdirp").sync;
 const DbService = require("moleculer-db");
-const MongoAdapter = require("moleculer-db-adapter-mongoose");
+
 
 export const DbMixin = (collection, model) => {
-	if(process.env.MONGO_URI) {
+	if (process.env.MONGO_URI) {
+		const MongooseAdapter = require("moleculer-db-adapter-mongoose");
 		return {
 			mixins: [DbService],
-			adapter: new MongoAdapter(process.env.MONGO_URI, {
+			adapter: new MongooseAdapter(process.env.MONGO_URI, {
 				user: process.env.MONGO_INITDB_ROOT_USERNAME,
 				pass: process.env.MONGO_INITDB_ROOT_PASSWORD,
 				keepAlive: true,
@@ -15,7 +16,6 @@ export const DbMixin = (collection, model) => {
 				family: 4,
 			}),
 			model,
-			collection,
 		};
 	}
 	mkdir(path.resolve("./data"));

models/comic.model.ts

@@ -25,15 +25,13 @@ const RawFileDetailsSchema = mongoose.Schema({
 	filePath: String,
 	fileSize: Number,
 	extension: String,
+	mimeType: String,
 	containedIn: String,
 	pageCount: Number,
 	cover: {
 		filePath: String,
 		stats: Object,
 	},
-	calibreMetadata: {
-		coverWriteResult: String,
-	},
 });
 
 const LOCGSchema = mongoose.Schema({

package.json

@@ -1,7 +1,7 @@
 {
   "name": "threetwo-core-service",
   "version": "0.0.1",
-  "description": "",
+  "description": "Endpoints for common operations in ThreeTwo",
   "scripts": {
     "build": "tsc --build tsconfig.json",
     "dev": "ts-node ./node_modules/moleculer/bin/moleculer-runner.js --hot --repl --config moleculer.config.ts services/**/*.service.ts",
@@ -18,26 +18,30 @@
     "microservices",
     "moleculer"
   ],
-  "author": "",
+  "author": "Rishi Ghan",
   "devDependencies": {
+    "@elastic/elasticsearch": "^8.6.0",
     "@types/lodash": "^4.14.168",
-    "@typescript-eslint/eslint-plugin": "^4.33.0",
+    "@typescript-eslint/eslint-plugin": "^5.56.0",
-    "@typescript-eslint/parser": "^4.33.0",
+    "@typescript-eslint/parser": "^5.56.0",
-    "eslint": "^7.32.0",
+    "eslint": "^8.36.0",
     "eslint-plugin-import": "^2.20.2",
     "eslint-plugin-prefer-arrow": "^1.2.2",
     "install": "^0.13.0",
-    "jest": "^27.5.1",
+    "jest": "^29.5.0",
-    "jest-cli": "^27.5.1",
+    "jest-cli": "^29.5.0",
     "moleculer-repl": "^0.7.0",
+    "node-calibre": "^2.1.1",
     "npm": "^8.4.1",
-    "ts-jest": "^27.1.4",
+    "ts-jest": "^29.0.5",
-    "ts-node": "^8.8.1",
+    "ts-node": "^10.9.1",
-    "typescript": "^4.6.4"
+    "typescript": "^5.0.2"
   },
   "dependencies": {
+    "@npcz/magic": "^1.3.14",
+    "redis": "^4.6.5",
+    "@socket.io/redis-adapter": "^8.1.0",
     "@bluelovers/fast-glob": "https://github.com/rishighan/fast-glob-v2-api.git",
-    "@elastic/elasticsearch": "^8.6.0",
     "@jorgeferrero/stream-to-buffer": "^2.0.6",
     "@root/walk": "^1.1.0",
     "@types/jest": "^27.4.1",
@@ -61,22 +65,20 @@
     "leven": "^3.1.0",
     "lodash": "^4.17.21",
     "mkdirp": "^0.5.5",
-    "moleculer": "^0.14.28",
+    "moleculer": "^0.14.29",
     "moleculer-bull": "github:rishighan/moleculer-bull#1.0.0",
     "moleculer-db": "^0.8.23",
-    "moleculer-db-adapter-mongo": "^0.4.17",
     "moleculer-db-adapter-mongoose": "^0.9.2",
     "moleculer-io": "^2.2.0",
     "moleculer-web": "^0.10.5",
     "mongoosastic-ts": "^6.0.3",
+    "mongoose": "^6.10.4",
     "mongoose-paginate-v2": "^1.3.18",
     "nats": "^1.3.2",
-    "node-calibre": "^2.1.1",
     "opds-extra": "^3.0.9",
     "p7zip-threetwo": "^1.0.4",
     "sanitize-filename-ts": "^1.0.2",
     "sharp": "^0.30.4",
-    "socket.io-redis": "^6.1.1",
     "threetwo-ui-typings": "^1.0.14",
     "through2": "^4.0.2",
     "unrar": "^0.2.0",

services/importqueue.service.ts

@@ -80,6 +80,7 @@ export default class QueueService extends Service {
 							filePath,
 							fileSize,
 							extension,
+							mimeType,
 							cover,
 							containedIn,
 							comicInfoJSON,
@@ -113,6 +114,7 @@ export default class QueueService extends Service {
 								filePath,
 								fileSize,
 								extension,
+								mimeType,
 								containedIn,
 								cover,
 							},

services/library.service.ts

@@ -75,6 +75,7 @@ export default class ImportService extends Service {
 						return await walkFolder(ctx.params.basePathToWalk, [
 							".cbz",
 							".cbr",
+							".cb7",
 						]);
 					},
 				},

services/socket.service.ts

@@ -1,10 +1,16 @@
 "use strict";
 import { Service, ServiceBroker, ServiceSchema } from "moleculer";
+import { createClient } from "redis";
+import { createAdapter } from "@socket.io/redis-adapter";
 const SocketIOService = require("moleculer-io");
-const redisAdapter = require("socket.io-redis");
 const redisURL = new URL(process.env.REDIS_URI);
-console.log(redisURL.hostname);
+// console.log(redisURL.hostname);
 
+const pubClient = createClient({ url: `redis://${redisURL.hostname}:6379` });
+(async () => {
+	await pubClient.connect();
+})();
+const subClient = pubClient.duplicate();
 export default class SocketService extends Service {
 	// @ts-ignore
 	public constructor(
@@ -66,10 +72,7 @@ export default class SocketService extends Service {
 						},
 					},
 					options: {
-						adapter: redisAdapter({
+						adapter: createAdapter(pubClient, subClient),
-							host: redisURL.hostname,
-							port: 6379,
-						}),
 					},
 				},
 			},

utils/file.utils.ts

@@ -3,6 +3,7 @@ const fse = require("fs-extra");
 
 import path from "path";
 import fs from "fs";
+import { FileMagic, MagicFlags } from "@npcz/magic";
 const { readdir, stat } = require("fs/promises");
 import {
 	IExplodedPathResponse,
@@ -17,6 +18,17 @@ import { sanitize } from "sanitize-filename-ts";
 
 const ALLOWED_IMAGE_FILE_FORMATS = [".jpg", ".jpeg", ".png"];
 
+// Tell FileMagic where to find the magic.mgc file
+FileMagic.magicFile = require.resolve("@npcz/magic/dist/magic.mgc");
+
+// We can onlu use MAGIC_PRESERVE_ATIME on operating suystems that support
+// it and that includes OS X for example. It's a good practice as we don't
+// want to change the last access time because we are just checking the file
+// contents type
+if (process.platform === "darwin" || process.platform === "linux") {
+	FileMagic.defaulFlags = MagicFlags.MAGIC_PRESERVE_ATIME;
+}
+
 export const walkFolder = async (
 	folder: string,
 	formats: string[]
@@ -73,11 +85,11 @@ export const explodePath = (filePath: string): IExplodedPathResponse => {
 // returns a promise which resolves true if file exists:
 export const checkFileExists = (filepath) => {
 	return new Promise((resolve, reject) => {
-		fs.access(filepath, fs.constants.F_OK, error => {
+		fs.access(filepath, fs.constants.F_OK, (error) => {
 			resolve(!error);
 		});
 	});
-}
+};
 
 export const getSizeOfDirectory = async (
 	directoryPath: string,
@@ -109,6 +121,12 @@ export const constructPaths = (
 		walkedFolder.extension,
 });
 
+/**
+ * Method that gets file metadata from a filepath.
+ * Extracts the file extension, file name with and without the extension
+ * @param {string} filePath
+ * @returns {Object} object
+ */
 export const getFileConstituents = (filePath: string) => {
 	const extension = path.extname(filePath);
 	const fileNameWithExtension = path.basename(filePath);
@@ -123,14 +141,33 @@ export const getFileConstituents = (filePath: string) => {
 	};
 };
 
+/**
+ * Method that infers MIME type from a filepath 
+ * @param {string} filePath
+ * @returns {Promise} string
+ */
+export const getMimeType = async (filePath: string) => {
+	return await FileMagic.getInstance().then((magic: FileMagic) => {
+		return magic.detect(
+			path.resolve(filePath),
+			magic.flags | MagicFlags.MAGIC_MIME
+		);
+	});
+};
+
 export const createDirectory = async (options: any, directoryPath: string) => {
 	try {
 		await fse.ensureDir(directoryPath, options);
 		console.info(`Directory [ %s ] was created.`, directoryPath);
 	} catch (error) {
-		throw new Errors.MoleculerError("Failed to create directory", 500, "FileOpsError", error);
+		throw new Errors.MoleculerError(
+			"Failed to create directory",
+			500,
+			"FileOpsError",
+			error
+		);
 	}
-}
+};
 
 const filterOutDotFiles = (entities) =>
 	entities.filter((ent) => !ent.name.startsWith("."));

utils/uncompression.utils.ts

@@ -44,6 +44,7 @@ import {
 	getFileConstituents,
 	createDirectory,
 	walkFolder,
+	getMimeType,
 } from "../utils/file.utils";
 import { convertXMLToJSON } from "./xml.utils";
 const fse = require("fs-extra");
@@ -62,7 +63,8 @@ interface RarFile {
 }
 
 const UNRAR_BIN_PATH = process.env.UNRAR_BIN_PATH || "/usr/local/bin/unrar";
-
+// errors array
+const errors = [];
 /**
  * Method that extracts comicInfo.xml file from a .rar archive, if one exists.
  * Also extracts the first image in the listing, which is assumed to be the cover.
@@ -70,7 +72,8 @@ const UNRAR_BIN_PATH = process.env.UNRAR_BIN_PATH || "/usr/local/bin/unrar";
  * @returns {any}
  */
 export const extractComicInfoXMLFromRar = async (
-	filePath: string
+	filePath: string,
+	mimeType: string,
 ): Promise<any> => {
 	try {
 		// Create the target directory
@@ -87,13 +90,13 @@ export const extractComicInfoXMLFromRar = async (
 		const archive = new Unrar({
 			path: path.resolve(filePath),
 			bin: `${UNRAR_BIN_PATH}`, // this will change depending on Docker base OS
-			arguments: ["-v"]
+			arguments: ["-v"],
 		});
 		const filesInArchive: [RarFile] = await new Promise(
 			(resolve, reject) => {
 				return archive.list((err, entries) => {
 					if (err) {
-       					console.log(`DEBUG: ${JSON.stringify(err, null, 2)}` )
+						console.log(`DEBUG: ${JSON.stringify(err, null, 2)}`);
 						reject(err);
 					}
 					resolve(entries);
@@ -151,7 +154,9 @@ export const extractComicInfoXMLFromRar = async (
 							const comicInfoJSON = await convertXMLToJSON(
 								comicinfostring.toString()
 							);
-							console.log(`comicInfo.xml successfully written: ${comicInfoJSON.comicinfo}`)
+							console.log(
+								`comicInfo.xml successfully written: ${comicInfoJSON.comicinfo}`
+							);
 							resolve({ comicInfoJSON: comicInfoJSON.comicinfo });
 						}
 					});
@@ -182,6 +187,7 @@ export const extractComicInfoXMLFromRar = async (
 								extension,
 								containedIn: targetDirectory,
 								fileSize: fse.statSync(filePath).size,
+								mimeType,
 								cover: {
 									filePath: path.relative(
 										process.cwd(),
@@ -202,7 +208,8 @@ export const extractComicInfoXMLFromRar = async (
 };
 
 export const extractComicInfoXMLFromZip = async (
-	filePath: string
+	filePath: string,
+	mimeType: string,
 ): Promise<any> => {
 	try {
 		// Create the target directory
@@ -247,7 +254,7 @@ export const extractComicInfoXMLFromZip = async (
 		// Push the first file (cover) to our extraction target
 		extractionTargets.push(files[0].name);
 		filesToWriteToDisk.coverFile = path.basename(files[0].name);
-			console.log(`sanitized or not, here I am: ${filesToWriteToDisk.coverFile}`);
+	
 		if (!isEmpty(comicInfoXMLFileObject)) {
 			filesToWriteToDisk.comicInfoXML = comicInfoXMLFileObject[0].name;
 			extractionTargets.push(filesToWriteToDisk.comicInfoXML);
@@ -318,6 +325,7 @@ export const extractComicInfoXMLFromZip = async (
 							filePath,
 							name: fileNameWithoutExtension,
 							extension,
+							mimeType,
 							containedIn: targetDirectory,
 							fileSize: fse.statSync(filePath).size,
 							cover: {
@@ -342,18 +350,17 @@ export const extractComicInfoXMLFromZip = async (
 export const extractFromArchive = async (filePath: string) => {
 	console.info(`Unrar is located at: ${UNRAR_BIN_PATH}`);
 	console.info(`p7zip is located at: ${process.env.SEVENZ_BINARY_PATH}`);
-	const { extension } = getFileConstituents(filePath);
+
-	console.log(
+	const mimeType = await getMimeType(filePath);
-		`Detected file type is ${extension}, looking for comicinfo.xml...`
+	console.log(`File has the following mime-type: ${mimeType}`);
-	);
+	switch (mimeType) {
-	switch (extension) {
+		case "application/x-7z-compressed; charset=binary":
-		case ".cbz":
+		case "application/zip; charset=binary":
-		case ".cb7":
+			const cbzResult = await extractComicInfoXMLFromZip(filePath, mimeType);
-			const cbzResult = await extractComicInfoXMLFromZip(filePath);
 			return Object.assign({}, ...cbzResult);
 
-		case ".cbr":
+		case "application/x-rar; charset=binary":
-			const cbrResult = await extractComicInfoXMLFromRar(filePath);
+			const cbrResult = await extractComicInfoXMLFromRar(filePath, mimeType);
 			return Object.assign({}, ...cbrResult);
 
 		default:
@@ -374,13 +381,20 @@ export const uncompressEntireArchive = async (
 	filePath: string,
 	options: any
 ) => {
-	const { extension } = getFileConstituents(filePath);
+	const mimeType = await getMimeType(filePath);
-	switch (extension) {
+	console.log(`File has the following mime-type: ${mimeType}`);
-		case ".cbz":
+	switch (mimeType) {
-		case ".cb7":
+		case "application/x-7z-compressed; charset=binary":
-			return await uncompressZipArchive(filePath, options);
+		case "application/zip; charset=binary":
-		case ".cbr":
+			return await uncompressZipArchive(filePath, {
-			return await uncompressRarArchive(filePath, options);
+				...options,
+				mimeType,
+			});
+		case "application/x-rar; charset=binary":
+			return await uncompressRarArchive(filePath, {
+				...options,
+				mimeType,
+			});
 	}
 };