Merge branch 'master' of https://github.com/rishighan/threetwo-import-service

models/comic.model.ts

@@ -12,23 +12,23 @@ const ComicSchema = mongoose.Schema({
 	userAddedMetadata: {
 		tags: [],
 	},
-	comicInfo: {
-		blackAndWhite: String,
-		characters: [String],
-		count: String,
-		genre: String,
-		manga: String,
-		month: String,
-		number: String,
-		pageCount: String,
-		pages: [],
-		publisher: String,
-		summary: String,
-		title: String,
-		writer: String,
-		year: String,
-	},
 	sourcedMetadata: {
+		comicInfo: {
+			blackAndWhite: String,
+			characters: [String],
+			count: String,
+			genre: String,
+			manga: String,
+			month: String,
+			number: String,
+			pageCount: String,
+			pages: [],
+			publisher: String,
+			summary: String,
+			title: String,
+			writer: String,
+			year: String,
+		},
 		comicvine: {},
 		shortboxed: {},
 		gcd: {},
@@ -37,7 +37,11 @@ const ComicSchema = mongoose.Schema({
 		name: String,
 		path: String,
 		fileSize: Number,
+		extension: String,
 		containedIn: String,
+		calibreMetadata :{
+			coverWriteResult: String,
+		}
 	},
 	acquisition: {
 		wanted: Boolean,
@@ -53,7 +57,7 @@ const ComicSchema = mongoose.Schema({
 			sourceApplication: String,
 		},
 	},
-});
+}, { timestamps: true});
 
 ComicSchema.plugin(paginate);
 const Comic = mongoose.model("Comic", ComicSchema);

package-lock.json

@@ -50,7 +50,7 @@
         "jest": "^25.1.0",
         "jest-cli": "^25.1.0",
         "moleculer-repl": "^0.6.2",
-        "threetwo-ui-typings": "^1.0.1-0",
+        "threetwo-ui-typings": "^1.0.2-0",
         "ts-jest": "^25.3.0",
         "ts-node": "^8.8.1"
       },
@@ -11303,9 +11303,9 @@
       "dev": true
     },
     "node_modules/threetwo-ui-typings": {
-      "version": "1.0.1",
+      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.2.tgz",
-      "integrity": "sha512-g5FWa069AT1WfUHHEVFOQ1q6cfK+9UOzewfKblheuDBSsGN/e89MJMEVfuInBaHgxHM32/P+mNUFBqnBoeRSLQ==",
+      "integrity": "sha512-sK5cb/fApFKseQNgcnGmAnPNxDDXT+dQ/Blei1N4q0mduO3kZfJTnlMYaXjO4FDXLP+jkiwsv0bf7PXm0DgF7g==",
       "dev": true,
       "dependencies": {
         "typescript": "^4.3.2"
@@ -21081,9 +21081,9 @@
       "dev": true
     },
     "threetwo-ui-typings": {
-      "version": "1.0.1",
+      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.2.tgz",
-      "integrity": "sha512-g5FWa069AT1WfUHHEVFOQ1q6cfK+9UOzewfKblheuDBSsGN/e89MJMEVfuInBaHgxHM32/P+mNUFBqnBoeRSLQ==",
+      "integrity": "sha512-sK5cb/fApFKseQNgcnGmAnPNxDDXT+dQ/Blei1N4q0mduO3kZfJTnlMYaXjO4FDXLP+jkiwsv0bf7PXm0DgF7g==",
       "dev": true,
       "requires": {
         "typescript": "^4.3.2"

package.json

@@ -27,7 +27,7 @@
     "jest": "^25.1.0",
     "jest-cli": "^25.1.0",
     "moleculer-repl": "^0.6.2",
-    "threetwo-ui-typings": "^1.0.1-0",
+    "threetwo-ui-typings": "^1.0.2-0",
     "ts-jest": "^25.3.0",
     "ts-node": "^8.8.1"
   },

services/api.service.ts

@@ -14,7 +14,6 @@ export default class ApiService extends Service {
 			// More info about settings: https://moleculer.services/docs/0.14/moleculer-web.html
 			settings: {
 				port: process.env.PORT || 3000,
-
 				routes: [
 					{
 						path: "/api",
@@ -39,7 +38,6 @@ export default class ApiService extends Service {
 						use: [],
 						mergeParams: true,
 						autoAliases: true,
-
 						aliases: {},
 
 						// Calling options. More info: https://moleculer.services/docs/0.14/moleculer-web.html#Calling-options
@@ -55,11 +53,7 @@ export default class ApiService extends Service {
 								limit: "1MB",
 							},
 						},
-
-						// Mapping policy setting. More info: https://moleculer.services/docs/0.14/moleculer-web.html#Mapping-policy
 						mappingPolicy: "all", // Available values: "all", "restrict"
-
-						// Enable/disable logging
 						logging: true,
 					},
 					{
@@ -71,15 +65,11 @@ export default class ApiService extends Service {
 						use: [ApiGateway.serveStatic("comics")],
 					},
 				],
-				// Do not log client side errors (does not log an error response when the error.code is 400<=X<500)
 				log4XXResponses: false,
-				// Logging the request parameters. Set to any log level to enable it. E.g. "info"
 				logRequestParams: null,
-				// Logging the response data. Set to any log level to enable it. E.g. "info"
 				logResponseData: null,
 				assets: {
 					folder: "public",
-					// Options to `server-static` module
 					options: {},
 				},
 			},
@@ -104,7 +94,7 @@ export default class ApiService extends Service {
 					this.logger.info("Client connected via websocket!");
 
 					client.on(
-						"importComicsInDB",
+						"importComicsToDB",
 						async ({ action, params, opts }, done) => {
 							this.logger.info(
 								"Received request from client! Action:",

services/imagetransformation.service.ts

@@ -8,12 +8,7 @@ import {
 } from "moleculer";
 import {
 	resizeImage,
-	calculateLevenshteinDistance,
 } from "../utils/imagetransformation.utils";
-import https from "https";
-import fs from "fs";
-import path from "path";
-
 export default class ProductsService extends Service {
 	// @ts-ignore
 	public constructor(
@@ -58,44 +53,6 @@ export default class ProductsService extends Service {
 								return { resizeOperationStatus: resizeResult };
 							},
 						},
-						calculateLevenshteinDistance: {
-							rest: "POST /calculateLevenshteinDistance",
-							params: {},
-							async handler(
-								ctx: Context<{
-									imagePath: string;
-									imagePath2: string;
-									options: {
-										match_id: string,
-									};
-								}>
-							) {
-								const fileName = ctx.params.options.match_id + "_" + path.basename(
-									ctx.params.imagePath
-								);
-								return new Promise((resolve, reject) => {
-									https.get(ctx.params.imagePath2, (response) => {
-										const fileStream = response.pipe(
-											fs.createWriteStream(
-												`./userdata/temporary/${fileName}`
-											)
-										);
-										fileStream.on("finish", async () => {
-											const levenshteinDistance = await calculateLevenshteinDistance(
-												ctx.params.imagePath,
-												path.resolve(
-													`./userdata/temporary/${fileName}`
-												)
-											);
-											resolve(levenshteinDistance);
-										});
-
-									}).end();
-								});
-
-
-							},
-						},
 					},
 					methods: {},
 				},

services/import.service.ts

@@ -1,4 +1,5 @@
 "use strict";
+import { isNil } from "lodash";
 import {
 	Context,
 	Service,
@@ -10,6 +11,9 @@ import { DbMixin } from "../mixins/db.mixin";
 import Comic from "../models/comic.model";
 import { walkFolder } from "../utils/file.utils";
 import { convertXMLToJSON } from "../utils/xml.utils";
+import https from "https";
+const ObjectId = require("mongoose").Types.ObjectId;
+
 
 export default class ProductsService extends Service {
 	public constructor(
@@ -74,8 +78,55 @@ export default class ProductsService extends Service {
 								});
 							},
 						},
-						getRecentlyImportedComicBooks: {
+						applyComicVineMetadata: {
-							rest: "POST /getRecentlyImportedComicBooks",
+							rest: "POST /applyComicVineMetadata",
+							params: {},
+							async handler(ctx: Context<{ match: { volume: { api_detail_url: string}, volumeInformation: object}, comicObjectId: string }>) {
+								// 1. find mongo object by id
+								// 2. import payload into sourcedMetadata.comicvine
+								const comicObjectId = new ObjectId(ctx.params.comicObjectId);
+								const matchedResult = ctx.params.match;
+								console.log(matchedResult.volume.api_detail_url);
+								let volumeDetailsPromise;
+								if (!isNil(matchedResult.volume)) {
+									volumeDetailsPromise = new Promise((resolve, reject) => {
+										return https.get(`${matchedResult.volume.api_detail_url}?api_key=a5fa0663683df8145a85d694b5da4b87e1c92c69&format=json&limit=1&offset=0&field_list=id,name,deck,image,first_issue,last_issue,publisher,count_of_issues`, (resp) => {
+
+											let data = '';
+											resp.on('data', (chunk) => {
+												data += chunk;
+											});
+
+											resp.on('end', () => {
+												const volumeInformation = JSON.parse(data);
+												resolve(volumeInformation);
+											});
+
+										}).on("error", (err) => {
+											console.log("Error: " + err.message);
+											reject(err);
+										});
+									});
+								}
+								return new Promise(async (resolve, reject) => {
+									const volumeDetails = await volumeDetailsPromise;
+									matchedResult.volumeInformation = volumeDetails.results; 
+									Comic.findByIdAndUpdate(comicObjectId, { sourcedMetadata: { comicvine: matchedResult } }, { new: true }, (err, result) => {
+										if (err) {
+											console.log(err);
+											reject(err);
+										} else {
+											// 3. Fetch and append volume information
+											resolve(result);
+										}
+									});
+								});
+							},
+
+						},
+
+						getComicBooks: {
+							rest: "POST /getComicBooks",
 							params: {},
 							async handler(
 								ctx: Context<{ paginationOptions: object }>

utils/imagetransformation.utils.ts

@@ -1,9 +1,6 @@
 const sharp = require("sharp");
-const imghash = require("imghash");
-const leven = require("leven");
-import path from "path";
-import { isNull, reject } from "lodash";
 import { logger } from "./logger.utils";
+import { ISharpResizedImageStats } from "threetwo-ui-typings";
 
 export const extractMetadataFromImage = async (
 	imageFilePath: string
@@ -21,35 +18,20 @@ export const resizeImage = async (
 	outputPath: string,
 	newWidth: number,
 	newHeight?: number
-): Promise<unknown> => {
+): Promise<ISharpResizedImageStats> => {
-	return await sharp(imageFile)
+	return new Promise((resolve, reject) => {
-		.resize(newWidth)
+		sharp(imageFile)
-		.toFile(`${outputPath}`, (err, info) => {
+			.resize(newWidth)
-			if (err) {
+			.toFile(`${outputPath}`, (err, info) => {
-				logger.error("Failed to resize image:");
+				if (err) {
-				logger.error(err);
+					logger.error("Failed to resize image:");
-				return err;
+					logger.error(err);
-			}
+					reject(err);
+				}
 
-			logger.info("Image file resized with the following parameters:");
+				logger.info("Image file resized with the following parameters:");
-			logger.info(info);
+				logger.info(info);
-			return info;
+				resolve(info);
-		});
+			});
-};
+	});
-
-export const calculateLevenshteinDistance = async (
-	imagePath1: string,
-	imagePath2: string
-): Promise<Record<string, unknown>> => {
-	console.log("AGANTUK", imagePath1)
-	const hash1 = await imghash.hash(imagePath1);
-	const hash2 = await imghash.hash(imagePath2);
-	console.log("HASHISH", hash1)
-	if (!isNull(hash1) && !isNull(hash2)) {
-		return new Promise((resolve, reject) => {
-			resolve({ levenshteinDistance: leven(hash1, hash2) });
-		});
-	} else {
-		reject("Can't calculate the Levenshtein distance")
-	}
 };

utils/uncompression.utils.ts

@@ -31,7 +31,7 @@ SOFTWARE.
  *     Initial:        2021/05/04        Rishi Ghan
  */
 
-import { createReadStream, createWriteStream, readFileSync } from "fs";
+import { createReadStream, createWriteStream, readFileSync, stat } from "fs";
 const fse = require("fs-extra");
 import path from "path";
 import { each, isEmpty, map, flatten } from "lodash";
@@ -42,6 +42,7 @@ import {
 	IExtractedComicBookCoverFile,
 	IExtractionOptions,
 	IFolderData,
+	ISharpResizedImageStats,
 } from "threetwo-ui-typings";
 import { logger } from "./logger.utils";
 import { validateComicBookMetadata } from "../utils/validation.utils";
@@ -84,6 +85,7 @@ export const extractCoverFromFile = async (
 			} catch (error) {
 				logger.error(`${error}: Couldn't create directory.`);
 			}
+			// extract the cover 
 			let result: string;
 			const targetCoverImageFilePath = path.resolve(constructedPaths.targetPath + "/" + walkedFolder.name + "_cover.jpg")
 			result = await calibre.run(
@@ -95,15 +97,17 @@ export const extractCoverFromFile = async (
 			);
 			// create renditions
 			const renditionPath = constructedPaths.targetPath + "/" + walkedFolder.name + "_200px.jpg";
-			await resizeImage(targetCoverImageFilePath, path.resolve(renditionPath), 200);
+			const stats:ISharpResizedImageStats = await resizeImage(targetCoverImageFilePath, path.resolve(renditionPath), 200);
-				
+
 			resolve({
 				name: walkedFolder.name,
-				path: constructedPaths.targetPath + "/" + walkedFolder.name + "_200px.jpg", //renditionPath 
+				path: renditionPath, 
-				fileSize: 0,
+				fileSize: stats.size,
+				extension: path.extname(constructedPaths.inputFilePath),
 				containedIn: walkedFolder.containedIn,
-				//originalPath:
+				calibreMetadata: {
-				//extension:
+					coverWriteResult: result,
+				}
 			});
 		} catch (error) {
 			console.log(error);
@@ -200,6 +204,7 @@ export const unrar = async (
 					mode: 0o2775,
 				};
 				try {
+					// read the file into a buffer
 					const fileBuffer = await readFile(
 						paths.inputFilePath
 					).catch((err) => console.error("Failed to read file", err));
@@ -238,8 +243,12 @@ export const unrar = async (
 					resolve({
 						name: `${extractedFiles[0].fileHeader.name}`,
 						path: paths.targetPath,
+						extension: path.extname(extractedFiles[0].fileHeader.name),
 						fileSize: extractedFiles[0].fileHeader.packSize,
 						containedIn: walkedFolder.containedIn,
+						calibreMetadata: {
+							coverWriteResult: "",
+						}
 					});
 				} catch (error) {
 					logger.error(`${error}: Couldn't write file.`);

yarn.lock

@@ -5906,10 +5906,10 @@
   "resolved" "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz"
   "version" "0.2.0"
 
-"threetwo-ui-typings@^1.0.1-0":
+"threetwo-ui-typings@^1.0.2-0":
-  "integrity" "sha512-g5FWa069AT1WfUHHEVFOQ1q6cfK+9UOzewfKblheuDBSsGN/e89MJMEVfuInBaHgxHM32/P+mNUFBqnBoeRSLQ=="
+  "integrity" "sha512-sK5cb/fApFKseQNgcnGmAnPNxDDXT+dQ/Blei1N4q0mduO3kZfJTnlMYaXjO4FDXLP+jkiwsv0bf7PXm0DgF7g=="
-  "resolved" "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.1.tgz"
+  "resolved" "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.2.tgz"
-  "version" "1.0.1"
+  "version" "1.0.2"
   dependencies:
     "typescript" "^4.3.2"