diff --git a/package-lock.json b/package-lock.json index 4739bad..e01d113 100644 --- a/package-lock.json +++ b/package-lock.json @@ -710,11 +710,11 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.22.13", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", - "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", + "version": "7.23.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz", + "integrity": "sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==", "dependencies": { - "@babel/highlight": "^7.22.13", + "@babel/highlight": "^7.23.4", "chalk": "^2.4.2" }, "engines": { @@ -840,12 +840,12 @@ } }, "node_modules/@babel/generator": { - "version": "7.22.10", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.22.10.tgz", - "integrity": "sha512-79KIf7YiWjjdZ81JnLujDRApWtl7BxTqWD88+FFdQEIOG8LJ0etDOM7CXuIgGJa55sGOwZVwuEsaLEm0PJ5/+A==", + "version": "7.23.6", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.6.tgz", + "integrity": "sha512-qrSfCYxYQB5owCmGLbl8XRpX1ytXlpueOb0N0UmQwA073KZxejgQTzAmJezxvpwQD9uGtK2shHdi55QT+MbjIw==", "dev": true, "dependencies": { - "@babel/types": "^7.22.10", + "@babel/types": "^7.23.6", "@jridgewell/gen-mapping": "^0.3.2", "@jridgewell/trace-mapping": "^0.3.17", "jsesc": "^2.5.1" @@ -892,22 +892,22 @@ } }, "node_modules/@babel/helper-environment-visitor": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz", - "integrity": "sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q==", + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", + "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-function-name": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz", - "integrity": "sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", + "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", "dev": true, "dependencies": { - "@babel/template": "^7.22.5", - "@babel/types": "^7.22.5" + "@babel/template": "^7.22.15", + "@babel/types": "^7.23.0" }, "engines": { "node": ">=6.9.0" @@ -990,18 +990,18 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", - "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", + "version": "7.23.4", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz", + "integrity": "sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz", - "integrity": "sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==", + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", + "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", "engines": { "node": ">=6.9.0" } @@ -1030,11 +1030,11 @@ } }, "node_modules/@babel/highlight": { - "version": "7.22.13", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.13.tgz", - "integrity": "sha512-C/BaXcnnvBCmHTpz/VGZ8jgtE2aYlW4hxDhseJAWZb7gqGM/qtCK6iZUb0TyKFf7BOUsBH7Q7fkRsDRhg1XklQ==", + "version": "7.23.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz", + "integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==", "dependencies": { - "@babel/helper-validator-identifier": "^7.22.5", + "@babel/helper-validator-identifier": "^7.22.20", "chalk": "^2.4.2", "js-tokens": "^4.0.0" }, @@ -1107,9 +1107,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.22.13", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.13.tgz", - "integrity": "sha512-3l6+4YOvc9wx7VlCSw4yQfcBo01ECA8TicQfbnCPuCEpRQrf+gTUyGdxNw+pyTUyywp6JRD1w0YQs9TpBXYlkw==", + "version": "7.23.6", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.6.tgz", + "integrity": "sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ==", "dev": true, "bin": { "parser": "bin/babel-parser.js" @@ -1319,34 +1319,34 @@ } }, "node_modules/@babel/template": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.5.tgz", - "integrity": "sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw==", + "version": "7.22.15", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", + "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", "dev": true, "dependencies": { - "@babel/code-frame": "^7.22.5", - "@babel/parser": "^7.22.5", - "@babel/types": "^7.22.5" + "@babel/code-frame": "^7.22.13", + "@babel/parser": "^7.22.15", + "@babel/types": "^7.22.15" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.22.11", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.22.11.tgz", - "integrity": "sha512-mzAenteTfomcB7mfPtyi+4oe5BZ6MXxWcn4CX+h4IRJ+OOGXBrWU6jDQavkQI9Vuc5P+donFabBfFCcmWka9lQ==", + "version": "7.23.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.7.tgz", + "integrity": "sha512-tY3mM8rH9jM0YHFGyfC0/xf+SB5eKUu7HPj7/k3fpi9dAlsMc5YbQvDi0Sh2QTPXqMhyaAtzAr807TIyfQrmyg==", "dev": true, "dependencies": { - "@babel/code-frame": "^7.22.10", - "@babel/generator": "^7.22.10", - "@babel/helper-environment-visitor": "^7.22.5", - "@babel/helper-function-name": "^7.22.5", + "@babel/code-frame": "^7.23.5", + "@babel/generator": "^7.23.6", + "@babel/helper-environment-visitor": "^7.22.20", + "@babel/helper-function-name": "^7.23.0", "@babel/helper-hoist-variables": "^7.22.5", "@babel/helper-split-export-declaration": "^7.22.6", - "@babel/parser": "^7.22.11", - "@babel/types": "^7.22.11", - "debug": "^4.1.0", + "@babel/parser": "^7.23.6", + "@babel/types": "^7.23.6", + "debug": "^4.3.1", "globals": "^11.1.0" }, "engines": { @@ -1363,13 +1363,13 @@ } }, "node_modules/@babel/types": { - "version": "7.22.11", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.22.11.tgz", - "integrity": "sha512-siazHiGuZRz9aB9NpHy9GOs9xiQPKnMzgdr493iI1M67vRXpnEq8ZOOKzezC5q7zwuQ6sDhdSp4SD9ixKSqKZg==", + "version": "7.23.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.6.tgz", + "integrity": "sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg==", "dev": true, "dependencies": { - "@babel/helper-string-parser": "^7.22.5", - "@babel/helper-validator-identifier": "^7.22.5", + "@babel/helper-string-parser": "^7.23.4", + "@babel/helper-validator-identifier": "^7.22.20", "to-fast-properties": "^2.0.0" }, "engines": { @@ -4272,17 +4272,6 @@ "node": ">=10" } }, - "node_modules/busboy": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", - "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", - "dependencies": { - "streamsearch": "^1.1.0" - }, - "engines": { - "node": ">=10.16.0" - } - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -9934,9 +9923,9 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/msgpackr": { - "version": "1.9.7", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.7.tgz", - "integrity": "sha512-baUNaLvKQvVhzfWTNO07njwbZK1Lxjtb0P1JL6/EhXdLTHzR57/mZqqJC39TtQKvOmkJA4pcejS4dbk7BDgLLA==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.1.tgz", + "integrity": "sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==", "optionalDependencies": { "msgpackr-extract": "^3.0.2" } @@ -14794,14 +14783,6 @@ "node": ">=0.10.0" } }, - "node_modules/streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -15668,16 +15649,24 @@ } }, "node_modules/undici": { - "version": "5.23.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.23.0.tgz", - "integrity": "sha512-1D7w+fvRsqlQ9GscLBwcAJinqcZGHUKjbOmXdlE/v8BvEGXjeWAax+341q44EuTcHXXnfyKNbKRq4Lg7OzhMmg==", + "version": "5.28.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz", + "integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==", "dependencies": { - "busboy": "^1.6.0" + "@fastify/busboy": "^2.0.0" }, "engines": { "node": ">=14.0" } }, + "node_modules/undici/node_modules/@fastify/busboy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", + "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==", + "engines": { + "node": ">=14" + } + }, "node_modules/unit-compare": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/unit-compare/-/unit-compare-1.0.1.tgz", diff --git a/services/jobqueue.service.ts b/services/jobqueue.service.ts index 591665d..f1105e4 100644 --- a/services/jobqueue.service.ts +++ b/services/jobqueue.service.ts @@ -2,7 +2,7 @@ import { Context, Service, ServiceBroker } from "moleculer"; import JobResult from "../models/jobresult.model"; import { refineQuery } from "filename-parser"; import BullMqMixin from "moleculer-bullmq"; -import { extractFromArchive } from "../utils/uncompression.utils"; +import { extractFromArchive, uncompressEntireArchive } from "../utils/uncompression.utils"; import { isNil, isUndefined } from "lodash"; import { pubClient } from "../config/redis.config"; @@ -47,17 +47,15 @@ export default class JobQueueService extends Service { enqueue: { queue: true, rest: "/GET enqueue", - handler: async (ctx: Context<{}>) => { + handler: async (ctx: Context<{ queueName: string; description: string }>) => { + console.log(ctx.params); + const { queueName, description } = ctx.params; // Enqueue the job - const job = await this.localQueue( - ctx, - "enqueue.async", - ctx.params, - { - priority: 10, - } - ); + const job = await this.localQueue(ctx, queueName, ctx.params, { + priority: 10, + }); console.log(`Job ${job.id} enqueued`); + console.log(`${description}`); return job.id; }, @@ -70,17 +68,13 @@ export default class JobQueueService extends Service { }> ) => { try { - console.log( - `Recieved Job ID ${ctx.locals.job.id}, processing...` - ); + console.log(`Recieved Job ID ${ctx.locals.job.id}, processing...`); console.log(ctx.params); // 1. De-structure the job params const { fileObject } = ctx.locals.job.data.params; // 2. Extract metadata from the archive - const result = await extractFromArchive( - fileObject.filePath - ); + const result = await extractFromArchive(fileObject.filePath); const { name, filePath, @@ -93,9 +87,7 @@ export default class JobQueueService extends Service { } = result; // 3a. Infer any issue-related metadata from the filename - const { inferredIssueDetails } = refineQuery( - result.name - ); + const { inferredIssueDetails } = refineQuery(result.name); console.log( "Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2) @@ -135,8 +127,7 @@ export default class JobQueueService extends Service { // "acquisition.directconnect.downloads": [], // mark the metadata source - "acquisition.source.name": - ctx.locals.job.data.params.sourcedFrom, + "acquisition.source.name": ctx.locals.job.data.params.sourcedFrom, }; // 3c. Add the bundleId, if present to the payload @@ -147,13 +138,8 @@ export default class JobQueueService extends Service { // 3d. Add the sourcedMetadata, if present if ( - !isNil( - ctx.locals.job.data.params.sourcedMetadata - ) && - !isUndefined( - ctx.locals.job.data.params.sourcedMetadata - .comicvine - ) + !isNil(ctx.locals.job.data.params.sourcedMetadata) && + !isUndefined(ctx.locals.job.data.params.sourcedMetadata.comicvine) ) { Object.assign( payload.sourcedMetadata, @@ -162,15 +148,11 @@ export default class JobQueueService extends Service { } // 4. write to mongo - const importResult = await this.broker.call( - "library.rawImportToDB", - { - importType: - ctx.locals.job.data.params.importType, - bundleId, - payload, - } - ); + const importResult = await this.broker.call("library.rawImportToDB", { + importType: ctx.locals.job.data.params.importType, + bundleId, + payload, + }); return { data: { importResult, @@ -182,14 +164,9 @@ export default class JobQueueService extends Service { console.error( `An error occurred processing Job ID ${ctx.locals.job.id}` ); - throw new MoleculerError( - error, - 500, - "IMPORT_JOB_ERROR", - { - data: ctx.params.sessionId, - } - ); + throw new MoleculerError(error, 500, "IMPORT_JOB_ERROR", { + data: ctx.params.sessionId, + }); } }, }, @@ -217,8 +194,7 @@ export default class JobQueueService extends Service { statuses: { $push: { status: "$_id.status", - earliestTimestamp: - "$earliestTimestamp", + earliestTimestamp: "$earliestTimestamp", count: "$count", }, }, @@ -238,10 +214,7 @@ export default class JobQueueService extends Service { { $cond: [ { - $eq: [ - "$$this.status", - "completed", - ], + $eq: ["$$this.status", "completed"], }, "$$this.count", 0, @@ -261,10 +234,7 @@ export default class JobQueueService extends Service { { $cond: [ { - $eq: [ - "$$this.status", - "failed", - ], + $eq: ["$$this.status", "failed"], }, "$$this.count", 0, @@ -282,9 +252,24 @@ export default class JobQueueService extends Service { ]); }, }, + "uncompressFullArchive.async": { + rest: "POST /uncompressFullArchive", + handler: async (ctx: Context<{ filePath: string; options: any }>) => { + const { filePath, options } = ctx.params; + console.log("asd", filePath); + // 2. Extract metadata from the archive + return await uncompressEntireArchive(filePath, options); + }, + }, }, events: { + async "uncompressFullArchive.async.active"(ctx: Context<{ id: number }>) { + console.log(`Uncompression Job ID ${ctx.params.id} is set to active.`); + }, + async "uncompressFullArchive.async.completed"(ctx: Context<{ id: number }>) { + console.log(`Uncompression Job ID ${ctx.params.id} completed.`); + }, // use the `${QUEUE_NAME}.QUEUE_EVENT` scheme async "enqueue.async.active"(ctx: Context<{ id: Number }>) { console.log(`Job ID ${ctx.params.id} is set to active.`); @@ -307,9 +292,7 @@ export default class JobQueueService extends Service { // 2. Increment the completed job counter await pubClient.incr("completedJobCount"); // 3. Fetch the completed job count for the final payload to be sent to the client - const completedJobCount = await pubClient.get( - "completedJobCount" - ); + const completedJobCount = await pubClient.get("completedJobCount"); // 4. Emit the LS_COVER_EXTRACTED event with the necessary details await this.broker.call("socket.broadcast", { namespace: "/", @@ -336,9 +319,7 @@ export default class JobQueueService extends Service { async "enqueue.async.failed"(ctx) { const job = await this.job(ctx.params.id); await pubClient.incr("failedJobCount"); - const failedJobCount = await pubClient.get( - "failedJobCount" - ); + const failedJobCount = await pubClient.get("failedJobCount"); await JobResult.create({ id: ctx.params.id, diff --git a/services/library.service.ts b/services/library.service.ts index bc372eb..3bc5284 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -33,13 +33,7 @@ SOFTWARE. "use strict"; import { isNil } from "lodash"; -import { - Context, - Service, - ServiceBroker, - ServiceSchema, - Errors, -} from "moleculer"; +import { Context, Service, ServiceBroker, ServiceSchema, Errors } from "moleculer"; import { DbMixin } from "../mixins/db.mixin"; import Comic from "../models/comic.model"; import { walkFolder, getSizeOfDirectory } from "../utils/file.utils"; @@ -101,9 +95,7 @@ export default class ImportService extends Service { uncompressFullArchive: { rest: "POST /uncompressFullArchive", params: {}, - handler: async ( - ctx: Context<{ filePath: string; options: any }> - ) => { + handler: async (ctx: Context<{ filePath: string; options: any }>) => { await broker.call("importqueue.uncompressResize", { filePath: ctx.params.filePath, options: ctx.params.options, @@ -121,8 +113,7 @@ export default class ImportService extends Service { }); // Determine source where the comic was added from // and gather identifying information about it - const sourceName = - referenceComicObject[0].acquisition.source.name; + const sourceName = referenceComicObject[0].acquisition.source.name; const { sourcedMetadata } = referenceComicObject[0]; const filePath = `${COMICS_DIRECTORY}/${ctx.params.bundle.data.name}`; @@ -166,14 +157,8 @@ export default class ImportService extends Service { // 1.1 Filter on .cb* extensions .pipe( through2.obj(function (item, enc, next) { - let fileExtension = path.extname( - item.path - ); - if ( - [".cbz", ".cbr", ".cb7"].includes( - fileExtension - ) - ) { + let fileExtension = path.extname(item.path); + if ([".cbz", ".cbr", ".cb7"].includes(fileExtension)) { this.push(item); } next(); @@ -182,10 +167,7 @@ export default class ImportService extends Service { // 1.2 Pipe filtered results to the next step // Enqueue the job in the queue .on("data", async (item) => { - console.info( - "Found a file at path: %s", - item.path - ); + console.info("Found a file at path: %s", item.path); let comicExists = await Comic.exists({ "rawFileDetails.name": `${path.basename( item.path, @@ -194,14 +176,8 @@ export default class ImportService extends Service { }); if (!comicExists) { // 2.1 Reset the job counters in Redis - await pubClient.set( - "completedJobCount", - 0 - ); - await pubClient.set( - "failedJobCount", - 0 - ); + await pubClient.set("completedJobCount", 0); + await pubClient.set("failedJobCount", 0); // 2.2 Send the extraction job to the queue this.broker.call("jobqueue.enqueue", { fileObject: { @@ -210,11 +186,10 @@ export default class ImportService extends Service { }, sessionId, importType: "new", + queueName: "enqueue.async", }); } else { - console.log( - "Comic already exists in the library." - ); + console.log("Comic already exists in the library."); } }) .on("end", () => { @@ -266,28 +241,19 @@ export default class ImportService extends Service { // we solicit volume information and add that to mongo if ( comicMetadata.sourcedMetadata.comicvine && - !isNil( - comicMetadata.sourcedMetadata.comicvine - .volume - ) + !isNil(comicMetadata.sourcedMetadata.comicvine.volume) ) { - volumeDetails = await this.broker.call( - "comicvine.getVolumes", - { - volumeURI: - comicMetadata.sourcedMetadata - .comicvine.volume - .api_detail_url, - } - ); + volumeDetails = await this.broker.call("comicvine.getVolumes", { + volumeURI: + comicMetadata.sourcedMetadata.comicvine.volume + .api_detail_url, + }); comicMetadata.sourcedMetadata.comicvine.volumeInformation = volumeDetails.results; } console.log("Saving to Mongo..."); - console.log( - `Import type: [${ctx.params.importType}]` - ); + console.log(`Import type: [${ctx.params.importType}]`); switch (ctx.params.importType) { case "new": return await Comic.create(comicMetadata); @@ -308,10 +274,7 @@ export default class ImportService extends Service { } } catch (error) { console.log(error); - throw new Errors.MoleculerError( - "Import failed.", - 500 - ); + throw new Errors.MoleculerError("Import failed.", 500); } }, }, @@ -329,9 +292,7 @@ export default class ImportService extends Service { ) { // 1. Find mongo object by id // 2. Import payload into sourcedMetadata.comicvine - const comicObjectId = new ObjectId( - ctx.params.comicObjectId - ); + const comicObjectId = new ObjectId(ctx.params.comicObjectId); return new Promise(async (resolve, reject) => { let volumeDetails = {}; @@ -340,18 +301,15 @@ export default class ImportService extends Service { const volumeDetails = await this.broker.call( "comicvine.getVolumes", { - volumeURI: - matchedResult.volume.api_detail_url, + volumeURI: matchedResult.volume.api_detail_url, } ); - matchedResult.volumeInformation = - volumeDetails.results; + matchedResult.volumeInformation = volumeDetails.results; Comic.findByIdAndUpdate( comicObjectId, { $set: { - "sourcedMetadata.comicvine": - matchedResult, + "sourcedMetadata.comicvine": matchedResult, }, }, { new: true }, @@ -382,9 +340,7 @@ export default class ImportService extends Service { }> ) { console.log(JSON.stringify(ctx.params, null, 2)); - const comicObjectId = new ObjectId( - ctx.params.comicObjectId - ); + const comicObjectId = new ObjectId(ctx.params.comicObjectId); return new Promise((resolve, reject) => { Comic.findByIdAndUpdate( @@ -439,9 +395,7 @@ export default class ImportService extends Service { params: { ids: "array" }, handler: async (ctx: Context<{ ids: [string] }>) => { console.log(ctx.params.ids); - const queryIds = ctx.params.ids.map( - (id) => new ObjectId(id) - ); + const queryIds = ctx.params.ids.map((id) => new ObjectId(id)); return await Comic.find({ _id: { $in: queryIds, @@ -457,8 +411,7 @@ export default class ImportService extends Service { const volumes = await Comic.aggregate([ { $project: { - volumeInfo: - "$sourcedMetadata.comicvine.volumeInformation", + volumeInfo: "$sourcedMetadata.comicvine.volumeInformation", }, }, { @@ -504,52 +457,46 @@ export default class ImportService extends Service { const { queryObjects } = ctx.params; // construct the query for ElasticSearch let elasticSearchQuery = {}; - const elasticSearchQueries = queryObjects.map( - (queryObject) => { - console.log("Volume: ", queryObject.volumeName); - console.log("Issue: ", queryObject.issueName); - if (queryObject.issueName === null) { - queryObject.issueName = ""; - } - if (queryObject.volumeName === null) { - queryObject.volumeName = ""; - } - elasticSearchQuery = { - bool: { - must: [ - { - match_phrase: { - "rawFileDetails.name": - queryObject.volumeName, - }, - }, - { - term: { - "inferredMetadata.issue.number": - parseInt( - queryObject.issueNumber, - 10 - ), - }, - }, - ], - }, - }; - - return [ - { - index: "comics", - search_type: "dfs_query_then_fetch", - }, - { - query: elasticSearchQuery, - }, - ]; + const elasticSearchQueries = queryObjects.map((queryObject) => { + console.log("Volume: ", queryObject.volumeName); + console.log("Issue: ", queryObject.issueName); + if (queryObject.issueName === null) { + queryObject.issueName = ""; } - ); - console.log( - JSON.stringify(elasticSearchQueries, null, 2) - ); + if (queryObject.volumeName === null) { + queryObject.volumeName = ""; + } + elasticSearchQuery = { + bool: { + must: [ + { + match_phrase: { + "rawFileDetails.name": queryObject.volumeName, + }, + }, + { + term: { + "inferredMetadata.issue.number": parseInt( + queryObject.issueNumber, + 10 + ), + }, + }, + ], + }, + }; + + return [ + { + index: "comics", + search_type: "dfs_query_then_fetch", + }, + { + query: elasticSearchQuery, + }, + ]; + }); + console.log(JSON.stringify(elasticSearchQueries, null, 2)); return await ctx.broker.call("search.searchComic", { elasticSearchQueries, @@ -562,10 +509,11 @@ export default class ImportService extends Service { rest: "GET /libraryStatistics", params: {}, handler: async (ctx: Context<{}>) => { - const comicDirectorySize = await getSizeOfDirectory( - COMICS_DIRECTORY, - [".cbz", ".cbr", ".cb7"] - ); + const comicDirectorySize = await getSizeOfDirectory(COMICS_DIRECTORY, [ + ".cbz", + ".cbr", + ".cb7", + ]); const totalCount = await Comic.countDocuments({}); const statistics = await Comic.aggregate([ { @@ -574,11 +522,7 @@ export default class ImportService extends Service { { $match: { "rawFileDetails.extension": { - $in: [ - ".cbr", - ".cbz", - ".cb7", - ], + $in: [".cbr", ".cbz", ".cb7"], }, }, }, @@ -592,10 +536,9 @@ export default class ImportService extends Service { issues: [ { $match: { - "sourcedMetadata.comicvine.volumeInformation": - { - $gt: {}, - }, + "sourcedMetadata.comicvine.volumeInformation": { + $gt: {}, + }, }, }, { @@ -658,23 +601,16 @@ export default class ImportService extends Service { .drop() .then(async (data) => { console.info(data); - const coversFolderDeleteResult = - fsExtra.emptyDirSync( - path.resolve( - `${USERDATA_DIRECTORY}/covers` - ) - ); - const expandedFolderDeleteResult = - fsExtra.emptyDirSync( - path.resolve( - `${USERDATA_DIRECTORY}/expanded` - ) - ); - const eSIndicesDeleteResult = - await ctx.broker.call( - "search.deleteElasticSearchIndices", - {} - ); + const coversFolderDeleteResult = fsExtra.emptyDirSync( + path.resolve(`${USERDATA_DIRECTORY}/covers`) + ); + const expandedFolderDeleteResult = fsExtra.emptyDirSync( + path.resolve(`${USERDATA_DIRECTORY}/expanded`) + ); + const eSIndicesDeleteResult = await ctx.broker.call( + "search.deleteElasticSearchIndices", + {} + ); return { data, coversFolderDeleteResult, diff --git a/utils/uncompression.utils.ts b/utils/uncompression.utils.ts index ef8e962..1304f8f 100644 --- a/utils/uncompression.utils.ts +++ b/utils/uncompression.utils.ts @@ -74,15 +74,14 @@ const errors = []; */ export const extractComicInfoXMLFromRar = async ( filePath: string, - mimeType: string, + mimeType: string ): Promise => { try { // Create the target directory const directoryOptions = { mode: 0o2775, }; - const { fileNameWithoutExtension, extension } = - getFileConstituents(filePath); + const { fileNameWithoutExtension, extension } = getFileConstituents(filePath); const targetDirectory = `${USERDATA_DIRECTORY}/covers/${sanitize( fileNameWithoutExtension )}`; @@ -93,17 +92,15 @@ export const extractComicInfoXMLFromRar = async ( bin: `${UNRAR_BIN_PATH}`, // this will change depending on Docker base OS arguments: ["-v"], }); - const filesInArchive: [RarFile] = await new Promise( - (resolve, reject) => { - return archive.list((err, entries) => { - if (err) { - console.log(`DEBUG: ${JSON.stringify(err, null, 2)}`); - reject(err); - } - resolve(entries); - }); - } - ); + const filesInArchive: [RarFile] = await new Promise((resolve, reject) => { + return archive.list((err, entries) => { + if (err) { + console.log(`DEBUG: ${JSON.stringify(err, null, 2)}`); + reject(err); + } + resolve(entries); + }); + }); remove(filesInArchive, ({ type }) => type === "Directory"); const comicInfoXML = remove( @@ -113,10 +110,7 @@ export const extractComicInfoXMLFromRar = async ( remove( filesInArchive, - ({ name }) => - !IMPORT_IMAGE_FILE_FORMATS.includes( - path.extname(name).toLowerCase() - ) + ({ name }) => !IMPORT_IMAGE_FILE_FORMATS.includes(path.extname(name).toLowerCase()) ); const files = filesInArchive.sort((a, b) => { if (!isUndefined(a) && !isUndefined(b)) { @@ -129,12 +123,8 @@ export const extractComicInfoXMLFromRar = async ( const comicInfoXMLFilePromise = new Promise((resolve, reject) => { let comicinfostring = ""; if (!isUndefined(comicInfoXML[0])) { - const comicInfoXMLFileName = path.basename( - comicInfoXML[0].name - ); - const writeStream = createWriteStream( - `${targetDirectory}/${comicInfoXMLFileName}` - ); + const comicInfoXMLFileName = path.basename(comicInfoXML[0].name); + const writeStream = createWriteStream(`${targetDirectory}/${comicInfoXMLFileName}`); archive.stream(comicInfoXML[0]["name"]).pipe(writeStream); writeStream.on("finish", async () => { @@ -147,11 +137,7 @@ export const extractComicInfoXMLFromRar = async ( }); readStream.on("error", (error) => reject(error)); readStream.on("end", async () => { - if ( - existsSync( - `${targetDirectory}/${comicInfoXMLFileName}` - ) - ) { + if (existsSync(`${targetDirectory}/${comicInfoXMLFileName}`)) { const comicInfoJSON = await convertXMLToJSON( comicinfostring.toString() ); @@ -172,34 +158,29 @@ export const extractComicInfoXMLFromRar = async ( const sharpStream = sharp().resize(275).toFormat("png"); const coverExtractionStream = archive.stream(files[0].name); const resizeStream = coverExtractionStream.pipe(sharpStream); - resizeStream.toFile( - `${targetDirectory}/${coverFile}`, - (err, info) => { - if (err) { - reject(err); - } - checkFileExists(`${targetDirectory}/${coverFile}`).then( - (bool) => { - console.log(`${coverFile} exists: ${bool}`); - // orchestrate result - resolve({ - filePath, - name: fileNameWithoutExtension, - extension, - containedIn: targetDirectory, - fileSize: fse.statSync(filePath).size, - mimeType, - cover: { - filePath: path.relative( - process.cwd(), - `${targetDirectory}/${coverFile}` - ), - }, - }); - } - ); + resizeStream.toFile(`${targetDirectory}/${coverFile}`, (err, info) => { + if (err) { + reject(err); } - ); + checkFileExists(`${targetDirectory}/${coverFile}`).then((bool) => { + console.log(`${coverFile} exists: ${bool}`); + // orchestrate result + resolve({ + filePath, + name: fileNameWithoutExtension, + extension, + containedIn: targetDirectory, + fileSize: fse.statSync(filePath).size, + mimeType, + cover: { + filePath: path.relative( + process.cwd(), + `${targetDirectory}/${coverFile}` + ), + }, + }); + }); + }); }); return Promise.all([comicInfoXMLFilePromise, coverFilePromise]); @@ -210,15 +191,14 @@ export const extractComicInfoXMLFromRar = async ( export const extractComicInfoXMLFromZip = async ( filePath: string, - mimeType: string, + mimeType: string ): Promise => { try { // Create the target directory const directoryOptions = { mode: 0o2775, }; - const { fileNameWithoutExtension, extension } = - getFileConstituents(filePath); + const { fileNameWithoutExtension, extension } = getFileConstituents(filePath); const targetDirectory = `${USERDATA_DIRECTORY}/covers/${sanitize( fileNameWithoutExtension )}`; @@ -237,10 +217,7 @@ export const extractComicInfoXMLFromZip = async ( // only allow allowed image formats remove( filesFromArchive.files, - ({ name }) => - !IMPORT_IMAGE_FILE_FORMATS.includes( - path.extname(name).toLowerCase() - ) + ({ name }) => !IMPORT_IMAGE_FILE_FORMATS.includes(path.extname(name).toLowerCase()) ); // Natural sort @@ -261,13 +238,7 @@ export const extractComicInfoXMLFromZip = async ( extractionTargets.push(filesToWriteToDisk.comicInfoXML); } // Extract the files. - await p7zip.extract( - filePath, - targetDirectory, - extractionTargets, - "", - false - ); + await p7zip.extract(filePath, targetDirectory, extractionTargets, "", false); // ComicInfoXML detection, parsing and conversion to JSON // Write ComicInfo.xml to disk @@ -275,26 +246,15 @@ export const extractComicInfoXMLFromZip = async ( const comicInfoXMLPromise = new Promise((resolve, reject) => { if ( !isNil(filesToWriteToDisk.comicInfoXML) && - existsSync( - `${targetDirectory}/${path.basename( - filesToWriteToDisk.comicInfoXML - )}` - ) + existsSync(`${targetDirectory}/${path.basename(filesToWriteToDisk.comicInfoXML)}`) ) { let comicinfoString = ""; const comicInfoXMLStream = createReadStream( - `${targetDirectory}/${path.basename( - filesToWriteToDisk.comicInfoXML - )}` - ); - comicInfoXMLStream.on( - "data", - (data) => (comicinfoString += data) + `${targetDirectory}/${path.basename(filesToWriteToDisk.comicInfoXML)}` ); + comicInfoXMLStream.on("data", (data) => (comicinfoString += data)); comicInfoXMLStream.on("end", async () => { - const comicInfoJSON = await convertXMLToJSON( - comicinfoString.toString() - ); + const comicInfoJSON = await convertXMLToJSON(comicinfoString.toString()); resolve({ comicInfoJSON: comicInfoJSON.comicinfo, }); @@ -314,9 +274,7 @@ export const extractComicInfoXMLFromZip = async ( coverStream .pipe(sharpStream) .toFile( - `${targetDirectory}/${path.basename( - filesToWriteToDisk.coverFile - )}`, + `${targetDirectory}/${path.basename(filesToWriteToDisk.coverFile)}`, (err, info) => { if (err) { reject(err); @@ -365,13 +323,10 @@ export const extractFromArchive = async (filePath: string) => { return Object.assign({}, ...cbrResult); default: - console.error( - "Error inferring filetype for comicinfo.xml extraction." - ); + console.error("Error inferring filetype for comicinfo.xml extraction."); throw new MoleculerError({}, 500, "FILETYPE_INFERENCE_ERROR", { - data: { message: "Cannot infer filetype."}, + data: { message: "Cannot infer filetype." }, }); - } }; @@ -381,10 +336,7 @@ export const extractFromArchive = async (filePath: string) => { * @param {any} options * @returns {Promise} A promise containing the contents of the uncompressed archive. */ -export const uncompressEntireArchive = async ( - filePath: string, - options: any -) => { +export const uncompressEntireArchive = async (filePath: string, options: any) => { const mimeType = await getMimeType(filePath); console.log(`File has the following mime-type: ${mimeType}`); switch (mimeType) { @@ -426,8 +378,7 @@ export const uncompressRarArchive = async (filePath: string, options: any) => { const directoryOptions = { mode: 0o2775, }; - const { fileNameWithoutExtension, extension } = - getFileConstituents(filePath); + const { fileNameWithoutExtension, extension } = getFileConstituents(filePath); const targetDirectory = `${USERDATA_DIRECTORY}/expanded/${options.purpose}/${fileNameWithoutExtension}`; await createDirectory(directoryOptions, targetDirectory); @@ -464,10 +415,7 @@ export const uncompressRarArchive = async (filePath: string, options: any) => { return await resizeImageDirectory(targetDirectory, options); }; -export const resizeImageDirectory = async ( - directoryPath: string, - options: any -) => { +export const resizeImageDirectory = async (directoryPath: string, options: any) => { const files = await walkFolder(directoryPath, [ ".jpg", ".jpeg", @@ -495,25 +443,15 @@ export const resizeImage = (directoryPath: string, file: any, options: any) => { const { baseWidth } = options.imageResizeOptions; const sharpResizeInstance = sharp().resize(baseWidth).toFormat("jpg"); return new Promise((resolve, reject) => { - const resizedStream = createReadStream( - `${directoryPath}/${file.name}${file.extension}` - ); + const resizedStream = createReadStream(`${directoryPath}/${file.name}${file.extension}`); if (fse.existsSync(`${directoryPath}/${file.name}${file.extension}`)) { resizedStream .pipe(sharpResizeInstance) - .toFile( - `${directoryPath}/${file.name}_${baseWidth}px${file.extension}` - ) + .toFile(`${directoryPath}/${file.name}_${baseWidth}px${file.extension}`) .then((data) => { - console.log( - `Resized image ${JSON.stringify(data, null, 4)}` - ); - fse.unlink( - `${directoryPath}/${file.name}${file.extension}` - ); - resolve( - `${directoryPath}/${file.name}_${baseWidth}px${file.extension}` - ); + console.log(`Resized image ${JSON.stringify(data, null, 4)}`); + fse.unlink(`${directoryPath}/${file.name}${file.extension}`); + resolve(`${directoryPath}/${file.name}_${baseWidth}px${file.extension}`); }); } });