From bbd2906ebf2bd3057164f4ae25548d85904d61d9 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Sat, 6 Jan 2024 11:17:40 -0500 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=8F=97=EF=B8=8F=20Added=20some=20arch?= =?UTF-8?q?ive-related=20keys=20to=20Comic=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/comic.model.ts | 4 + services/imagetransformation.service.ts | 11 +- services/jobqueue.service.ts | 153 ++++++++++++---- services/library.service.ts | 234 ++++++++++++++++-------- utils/uncompression.utils.ts | 177 ++++++++++++------ 5 files changed, 404 insertions(+), 175 deletions(-) diff --git a/models/comic.model.ts b/models/comic.model.ts index 0483445..8778cd6 100644 --- a/models/comic.model.ts +++ b/models/comic.model.ts @@ -28,6 +28,10 @@ const RawFileDetailsSchema = mongoose.Schema({ mimeType: String, containedIn: String, pageCount: Number, + archive: { + uncompressed: Boolean, + expandedPath: String, + }, cover: { filePath: String, stats: Object, diff --git a/services/imagetransformation.service.ts b/services/imagetransformation.service.ts index 4cac0a6..9202dc7 100644 --- a/services/imagetransformation.service.ts +++ b/services/imagetransformation.service.ts @@ -7,6 +7,8 @@ import { ServiceSchema, Errors, } from "moleculer"; +import { DbMixin } from "../mixins/db.mixin"; +import Comic from "../models/comic.model"; import path from "path"; import { analyze, @@ -22,16 +24,13 @@ export default class ImageTransformation extends Service { super(broker); this.parseServiceSchema({ name: "imagetransformation", - mixins: [], + mixins: [DbMixin("comics", Comic)], settings: { // Available fields in the responses - fields: ["_id", "name", "quantity", "price"], + fields: ["_id"], // Validator for the `create` & `insert` actions. - entityValidator: { - name: "string|min:3", - price: "number|positive", - }, + entityValidator: {}, }, hooks: {}, actions: { diff --git a/services/jobqueue.service.ts b/services/jobqueue.service.ts index f1105e4..eea9f84 100644 --- a/services/jobqueue.service.ts +++ b/services/jobqueue.service.ts @@ -2,9 +2,16 @@ import { Context, Service, ServiceBroker } from "moleculer"; import JobResult from "../models/jobresult.model"; import { refineQuery } from "filename-parser"; import BullMqMixin from "moleculer-bullmq"; -import { extractFromArchive, uncompressEntireArchive } from "../utils/uncompression.utils"; +import { DbMixin } from "../mixins/db.mixin"; +import Comic from "../models/comic.model"; +const ObjectId = require("mongoose").Types.ObjectId; +import { + extractFromArchive, + uncompressEntireArchive, +} from "../utils/uncompression.utils"; import { isNil, isUndefined } from "lodash"; import { pubClient } from "../config/redis.config"; +import path from "path"; const { MoleculerError } = require("moleculer").Errors; @@ -15,7 +22,7 @@ export default class JobQueueService extends Service { this.parseServiceSchema({ name: "jobqueue", hooks: {}, - mixins: [BullMqMixin], + mixins: [DbMixin("comics", Comic), BullMqMixin], settings: { bullmq: { client: process.env.REDIS_URI, @@ -47,13 +54,20 @@ export default class JobQueueService extends Service { enqueue: { queue: true, rest: "/GET enqueue", - handler: async (ctx: Context<{ queueName: string; description: string }>) => { + handler: async ( + ctx: Context<{ queueName: string; description: string }> + ) => { console.log(ctx.params); const { queueName, description } = ctx.params; // Enqueue the job - const job = await this.localQueue(ctx, queueName, ctx.params, { - priority: 10, - }); + const job = await this.localQueue( + ctx, + queueName, + ctx.params, + { + priority: 10, + } + ); console.log(`Job ${job.id} enqueued`); console.log(`${description}`); @@ -68,13 +82,17 @@ export default class JobQueueService extends Service { }> ) => { try { - console.log(`Recieved Job ID ${ctx.locals.job.id}, processing...`); + console.log( + `Recieved Job ID ${ctx.locals.job.id}, processing...` + ); console.log(ctx.params); // 1. De-structure the job params const { fileObject } = ctx.locals.job.data.params; // 2. Extract metadata from the archive - const result = await extractFromArchive(fileObject.filePath); + const result = await extractFromArchive( + fileObject.filePath + ); const { name, filePath, @@ -87,7 +105,9 @@ export default class JobQueueService extends Service { } = result; // 3a. Infer any issue-related metadata from the filename - const { inferredIssueDetails } = refineQuery(result.name); + const { inferredIssueDetails } = refineQuery( + result.name + ); console.log( "Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2) @@ -127,7 +147,8 @@ export default class JobQueueService extends Service { // "acquisition.directconnect.downloads": [], // mark the metadata source - "acquisition.source.name": ctx.locals.job.data.params.sourcedFrom, + "acquisition.source.name": + ctx.locals.job.data.params.sourcedFrom, }; // 3c. Add the bundleId, if present to the payload @@ -138,8 +159,13 @@ export default class JobQueueService extends Service { // 3d. Add the sourcedMetadata, if present if ( - !isNil(ctx.locals.job.data.params.sourcedMetadata) && - !isUndefined(ctx.locals.job.data.params.sourcedMetadata.comicvine) + !isNil( + ctx.locals.job.data.params.sourcedMetadata + ) && + !isUndefined( + ctx.locals.job.data.params.sourcedMetadata + .comicvine + ) ) { Object.assign( payload.sourcedMetadata, @@ -148,11 +174,15 @@ export default class JobQueueService extends Service { } // 4. write to mongo - const importResult = await this.broker.call("library.rawImportToDB", { - importType: ctx.locals.job.data.params.importType, - bundleId, - payload, - }); + const importResult = await this.broker.call( + "library.rawImportToDB", + { + importType: + ctx.locals.job.data.params.importType, + bundleId, + payload, + } + ); return { data: { importResult, @@ -164,9 +194,14 @@ export default class JobQueueService extends Service { console.error( `An error occurred processing Job ID ${ctx.locals.job.id}` ); - throw new MoleculerError(error, 500, "IMPORT_JOB_ERROR", { - data: ctx.params.sessionId, - }); + throw new MoleculerError( + error, + 500, + "IMPORT_JOB_ERROR", + { + data: ctx.params.sessionId, + } + ); } }, }, @@ -194,7 +229,8 @@ export default class JobQueueService extends Service { statuses: { $push: { status: "$_id.status", - earliestTimestamp: "$earliestTimestamp", + earliestTimestamp: + "$earliestTimestamp", count: "$count", }, }, @@ -214,7 +250,10 @@ export default class JobQueueService extends Service { { $cond: [ { - $eq: ["$$this.status", "completed"], + $eq: [ + "$$this.status", + "completed", + ], }, "$$this.count", 0, @@ -234,7 +273,10 @@ export default class JobQueueService extends Service { { $cond: [ { - $eq: ["$$this.status", "failed"], + $eq: [ + "$$this.status", + "failed", + ], }, "$$this.count", 0, @@ -254,21 +296,62 @@ export default class JobQueueService extends Service { }, "uncompressFullArchive.async": { rest: "POST /uncompressFullArchive", - handler: async (ctx: Context<{ filePath: string; options: any }>) => { - const { filePath, options } = ctx.params; - console.log("asd", filePath); + handler: async ( + ctx: Context<{ + filePath: string; + comicObjectId: string; + options: any; + }> + ) => { + console.log( + `Recieved Job ID ${JSON.stringify( + ctx.locals + )}, processing...` + ); + const { filePath, options, comicObjectId } = ctx.params; + const comicId = new ObjectId(comicObjectId); // 2. Extract metadata from the archive - return await uncompressEntireArchive(filePath, options); + const result: string[] = await uncompressEntireArchive( + filePath, + options + ); + + if (Array.isArray(result) && result.length !== 0) { + // Get the containing directory of the uncompressed archive + const directoryPath = path.dirname(result[0]); + // Add to mongo object + await Comic.findByIdAndUpdate( + comicId, + { + $set: { + "rawFileDetails.archive": { + uncompressed: true, + expandedPath: directoryPath, + }, + }, + }, + { new: true, safe: true, upsert: true } + ); + return result; + } }, }, }, events: { - async "uncompressFullArchive.async.active"(ctx: Context<{ id: number }>) { - console.log(`Uncompression Job ID ${ctx.params.id} is set to active.`); + async "uncompressFullArchive.async.active"( + ctx: Context<{ id: number }> + ) { + console.log( + `Uncompression Job ID ${ctx.params.id} is set to active.` + ); }, - async "uncompressFullArchive.async.completed"(ctx: Context<{ id: number }>) { - console.log(`Uncompression Job ID ${ctx.params.id} completed.`); + async "uncompressFullArchive.async.completed"( + ctx: Context<{ id: number }> + ) { + console.log( + `Uncompression Job ID ${ctx.params.id} completed.` + ); }, // use the `${QUEUE_NAME}.QUEUE_EVENT` scheme async "enqueue.async.active"(ctx: Context<{ id: Number }>) { @@ -292,7 +375,9 @@ export default class JobQueueService extends Service { // 2. Increment the completed job counter await pubClient.incr("completedJobCount"); // 3. Fetch the completed job count for the final payload to be sent to the client - const completedJobCount = await pubClient.get("completedJobCount"); + const completedJobCount = await pubClient.get( + "completedJobCount" + ); // 4. Emit the LS_COVER_EXTRACTED event with the necessary details await this.broker.call("socket.broadcast", { namespace: "/", @@ -319,7 +404,9 @@ export default class JobQueueService extends Service { async "enqueue.async.failed"(ctx) { const job = await this.job(ctx.params.id); await pubClient.incr("failedJobCount"); - const failedJobCount = await pubClient.get("failedJobCount"); + const failedJobCount = await pubClient.get( + "failedJobCount" + ); await JobResult.create({ id: ctx.params.id, diff --git a/services/library.service.ts b/services/library.service.ts index 3bc5284..4aa3dab 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -33,7 +33,13 @@ SOFTWARE. "use strict"; import { isNil } from "lodash"; -import { Context, Service, ServiceBroker, ServiceSchema, Errors } from "moleculer"; +import { + Context, + Service, + ServiceBroker, + ServiceSchema, + Errors, +} from "moleculer"; import { DbMixin } from "../mixins/db.mixin"; import Comic from "../models/comic.model"; import { walkFolder, getSizeOfDirectory } from "../utils/file.utils"; @@ -95,10 +101,19 @@ export default class ImportService extends Service { uncompressFullArchive: { rest: "POST /uncompressFullArchive", params: {}, - handler: async (ctx: Context<{ filePath: string; options: any }>) => { - await broker.call("importqueue.uncompressResize", { + handler: async ( + ctx: Context<{ + filePath: string; + comicObjectId: string; + options: any; + }> + ) => { + this.broker.call("jobqueue.enqueue", { filePath: ctx.params.filePath, + comicObjectId: ctx.params.comicObjectId, options: ctx.params.options, + queueName: "uncompressFullArchive.async", + description: `Job for uncompressing archive at ${ctx.params.filePath}`, }); }, }, @@ -113,7 +128,8 @@ export default class ImportService extends Service { }); // Determine source where the comic was added from // and gather identifying information about it - const sourceName = referenceComicObject[0].acquisition.source.name; + const sourceName = + referenceComicObject[0].acquisition.source.name; const { sourcedMetadata } = referenceComicObject[0]; const filePath = `${COMICS_DIRECTORY}/${ctx.params.bundle.data.name}`; @@ -157,8 +173,14 @@ export default class ImportService extends Service { // 1.1 Filter on .cb* extensions .pipe( through2.obj(function (item, enc, next) { - let fileExtension = path.extname(item.path); - if ([".cbz", ".cbr", ".cb7"].includes(fileExtension)) { + let fileExtension = path.extname( + item.path + ); + if ( + [".cbz", ".cbr", ".cb7"].includes( + fileExtension + ) + ) { this.push(item); } next(); @@ -167,7 +189,10 @@ export default class ImportService extends Service { // 1.2 Pipe filtered results to the next step // Enqueue the job in the queue .on("data", async (item) => { - console.info("Found a file at path: %s", item.path); + console.info( + "Found a file at path: %s", + item.path + ); let comicExists = await Comic.exists({ "rawFileDetails.name": `${path.basename( item.path, @@ -176,8 +201,14 @@ export default class ImportService extends Service { }); if (!comicExists) { // 2.1 Reset the job counters in Redis - await pubClient.set("completedJobCount", 0); - await pubClient.set("failedJobCount", 0); + await pubClient.set( + "completedJobCount", + 0 + ); + await pubClient.set( + "failedJobCount", + 0 + ); // 2.2 Send the extraction job to the queue this.broker.call("jobqueue.enqueue", { fileObject: { @@ -189,7 +220,9 @@ export default class ImportService extends Service { queueName: "enqueue.async", }); } else { - console.log("Comic already exists in the library."); + console.log( + "Comic already exists in the library." + ); } }) .on("end", () => { @@ -241,19 +274,28 @@ export default class ImportService extends Service { // we solicit volume information and add that to mongo if ( comicMetadata.sourcedMetadata.comicvine && - !isNil(comicMetadata.sourcedMetadata.comicvine.volume) + !isNil( + comicMetadata.sourcedMetadata.comicvine + .volume + ) ) { - volumeDetails = await this.broker.call("comicvine.getVolumes", { - volumeURI: - comicMetadata.sourcedMetadata.comicvine.volume - .api_detail_url, - }); + volumeDetails = await this.broker.call( + "comicvine.getVolumes", + { + volumeURI: + comicMetadata.sourcedMetadata + .comicvine.volume + .api_detail_url, + } + ); comicMetadata.sourcedMetadata.comicvine.volumeInformation = volumeDetails.results; } console.log("Saving to Mongo..."); - console.log(`Import type: [${ctx.params.importType}]`); + console.log( + `Import type: [${ctx.params.importType}]` + ); switch (ctx.params.importType) { case "new": return await Comic.create(comicMetadata); @@ -274,7 +316,10 @@ export default class ImportService extends Service { } } catch (error) { console.log(error); - throw new Errors.MoleculerError("Import failed.", 500); + throw new Errors.MoleculerError( + "Import failed.", + 500 + ); } }, }, @@ -292,7 +337,9 @@ export default class ImportService extends Service { ) { // 1. Find mongo object by id // 2. Import payload into sourcedMetadata.comicvine - const comicObjectId = new ObjectId(ctx.params.comicObjectId); + const comicObjectId = new ObjectId( + ctx.params.comicObjectId + ); return new Promise(async (resolve, reject) => { let volumeDetails = {}; @@ -301,15 +348,18 @@ export default class ImportService extends Service { const volumeDetails = await this.broker.call( "comicvine.getVolumes", { - volumeURI: matchedResult.volume.api_detail_url, + volumeURI: + matchedResult.volume.api_detail_url, } ); - matchedResult.volumeInformation = volumeDetails.results; + matchedResult.volumeInformation = + volumeDetails.results; Comic.findByIdAndUpdate( comicObjectId, { $set: { - "sourcedMetadata.comicvine": matchedResult, + "sourcedMetadata.comicvine": + matchedResult, }, }, { new: true }, @@ -340,7 +390,9 @@ export default class ImportService extends Service { }> ) { console.log(JSON.stringify(ctx.params, null, 2)); - const comicObjectId = new ObjectId(ctx.params.comicObjectId); + const comicObjectId = new ObjectId( + ctx.params.comicObjectId + ); return new Promise((resolve, reject) => { Comic.findByIdAndUpdate( @@ -395,7 +447,9 @@ export default class ImportService extends Service { params: { ids: "array" }, handler: async (ctx: Context<{ ids: [string] }>) => { console.log(ctx.params.ids); - const queryIds = ctx.params.ids.map((id) => new ObjectId(id)); + const queryIds = ctx.params.ids.map( + (id) => new ObjectId(id) + ); return await Comic.find({ _id: { $in: queryIds, @@ -411,7 +465,8 @@ export default class ImportService extends Service { const volumes = await Comic.aggregate([ { $project: { - volumeInfo: "$sourcedMetadata.comicvine.volumeInformation", + volumeInfo: + "$sourcedMetadata.comicvine.volumeInformation", }, }, { @@ -457,46 +512,52 @@ export default class ImportService extends Service { const { queryObjects } = ctx.params; // construct the query for ElasticSearch let elasticSearchQuery = {}; - const elasticSearchQueries = queryObjects.map((queryObject) => { - console.log("Volume: ", queryObject.volumeName); - console.log("Issue: ", queryObject.issueName); - if (queryObject.issueName === null) { - queryObject.issueName = ""; - } - if (queryObject.volumeName === null) { - queryObject.volumeName = ""; - } - elasticSearchQuery = { - bool: { - must: [ - { - match_phrase: { - "rawFileDetails.name": queryObject.volumeName, + const elasticSearchQueries = queryObjects.map( + (queryObject) => { + console.log("Volume: ", queryObject.volumeName); + console.log("Issue: ", queryObject.issueName); + if (queryObject.issueName === null) { + queryObject.issueName = ""; + } + if (queryObject.volumeName === null) { + queryObject.volumeName = ""; + } + elasticSearchQuery = { + bool: { + must: [ + { + match_phrase: { + "rawFileDetails.name": + queryObject.volumeName, + }, }, - }, - { - term: { - "inferredMetadata.issue.number": parseInt( - queryObject.issueNumber, - 10 - ), + { + term: { + "inferredMetadata.issue.number": + parseInt( + queryObject.issueNumber, + 10 + ), + }, }, - }, - ], - }, - }; + ], + }, + }; - return [ - { - index: "comics", - search_type: "dfs_query_then_fetch", - }, - { - query: elasticSearchQuery, - }, - ]; - }); - console.log(JSON.stringify(elasticSearchQueries, null, 2)); + return [ + { + index: "comics", + search_type: "dfs_query_then_fetch", + }, + { + query: elasticSearchQuery, + }, + ]; + } + ); + console.log( + JSON.stringify(elasticSearchQueries, null, 2) + ); return await ctx.broker.call("search.searchComic", { elasticSearchQueries, @@ -509,11 +570,10 @@ export default class ImportService extends Service { rest: "GET /libraryStatistics", params: {}, handler: async (ctx: Context<{}>) => { - const comicDirectorySize = await getSizeOfDirectory(COMICS_DIRECTORY, [ - ".cbz", - ".cbr", - ".cb7", - ]); + const comicDirectorySize = await getSizeOfDirectory( + COMICS_DIRECTORY, + [".cbz", ".cbr", ".cb7"] + ); const totalCount = await Comic.countDocuments({}); const statistics = await Comic.aggregate([ { @@ -522,7 +582,11 @@ export default class ImportService extends Service { { $match: { "rawFileDetails.extension": { - $in: [".cbr", ".cbz", ".cb7"], + $in: [ + ".cbr", + ".cbz", + ".cb7", + ], }, }, }, @@ -536,9 +600,10 @@ export default class ImportService extends Service { issues: [ { $match: { - "sourcedMetadata.comicvine.volumeInformation": { - $gt: {}, - }, + "sourcedMetadata.comicvine.volumeInformation": + { + $gt: {}, + }, }, }, { @@ -601,16 +666,23 @@ export default class ImportService extends Service { .drop() .then(async (data) => { console.info(data); - const coversFolderDeleteResult = fsExtra.emptyDirSync( - path.resolve(`${USERDATA_DIRECTORY}/covers`) - ); - const expandedFolderDeleteResult = fsExtra.emptyDirSync( - path.resolve(`${USERDATA_DIRECTORY}/expanded`) - ); - const eSIndicesDeleteResult = await ctx.broker.call( - "search.deleteElasticSearchIndices", - {} - ); + const coversFolderDeleteResult = + fsExtra.emptyDirSync( + path.resolve( + `${USERDATA_DIRECTORY}/covers` + ) + ); + const expandedFolderDeleteResult = + fsExtra.emptyDirSync( + path.resolve( + `${USERDATA_DIRECTORY}/expanded` + ) + ); + const eSIndicesDeleteResult = + await ctx.broker.call( + "search.deleteElasticSearchIndices", + {} + ); return { data, coversFolderDeleteResult, diff --git a/utils/uncompression.utils.ts b/utils/uncompression.utils.ts index 1304f8f..ea6f174 100644 --- a/utils/uncompression.utils.ts +++ b/utils/uncompression.utils.ts @@ -81,7 +81,8 @@ export const extractComicInfoXMLFromRar = async ( const directoryOptions = { mode: 0o2775, }; - const { fileNameWithoutExtension, extension } = getFileConstituents(filePath); + const { fileNameWithoutExtension, extension } = + getFileConstituents(filePath); const targetDirectory = `${USERDATA_DIRECTORY}/covers/${sanitize( fileNameWithoutExtension )}`; @@ -92,15 +93,17 @@ export const extractComicInfoXMLFromRar = async ( bin: `${UNRAR_BIN_PATH}`, // this will change depending on Docker base OS arguments: ["-v"], }); - const filesInArchive: [RarFile] = await new Promise((resolve, reject) => { - return archive.list((err, entries) => { - if (err) { - console.log(`DEBUG: ${JSON.stringify(err, null, 2)}`); - reject(err); - } - resolve(entries); - }); - }); + const filesInArchive: [RarFile] = await new Promise( + (resolve, reject) => { + return archive.list((err, entries) => { + if (err) { + console.log(`DEBUG: ${JSON.stringify(err, null, 2)}`); + reject(err); + } + resolve(entries); + }); + } + ); remove(filesInArchive, ({ type }) => type === "Directory"); const comicInfoXML = remove( @@ -110,7 +113,10 @@ export const extractComicInfoXMLFromRar = async ( remove( filesInArchive, - ({ name }) => !IMPORT_IMAGE_FILE_FORMATS.includes(path.extname(name).toLowerCase()) + ({ name }) => + !IMPORT_IMAGE_FILE_FORMATS.includes( + path.extname(name).toLowerCase() + ) ); const files = filesInArchive.sort((a, b) => { if (!isUndefined(a) && !isUndefined(b)) { @@ -123,8 +129,12 @@ export const extractComicInfoXMLFromRar = async ( const comicInfoXMLFilePromise = new Promise((resolve, reject) => { let comicinfostring = ""; if (!isUndefined(comicInfoXML[0])) { - const comicInfoXMLFileName = path.basename(comicInfoXML[0].name); - const writeStream = createWriteStream(`${targetDirectory}/${comicInfoXMLFileName}`); + const comicInfoXMLFileName = path.basename( + comicInfoXML[0].name + ); + const writeStream = createWriteStream( + `${targetDirectory}/${comicInfoXMLFileName}` + ); archive.stream(comicInfoXML[0]["name"]).pipe(writeStream); writeStream.on("finish", async () => { @@ -137,7 +147,11 @@ export const extractComicInfoXMLFromRar = async ( }); readStream.on("error", (error) => reject(error)); readStream.on("end", async () => { - if (existsSync(`${targetDirectory}/${comicInfoXMLFileName}`)) { + if ( + existsSync( + `${targetDirectory}/${comicInfoXMLFileName}` + ) + ) { const comicInfoJSON = await convertXMLToJSON( comicinfostring.toString() ); @@ -158,29 +172,34 @@ export const extractComicInfoXMLFromRar = async ( const sharpStream = sharp().resize(275).toFormat("png"); const coverExtractionStream = archive.stream(files[0].name); const resizeStream = coverExtractionStream.pipe(sharpStream); - resizeStream.toFile(`${targetDirectory}/${coverFile}`, (err, info) => { - if (err) { - reject(err); + resizeStream.toFile( + `${targetDirectory}/${coverFile}`, + (err, info) => { + if (err) { + reject(err); + } + checkFileExists(`${targetDirectory}/${coverFile}`).then( + (bool) => { + console.log(`${coverFile} exists: ${bool}`); + // orchestrate result + resolve({ + filePath, + name: fileNameWithoutExtension, + extension, + containedIn: targetDirectory, + fileSize: fse.statSync(filePath).size, + mimeType, + cover: { + filePath: path.relative( + process.cwd(), + `${targetDirectory}/${coverFile}` + ), + }, + }); + } + ); } - checkFileExists(`${targetDirectory}/${coverFile}`).then((bool) => { - console.log(`${coverFile} exists: ${bool}`); - // orchestrate result - resolve({ - filePath, - name: fileNameWithoutExtension, - extension, - containedIn: targetDirectory, - fileSize: fse.statSync(filePath).size, - mimeType, - cover: { - filePath: path.relative( - process.cwd(), - `${targetDirectory}/${coverFile}` - ), - }, - }); - }); - }); + ); }); return Promise.all([comicInfoXMLFilePromise, coverFilePromise]); @@ -198,7 +217,8 @@ export const extractComicInfoXMLFromZip = async ( const directoryOptions = { mode: 0o2775, }; - const { fileNameWithoutExtension, extension } = getFileConstituents(filePath); + const { fileNameWithoutExtension, extension } = + getFileConstituents(filePath); const targetDirectory = `${USERDATA_DIRECTORY}/covers/${sanitize( fileNameWithoutExtension )}`; @@ -217,7 +237,10 @@ export const extractComicInfoXMLFromZip = async ( // only allow allowed image formats remove( filesFromArchive.files, - ({ name }) => !IMPORT_IMAGE_FILE_FORMATS.includes(path.extname(name).toLowerCase()) + ({ name }) => + !IMPORT_IMAGE_FILE_FORMATS.includes( + path.extname(name).toLowerCase() + ) ); // Natural sort @@ -238,7 +261,13 @@ export const extractComicInfoXMLFromZip = async ( extractionTargets.push(filesToWriteToDisk.comicInfoXML); } // Extract the files. - await p7zip.extract(filePath, targetDirectory, extractionTargets, "", false); + await p7zip.extract( + filePath, + targetDirectory, + extractionTargets, + "", + false + ); // ComicInfoXML detection, parsing and conversion to JSON // Write ComicInfo.xml to disk @@ -246,15 +275,26 @@ export const extractComicInfoXMLFromZip = async ( const comicInfoXMLPromise = new Promise((resolve, reject) => { if ( !isNil(filesToWriteToDisk.comicInfoXML) && - existsSync(`${targetDirectory}/${path.basename(filesToWriteToDisk.comicInfoXML)}`) + existsSync( + `${targetDirectory}/${path.basename( + filesToWriteToDisk.comicInfoXML + )}` + ) ) { let comicinfoString = ""; const comicInfoXMLStream = createReadStream( - `${targetDirectory}/${path.basename(filesToWriteToDisk.comicInfoXML)}` + `${targetDirectory}/${path.basename( + filesToWriteToDisk.comicInfoXML + )}` + ); + comicInfoXMLStream.on( + "data", + (data) => (comicinfoString += data) ); - comicInfoXMLStream.on("data", (data) => (comicinfoString += data)); comicInfoXMLStream.on("end", async () => { - const comicInfoJSON = await convertXMLToJSON(comicinfoString.toString()); + const comicInfoJSON = await convertXMLToJSON( + comicinfoString.toString() + ); resolve({ comicInfoJSON: comicInfoJSON.comicinfo, }); @@ -274,7 +314,9 @@ export const extractComicInfoXMLFromZip = async ( coverStream .pipe(sharpStream) .toFile( - `${targetDirectory}/${path.basename(filesToWriteToDisk.coverFile)}`, + `${targetDirectory}/${path.basename( + filesToWriteToDisk.coverFile + )}`, (err, info) => { if (err) { reject(err); @@ -315,15 +357,23 @@ export const extractFromArchive = async (filePath: string) => { switch (mimeType) { case "application/x-7z-compressed; charset=binary": case "application/zip; charset=binary": - const cbzResult = await extractComicInfoXMLFromZip(filePath, mimeType); + const cbzResult = await extractComicInfoXMLFromZip( + filePath, + mimeType + ); return Object.assign({}, ...cbzResult); case "application/x-rar; charset=binary": - const cbrResult = await extractComicInfoXMLFromRar(filePath, mimeType); + const cbrResult = await extractComicInfoXMLFromRar( + filePath, + mimeType + ); return Object.assign({}, ...cbrResult); default: - console.error("Error inferring filetype for comicinfo.xml extraction."); + console.error( + "Error inferring filetype for comicinfo.xml extraction." + ); throw new MoleculerError({}, 500, "FILETYPE_INFERENCE_ERROR", { data: { message: "Cannot infer filetype." }, }); @@ -336,7 +386,10 @@ export const extractFromArchive = async (filePath: string) => { * @param {any} options * @returns {Promise} A promise containing the contents of the uncompressed archive. */ -export const uncompressEntireArchive = async (filePath: string, options: any) => { +export const uncompressEntireArchive = async ( + filePath: string, + options: any +) => { const mimeType = await getMimeType(filePath); console.log(`File has the following mime-type: ${mimeType}`); switch (mimeType) { @@ -378,7 +431,8 @@ export const uncompressRarArchive = async (filePath: string, options: any) => { const directoryOptions = { mode: 0o2775, }; - const { fileNameWithoutExtension, extension } = getFileConstituents(filePath); + const { fileNameWithoutExtension, extension } = + getFileConstituents(filePath); const targetDirectory = `${USERDATA_DIRECTORY}/expanded/${options.purpose}/${fileNameWithoutExtension}`; await createDirectory(directoryOptions, targetDirectory); @@ -415,7 +469,10 @@ export const uncompressRarArchive = async (filePath: string, options: any) => { return await resizeImageDirectory(targetDirectory, options); }; -export const resizeImageDirectory = async (directoryPath: string, options: any) => { +export const resizeImageDirectory = async ( + directoryPath: string, + options: any +) => { const files = await walkFolder(directoryPath, [ ".jpg", ".jpeg", @@ -443,15 +500,25 @@ export const resizeImage = (directoryPath: string, file: any, options: any) => { const { baseWidth } = options.imageResizeOptions; const sharpResizeInstance = sharp().resize(baseWidth).toFormat("jpg"); return new Promise((resolve, reject) => { - const resizedStream = createReadStream(`${directoryPath}/${file.name}${file.extension}`); + const resizedStream = createReadStream( + `${directoryPath}/${file.name}${file.extension}` + ); if (fse.existsSync(`${directoryPath}/${file.name}${file.extension}`)) { resizedStream .pipe(sharpResizeInstance) - .toFile(`${directoryPath}/${file.name}_${baseWidth}px${file.extension}`) + .toFile( + `${directoryPath}/${file.name}_${baseWidth}px${file.extension}` + ) .then((data) => { - console.log(`Resized image ${JSON.stringify(data, null, 4)}`); - fse.unlink(`${directoryPath}/${file.name}${file.extension}`); - resolve(`${directoryPath}/${file.name}_${baseWidth}px${file.extension}`); + console.log( + `Resized image ${JSON.stringify(data, null, 4)}` + ); + fse.unlink( + `${directoryPath}/${file.name}${file.extension}` + ); + resolve( + `${directoryPath}/${file.name}_${baseWidth}px${file.extension}` + ); }); } }); From 78f7c1b59585057f6f45eee5455b23a369ea500d Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Sun, 7 Jan 2024 22:13:02 -0500 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=A4=90=20Added=20uncompression=20even?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/jobqueue.service.ts | 11 +++++++++++ services/library.service.ts | 13 +++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/services/jobqueue.service.ts b/services/jobqueue.service.ts index eea9f84..190b330 100644 --- a/services/jobqueue.service.ts +++ b/services/jobqueue.service.ts @@ -352,6 +352,17 @@ export default class JobQueueService extends Service { console.log( `Uncompression Job ID ${ctx.params.id} completed.` ); + const job = await this.job(ctx.params.id); + await this.broker.call("socket.broadcast", { + namespace: "/", + event: "LS_UNCOMPRESSION_JOB_COMPLETE", + args: [ + { + uncompressedArchive: job.returnvalue, + }, + ], + }); + return job.returnvalue; }, // use the `${QUEUE_NAME}.QUEUE_EVENT` scheme async "enqueue.async.active"(ctx: Context<{ id: Number }>) { diff --git a/services/library.service.ts b/services/library.service.ts index 4aa3dab..ce198d8 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -80,14 +80,19 @@ export default class ImportService extends Service { }, walkFolders: { rest: "POST /walkFolders", - params: { - basePathToWalk: "string", - }, - async handler(ctx: Context<{ basePathToWalk: string }>) { + params: {}, + async handler( + ctx: Context<{ + basePathToWalk: string; + extensions: string[]; + }> + ) { + console.log(ctx.params); return await walkFolder(ctx.params.basePathToWalk, [ ".cbz", ".cbr", ".cb7", + ...ctx.params.extensions, ]); }, }, From 4cdb11fcbd6f701b61b491358aade738d48cdd85 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Mon, 8 Jan 2024 16:40:12 -0500 Subject: [PATCH 3/3] =?UTF-8?q?=E2=9D=8C=20Cleaned=20the=20console.logs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/jobqueue.service.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/jobqueue.service.ts b/services/jobqueue.service.ts index 190b330..5e1cc08 100644 --- a/services/jobqueue.service.ts +++ b/services/jobqueue.service.ts @@ -57,7 +57,6 @@ export default class JobQueueService extends Service { handler: async ( ctx: Context<{ queueName: string; description: string }> ) => { - console.log(ctx.params); const { queueName, description } = ctx.params; // Enqueue the job const job = await this.localQueue( @@ -85,7 +84,6 @@ export default class JobQueueService extends Service { console.log( `Recieved Job ID ${ctx.locals.job.id}, processing...` ); - console.log(ctx.params); // 1. De-structure the job params const { fileObject } = ctx.locals.job.data.params; @@ -315,7 +313,6 @@ export default class JobQueueService extends Service { filePath, options ); - if (Array.isArray(result) && result.length !== 0) { // Get the containing directory of the uncompressed archive const directoryPath = path.dirname(result[0]);