frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

🔧 Refactoring the import queue process

Browse Source
This commit is contained in:
Rishi Ghan
2022-04-18 15:22:34 -07:00
parent 38a80e41b3
commit 4c071f2fc7
2 changed files with 74 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
services/importqueue.service.ts
View File

@@ -34,12 +34,7 @@ SOFTWARE.
"use strict";
import { refineQuery } from "filename-parser";
import {
Context,
Service,
ServiceBroker,
ServiceSchema
} from "moleculer";
import { Context, Service, ServiceBroker, ServiceSchema } from "moleculer";
import BullMQMixin, { SandboxedJob } from "moleculer-bull";
import { DbMixin } from "../mixins/db.mixin";
import Comic from "../models/comic.model";
@@ -56,18 +51,14 @@ export default class QueueService extends Service {
schema: ServiceSchema<{}> = { name: "importqueue" }
) {
super(broker);
console.log(this.io);
this.parseServiceSchema({
name: "importqueue",
mixins: [
BullMQMixin(REDIS_URI),
DbMixin("comics", Comic),
],
mixins: [BullMQMixin(REDIS_URI), DbMixin("comics", Comic)],
settings: {},
hooks: {},
queues: {
"process.import": {
concurrency: 30,
concurrency: 1,
async process(job: SandboxedJob) {
console.info("New job received!", job.data);
console.info(`Processing queue...`);
@@ -130,11 +121,12 @@ export default class QueueService extends Service {
},
}
);
return Promise.resolve({
console.log("JAMA", dbImportResult);
return {
dbImportResult,
id: job.id,
worker: process.pid,
});
};
},
},
},
@@ -195,12 +187,11 @@ export default class QueueService extends Service {
await this.getQueue("process.import").on(
"completed",
async (job, res) => {
await this.broker.call('socket.broadcast', {
namespace: '/', //optional
await this.broker.call("socket.broadcast", {
namespace: "/", //optional
event: "action",
args: [{ type: "LS_COVER_EXTRACTED", result: res }], //optional
})
});
console.info(`Job with the id '${job.id}' completed.`);
}
);

124
services/library.service.ts
View File

@@ -101,7 +101,7 @@ export default class ImportService extends Service {
klaw(path.resolve(COMICS_DIRECTORY))
// 1.1 Filter on .cb* extensions
.pipe(
through2.obj(function (item, enc, next) {
through2.obj(function(item, enc, next) {
let fileExtension = path.extname(item.path);
if (
[".cbz", ".cbr", ".cb7"].includes(
@@ -123,18 +123,20 @@ export default class ImportService extends Service {
let comicExists = await Comic.exists({
"rawFileDetails.name": `${path.basename(
item.path,
path.extname(item.path),
path.extname(item.path)
)}`,
});
if (!comicExists) {
// 2. Send the extraction job to the queue
await broker.call("importqueue.processImport", {
fileObject: {
filePath: item.path,
fileSize: item.stats.size,
},
});
await broker.call(
"importqueue.processImport",
{
fileObject: {
filePath: item.path,
fileSize: item.stats.size,
},
}
);
} else {
console.log(
"Comic already exists in the library."
@@ -169,45 +171,34 @@ export default class ImportService extends Service {
};
}>
) {
let volumeDetails;
const comicMetadata = ctx.params;
console.log(comicMetadata);
// When an issue is added from the search CV feature
if (
comicMetadata.sourcedMetadata.comicvine &&
!isNil(
comicMetadata.sourcedMetadata.comicvine.volume
)
) {
volumeDetails = await this.broker.call(
"comicvine.getVolumes",
{
volumeURI:
comicMetadata.sourcedMetadata.comicvine
.volume.api_detail_url,
}
);
comicMetadata.sourcedMetadata.comicvine.volumeInformation =
volumeDetails.results;
}
return await Comic.create(
ctx.params,
(error, data) => {
if (data) {
return data;
} else if (error) {
console.log("data", data);
console.log("error", error);
throw new Errors.MoleculerError(
"Failed to import comic book",
400,
"IMS_FAILED_COMIC_BOOK_IMPORT",
error
);
}
try {
let volumeDetails;
const comicMetadata = ctx.params;
console.log(JSON.stringify(comicMetadata, null, 4));
// When an issue is added from the search CV feature
if (
comicMetadata.sourcedMetadata.comicvine &&
!isNil(
comicMetadata.sourcedMetadata.comicvine
.volume
)
) {
volumeDetails = await this.broker.call(
"comicvine.getVolumes",
{
volumeURI:
comicMetadata.sourcedMetadata
.comicvine.volume
.api_detail_url,
}
);
comicMetadata.sourcedMetadata.comicvine.volumeInformation =
volumeDetails.results;
}
);
return await Comic.create(ctx.params);
} catch (error) {
console.log(error);
}
},
},
applyComicVineMetadata: {
@@ -487,9 +478,9 @@ export default class ImportService extends Service {
{
$match: {
"sourcedMetadata.comicvine.volumeInformation":
{
$gt: {},
},
{
$gt: {},
},
},
},
{
@@ -552,16 +543,29 @@ export default class ImportService extends Service {
.drop()
.then(async (data) => {
console.info(data);
const coversFolderDeleteResult = fsExtra.emptyDirSync(
path.resolve(`${USERDATA_DIRECTORY}/covers`)
);
const expandedFolderDeleteResult = fsExtra.emptyDirSync(
path.resolve(
`${USERDATA_DIRECTORY}/expanded`
)
);
const eSIndicesDeleteResult = await ctx.broker.call("search.deleteElasticSearchIndices", {});
return { data, coversFolderDeleteResult, expandedFolderDeleteResult, eSIndicesDeleteResult };
const coversFolderDeleteResult =
fsExtra.emptyDirSync(
path.resolve(
`${USERDATA_DIRECTORY}/covers`
)
);
const expandedFolderDeleteResult =
fsExtra.emptyDirSync(
path.resolve(
`${USERDATA_DIRECTORY}/expanded`
)
);
const eSIndicesDeleteResult =
await ctx.broker.call(
"search.deleteElasticSearchIndices",
{}
);
return {
data,
coversFolderDeleteResult,
expandedFolderDeleteResult,
eSIndicesDeleteResult,
};
})
.catch((error) => error);
},