🏗 Added a job for full archive extraction
This commit is contained in:
141
package-lock.json
generated
141
package-lock.json
generated
@@ -710,11 +710,11 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/code-frame": {
|
||||
"version": "7.22.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz",
|
||||
"integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==",
|
||||
"version": "7.23.5",
|
||||
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz",
|
||||
"integrity": "sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==",
|
||||
"dependencies": {
|
||||
"@babel/highlight": "^7.22.13",
|
||||
"@babel/highlight": "^7.23.4",
|
||||
"chalk": "^2.4.2"
|
||||
},
|
||||
"engines": {
|
||||
@@ -840,12 +840,12 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/generator": {
|
||||
"version": "7.22.10",
|
||||
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.22.10.tgz",
|
||||
"integrity": "sha512-79KIf7YiWjjdZ81JnLujDRApWtl7BxTqWD88+FFdQEIOG8LJ0etDOM7CXuIgGJa55sGOwZVwuEsaLEm0PJ5/+A==",
|
||||
"version": "7.23.6",
|
||||
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.6.tgz",
|
||||
"integrity": "sha512-qrSfCYxYQB5owCmGLbl8XRpX1ytXlpueOb0N0UmQwA073KZxejgQTzAmJezxvpwQD9uGtK2shHdi55QT+MbjIw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@babel/types": "^7.22.10",
|
||||
"@babel/types": "^7.23.6",
|
||||
"@jridgewell/gen-mapping": "^0.3.2",
|
||||
"@jridgewell/trace-mapping": "^0.3.17",
|
||||
"jsesc": "^2.5.1"
|
||||
@@ -892,22 +892,22 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helper-environment-visitor": {
|
||||
"version": "7.22.5",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz",
|
||||
"integrity": "sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q==",
|
||||
"version": "7.22.20",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz",
|
||||
"integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helper-function-name": {
|
||||
"version": "7.22.5",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz",
|
||||
"integrity": "sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ==",
|
||||
"version": "7.23.0",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz",
|
||||
"integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@babel/template": "^7.22.5",
|
||||
"@babel/types": "^7.22.5"
|
||||
"@babel/template": "^7.22.15",
|
||||
"@babel/types": "^7.23.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
@@ -990,18 +990,18 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helper-string-parser": {
|
||||
"version": "7.22.5",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz",
|
||||
"integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==",
|
||||
"version": "7.23.4",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.23.4.tgz",
|
||||
"integrity": "sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/helper-validator-identifier": {
|
||||
"version": "7.22.5",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz",
|
||||
"integrity": "sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ==",
|
||||
"version": "7.22.20",
|
||||
"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz",
|
||||
"integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==",
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
@@ -1030,11 +1030,11 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/highlight": {
|
||||
"version": "7.22.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.13.tgz",
|
||||
"integrity": "sha512-C/BaXcnnvBCmHTpz/VGZ8jgtE2aYlW4hxDhseJAWZb7gqGM/qtCK6iZUb0TyKFf7BOUsBH7Q7fkRsDRhg1XklQ==",
|
||||
"version": "7.23.4",
|
||||
"resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz",
|
||||
"integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==",
|
||||
"dependencies": {
|
||||
"@babel/helper-validator-identifier": "^7.22.5",
|
||||
"@babel/helper-validator-identifier": "^7.22.20",
|
||||
"chalk": "^2.4.2",
|
||||
"js-tokens": "^4.0.0"
|
||||
},
|
||||
@@ -1107,9 +1107,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/parser": {
|
||||
"version": "7.22.13",
|
||||
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.13.tgz",
|
||||
"integrity": "sha512-3l6+4YOvc9wx7VlCSw4yQfcBo01ECA8TicQfbnCPuCEpRQrf+gTUyGdxNw+pyTUyywp6JRD1w0YQs9TpBXYlkw==",
|
||||
"version": "7.23.6",
|
||||
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.6.tgz",
|
||||
"integrity": "sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ==",
|
||||
"dev": true,
|
||||
"bin": {
|
||||
"parser": "bin/babel-parser.js"
|
||||
@@ -1319,34 +1319,34 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/template": {
|
||||
"version": "7.22.5",
|
||||
"resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.5.tgz",
|
||||
"integrity": "sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw==",
|
||||
"version": "7.22.15",
|
||||
"resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz",
|
||||
"integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.22.5",
|
||||
"@babel/parser": "^7.22.5",
|
||||
"@babel/types": "^7.22.5"
|
||||
"@babel/code-frame": "^7.22.13",
|
||||
"@babel/parser": "^7.22.15",
|
||||
"@babel/types": "^7.22.15"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/traverse": {
|
||||
"version": "7.22.11",
|
||||
"resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.22.11.tgz",
|
||||
"integrity": "sha512-mzAenteTfomcB7mfPtyi+4oe5BZ6MXxWcn4CX+h4IRJ+OOGXBrWU6jDQavkQI9Vuc5P+donFabBfFCcmWka9lQ==",
|
||||
"version": "7.23.7",
|
||||
"resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.7.tgz",
|
||||
"integrity": "sha512-tY3mM8rH9jM0YHFGyfC0/xf+SB5eKUu7HPj7/k3fpi9dAlsMc5YbQvDi0Sh2QTPXqMhyaAtzAr807TIyfQrmyg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.22.10",
|
||||
"@babel/generator": "^7.22.10",
|
||||
"@babel/helper-environment-visitor": "^7.22.5",
|
||||
"@babel/helper-function-name": "^7.22.5",
|
||||
"@babel/code-frame": "^7.23.5",
|
||||
"@babel/generator": "^7.23.6",
|
||||
"@babel/helper-environment-visitor": "^7.22.20",
|
||||
"@babel/helper-function-name": "^7.23.0",
|
||||
"@babel/helper-hoist-variables": "^7.22.5",
|
||||
"@babel/helper-split-export-declaration": "^7.22.6",
|
||||
"@babel/parser": "^7.22.11",
|
||||
"@babel/types": "^7.22.11",
|
||||
"debug": "^4.1.0",
|
||||
"@babel/parser": "^7.23.6",
|
||||
"@babel/types": "^7.23.6",
|
||||
"debug": "^4.3.1",
|
||||
"globals": "^11.1.0"
|
||||
},
|
||||
"engines": {
|
||||
@@ -1363,13 +1363,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/types": {
|
||||
"version": "7.22.11",
|
||||
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.22.11.tgz",
|
||||
"integrity": "sha512-siazHiGuZRz9aB9NpHy9GOs9xiQPKnMzgdr493iI1M67vRXpnEq8ZOOKzezC5q7zwuQ6sDhdSp4SD9ixKSqKZg==",
|
||||
"version": "7.23.6",
|
||||
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.6.tgz",
|
||||
"integrity": "sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@babel/helper-string-parser": "^7.22.5",
|
||||
"@babel/helper-validator-identifier": "^7.22.5",
|
||||
"@babel/helper-string-parser": "^7.23.4",
|
||||
"@babel/helper-validator-identifier": "^7.22.20",
|
||||
"to-fast-properties": "^2.0.0"
|
||||
},
|
||||
"engines": {
|
||||
@@ -4272,17 +4272,6 @@
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/busboy": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
|
||||
"integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
|
||||
"dependencies": {
|
||||
"streamsearch": "^1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/bytes": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
|
||||
@@ -9934,9 +9923,9 @@
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
|
||||
},
|
||||
"node_modules/msgpackr": {
|
||||
"version": "1.9.7",
|
||||
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.7.tgz",
|
||||
"integrity": "sha512-baUNaLvKQvVhzfWTNO07njwbZK1Lxjtb0P1JL6/EhXdLTHzR57/mZqqJC39TtQKvOmkJA4pcejS4dbk7BDgLLA==",
|
||||
"version": "1.10.1",
|
||||
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.1.tgz",
|
||||
"integrity": "sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==",
|
||||
"optionalDependencies": {
|
||||
"msgpackr-extract": "^3.0.2"
|
||||
}
|
||||
@@ -14794,14 +14783,6 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/streamsearch": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
|
||||
"integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/string_decoder": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
|
||||
@@ -15668,16 +15649,24 @@
|
||||
}
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "5.23.0",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-5.23.0.tgz",
|
||||
"integrity": "sha512-1D7w+fvRsqlQ9GscLBwcAJinqcZGHUKjbOmXdlE/v8BvEGXjeWAax+341q44EuTcHXXnfyKNbKRq4Lg7OzhMmg==",
|
||||
"version": "5.28.2",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz",
|
||||
"integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==",
|
||||
"dependencies": {
|
||||
"busboy": "^1.6.0"
|
||||
"@fastify/busboy": "^2.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.0"
|
||||
}
|
||||
},
|
||||
"node_modules/undici/node_modules/@fastify/busboy": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
|
||||
"integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/unit-compare": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/unit-compare/-/unit-compare-1.0.1.tgz",
|
||||
|
||||
@@ -2,7 +2,7 @@ import { Context, Service, ServiceBroker } from "moleculer";
|
||||
import JobResult from "../models/jobresult.model";
|
||||
import { refineQuery } from "filename-parser";
|
||||
import BullMqMixin from "moleculer-bullmq";
|
||||
import { extractFromArchive } from "../utils/uncompression.utils";
|
||||
import { extractFromArchive, uncompressEntireArchive } from "../utils/uncompression.utils";
|
||||
import { isNil, isUndefined } from "lodash";
|
||||
import { pubClient } from "../config/redis.config";
|
||||
|
||||
@@ -47,17 +47,15 @@ export default class JobQueueService extends Service {
|
||||
enqueue: {
|
||||
queue: true,
|
||||
rest: "/GET enqueue",
|
||||
handler: async (ctx: Context<{}>) => {
|
||||
handler: async (ctx: Context<{ queueName: string; description: string }>) => {
|
||||
console.log(ctx.params);
|
||||
const { queueName, description } = ctx.params;
|
||||
// Enqueue the job
|
||||
const job = await this.localQueue(
|
||||
ctx,
|
||||
"enqueue.async",
|
||||
ctx.params,
|
||||
{
|
||||
priority: 10,
|
||||
}
|
||||
);
|
||||
const job = await this.localQueue(ctx, queueName, ctx.params, {
|
||||
priority: 10,
|
||||
});
|
||||
console.log(`Job ${job.id} enqueued`);
|
||||
console.log(`${description}`);
|
||||
|
||||
return job.id;
|
||||
},
|
||||
@@ -70,17 +68,13 @@ export default class JobQueueService extends Service {
|
||||
}>
|
||||
) => {
|
||||
try {
|
||||
console.log(
|
||||
`Recieved Job ID ${ctx.locals.job.id}, processing...`
|
||||
);
|
||||
console.log(`Recieved Job ID ${ctx.locals.job.id}, processing...`);
|
||||
console.log(ctx.params);
|
||||
// 1. De-structure the job params
|
||||
const { fileObject } = ctx.locals.job.data.params;
|
||||
|
||||
// 2. Extract metadata from the archive
|
||||
const result = await extractFromArchive(
|
||||
fileObject.filePath
|
||||
);
|
||||
const result = await extractFromArchive(fileObject.filePath);
|
||||
const {
|
||||
name,
|
||||
filePath,
|
||||
@@ -93,9 +87,7 @@ export default class JobQueueService extends Service {
|
||||
} = result;
|
||||
|
||||
// 3a. Infer any issue-related metadata from the filename
|
||||
const { inferredIssueDetails } = refineQuery(
|
||||
result.name
|
||||
);
|
||||
const { inferredIssueDetails } = refineQuery(result.name);
|
||||
console.log(
|
||||
"Issue metadata inferred: ",
|
||||
JSON.stringify(inferredIssueDetails, null, 2)
|
||||
@@ -135,8 +127,7 @@ export default class JobQueueService extends Service {
|
||||
// "acquisition.directconnect.downloads": [],
|
||||
|
||||
// mark the metadata source
|
||||
"acquisition.source.name":
|
||||
ctx.locals.job.data.params.sourcedFrom,
|
||||
"acquisition.source.name": ctx.locals.job.data.params.sourcedFrom,
|
||||
};
|
||||
|
||||
// 3c. Add the bundleId, if present to the payload
|
||||
@@ -147,13 +138,8 @@ export default class JobQueueService extends Service {
|
||||
|
||||
// 3d. Add the sourcedMetadata, if present
|
||||
if (
|
||||
!isNil(
|
||||
ctx.locals.job.data.params.sourcedMetadata
|
||||
) &&
|
||||
!isUndefined(
|
||||
ctx.locals.job.data.params.sourcedMetadata
|
||||
.comicvine
|
||||
)
|
||||
!isNil(ctx.locals.job.data.params.sourcedMetadata) &&
|
||||
!isUndefined(ctx.locals.job.data.params.sourcedMetadata.comicvine)
|
||||
) {
|
||||
Object.assign(
|
||||
payload.sourcedMetadata,
|
||||
@@ -162,15 +148,11 @@ export default class JobQueueService extends Service {
|
||||
}
|
||||
|
||||
// 4. write to mongo
|
||||
const importResult = await this.broker.call(
|
||||
"library.rawImportToDB",
|
||||
{
|
||||
importType:
|
||||
ctx.locals.job.data.params.importType,
|
||||
bundleId,
|
||||
payload,
|
||||
}
|
||||
);
|
||||
const importResult = await this.broker.call("library.rawImportToDB", {
|
||||
importType: ctx.locals.job.data.params.importType,
|
||||
bundleId,
|
||||
payload,
|
||||
});
|
||||
return {
|
||||
data: {
|
||||
importResult,
|
||||
@@ -182,14 +164,9 @@ export default class JobQueueService extends Service {
|
||||
console.error(
|
||||
`An error occurred processing Job ID ${ctx.locals.job.id}`
|
||||
);
|
||||
throw new MoleculerError(
|
||||
error,
|
||||
500,
|
||||
"IMPORT_JOB_ERROR",
|
||||
{
|
||||
data: ctx.params.sessionId,
|
||||
}
|
||||
);
|
||||
throw new MoleculerError(error, 500, "IMPORT_JOB_ERROR", {
|
||||
data: ctx.params.sessionId,
|
||||
});
|
||||
}
|
||||
},
|
||||
},
|
||||
@@ -217,8 +194,7 @@ export default class JobQueueService extends Service {
|
||||
statuses: {
|
||||
$push: {
|
||||
status: "$_id.status",
|
||||
earliestTimestamp:
|
||||
"$earliestTimestamp",
|
||||
earliestTimestamp: "$earliestTimestamp",
|
||||
count: "$count",
|
||||
},
|
||||
},
|
||||
@@ -238,10 +214,7 @@ export default class JobQueueService extends Service {
|
||||
{
|
||||
$cond: [
|
||||
{
|
||||
$eq: [
|
||||
"$$this.status",
|
||||
"completed",
|
||||
],
|
||||
$eq: ["$$this.status", "completed"],
|
||||
},
|
||||
"$$this.count",
|
||||
0,
|
||||
@@ -261,10 +234,7 @@ export default class JobQueueService extends Service {
|
||||
{
|
||||
$cond: [
|
||||
{
|
||||
$eq: [
|
||||
"$$this.status",
|
||||
"failed",
|
||||
],
|
||||
$eq: ["$$this.status", "failed"],
|
||||
},
|
||||
"$$this.count",
|
||||
0,
|
||||
@@ -282,9 +252,24 @@ export default class JobQueueService extends Service {
|
||||
]);
|
||||
},
|
||||
},
|
||||
"uncompressFullArchive.async": {
|
||||
rest: "POST /uncompressFullArchive",
|
||||
handler: async (ctx: Context<{ filePath: string; options: any }>) => {
|
||||
const { filePath, options } = ctx.params;
|
||||
console.log("asd", filePath);
|
||||
// 2. Extract metadata from the archive
|
||||
return await uncompressEntireArchive(filePath, options);
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
events: {
|
||||
async "uncompressFullArchive.async.active"(ctx: Context<{ id: number }>) {
|
||||
console.log(`Uncompression Job ID ${ctx.params.id} is set to active.`);
|
||||
},
|
||||
async "uncompressFullArchive.async.completed"(ctx: Context<{ id: number }>) {
|
||||
console.log(`Uncompression Job ID ${ctx.params.id} completed.`);
|
||||
},
|
||||
// use the `${QUEUE_NAME}.QUEUE_EVENT` scheme
|
||||
async "enqueue.async.active"(ctx: Context<{ id: Number }>) {
|
||||
console.log(`Job ID ${ctx.params.id} is set to active.`);
|
||||
@@ -307,9 +292,7 @@ export default class JobQueueService extends Service {
|
||||
// 2. Increment the completed job counter
|
||||
await pubClient.incr("completedJobCount");
|
||||
// 3. Fetch the completed job count for the final payload to be sent to the client
|
||||
const completedJobCount = await pubClient.get(
|
||||
"completedJobCount"
|
||||
);
|
||||
const completedJobCount = await pubClient.get("completedJobCount");
|
||||
// 4. Emit the LS_COVER_EXTRACTED event with the necessary details
|
||||
await this.broker.call("socket.broadcast", {
|
||||
namespace: "/",
|
||||
@@ -336,9 +319,7 @@ export default class JobQueueService extends Service {
|
||||
async "enqueue.async.failed"(ctx) {
|
||||
const job = await this.job(ctx.params.id);
|
||||
await pubClient.incr("failedJobCount");
|
||||
const failedJobCount = await pubClient.get(
|
||||
"failedJobCount"
|
||||
);
|
||||
const failedJobCount = await pubClient.get("failedJobCount");
|
||||
|
||||
await JobResult.create({
|
||||
id: ctx.params.id,
|
||||
|
||||
@@ -33,13 +33,7 @@ SOFTWARE.
|
||||
|
||||
"use strict";
|
||||
import { isNil } from "lodash";
|
||||
import {
|
||||
Context,
|
||||
Service,
|
||||
ServiceBroker,
|
||||
ServiceSchema,
|
||||
Errors,
|
||||
} from "moleculer";
|
||||
import { Context, Service, ServiceBroker, ServiceSchema, Errors } from "moleculer";
|
||||
import { DbMixin } from "../mixins/db.mixin";
|
||||
import Comic from "../models/comic.model";
|
||||
import { walkFolder, getSizeOfDirectory } from "../utils/file.utils";
|
||||
@@ -101,9 +95,7 @@ export default class ImportService extends Service {
|
||||
uncompressFullArchive: {
|
||||
rest: "POST /uncompressFullArchive",
|
||||
params: {},
|
||||
handler: async (
|
||||
ctx: Context<{ filePath: string; options: any }>
|
||||
) => {
|
||||
handler: async (ctx: Context<{ filePath: string; options: any }>) => {
|
||||
await broker.call("importqueue.uncompressResize", {
|
||||
filePath: ctx.params.filePath,
|
||||
options: ctx.params.options,
|
||||
@@ -121,8 +113,7 @@ export default class ImportService extends Service {
|
||||
});
|
||||
// Determine source where the comic was added from
|
||||
// and gather identifying information about it
|
||||
const sourceName =
|
||||
referenceComicObject[0].acquisition.source.name;
|
||||
const sourceName = referenceComicObject[0].acquisition.source.name;
|
||||
const { sourcedMetadata } = referenceComicObject[0];
|
||||
|
||||
const filePath = `${COMICS_DIRECTORY}/${ctx.params.bundle.data.name}`;
|
||||
@@ -166,14 +157,8 @@ export default class ImportService extends Service {
|
||||
// 1.1 Filter on .cb* extensions
|
||||
.pipe(
|
||||
through2.obj(function (item, enc, next) {
|
||||
let fileExtension = path.extname(
|
||||
item.path
|
||||
);
|
||||
if (
|
||||
[".cbz", ".cbr", ".cb7"].includes(
|
||||
fileExtension
|
||||
)
|
||||
) {
|
||||
let fileExtension = path.extname(item.path);
|
||||
if ([".cbz", ".cbr", ".cb7"].includes(fileExtension)) {
|
||||
this.push(item);
|
||||
}
|
||||
next();
|
||||
@@ -182,10 +167,7 @@ export default class ImportService extends Service {
|
||||
// 1.2 Pipe filtered results to the next step
|
||||
// Enqueue the job in the queue
|
||||
.on("data", async (item) => {
|
||||
console.info(
|
||||
"Found a file at path: %s",
|
||||
item.path
|
||||
);
|
||||
console.info("Found a file at path: %s", item.path);
|
||||
let comicExists = await Comic.exists({
|
||||
"rawFileDetails.name": `${path.basename(
|
||||
item.path,
|
||||
@@ -194,14 +176,8 @@ export default class ImportService extends Service {
|
||||
});
|
||||
if (!comicExists) {
|
||||
// 2.1 Reset the job counters in Redis
|
||||
await pubClient.set(
|
||||
"completedJobCount",
|
||||
0
|
||||
);
|
||||
await pubClient.set(
|
||||
"failedJobCount",
|
||||
0
|
||||
);
|
||||
await pubClient.set("completedJobCount", 0);
|
||||
await pubClient.set("failedJobCount", 0);
|
||||
// 2.2 Send the extraction job to the queue
|
||||
this.broker.call("jobqueue.enqueue", {
|
||||
fileObject: {
|
||||
@@ -210,11 +186,10 @@ export default class ImportService extends Service {
|
||||
},
|
||||
sessionId,
|
||||
importType: "new",
|
||||
queueName: "enqueue.async",
|
||||
});
|
||||
} else {
|
||||
console.log(
|
||||
"Comic already exists in the library."
|
||||
);
|
||||
console.log("Comic already exists in the library.");
|
||||
}
|
||||
})
|
||||
.on("end", () => {
|
||||
@@ -266,28 +241,19 @@ export default class ImportService extends Service {
|
||||
// we solicit volume information and add that to mongo
|
||||
if (
|
||||
comicMetadata.sourcedMetadata.comicvine &&
|
||||
!isNil(
|
||||
comicMetadata.sourcedMetadata.comicvine
|
||||
.volume
|
||||
)
|
||||
!isNil(comicMetadata.sourcedMetadata.comicvine.volume)
|
||||
) {
|
||||
volumeDetails = await this.broker.call(
|
||||
"comicvine.getVolumes",
|
||||
{
|
||||
volumeURI:
|
||||
comicMetadata.sourcedMetadata
|
||||
.comicvine.volume
|
||||
.api_detail_url,
|
||||
}
|
||||
);
|
||||
volumeDetails = await this.broker.call("comicvine.getVolumes", {
|
||||
volumeURI:
|
||||
comicMetadata.sourcedMetadata.comicvine.volume
|
||||
.api_detail_url,
|
||||
});
|
||||
comicMetadata.sourcedMetadata.comicvine.volumeInformation =
|
||||
volumeDetails.results;
|
||||
}
|
||||
|
||||
console.log("Saving to Mongo...");
|
||||
console.log(
|
||||
`Import type: [${ctx.params.importType}]`
|
||||
);
|
||||
console.log(`Import type: [${ctx.params.importType}]`);
|
||||
switch (ctx.params.importType) {
|
||||
case "new":
|
||||
return await Comic.create(comicMetadata);
|
||||
@@ -308,10 +274,7 @@ export default class ImportService extends Service {
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
throw new Errors.MoleculerError(
|
||||
"Import failed.",
|
||||
500
|
||||
);
|
||||
throw new Errors.MoleculerError("Import failed.", 500);
|
||||
}
|
||||
},
|
||||
},
|
||||
@@ -329,9 +292,7 @@ export default class ImportService extends Service {
|
||||
) {
|
||||
// 1. Find mongo object by id
|
||||
// 2. Import payload into sourcedMetadata.comicvine
|
||||
const comicObjectId = new ObjectId(
|
||||
ctx.params.comicObjectId
|
||||
);
|
||||
const comicObjectId = new ObjectId(ctx.params.comicObjectId);
|
||||
|
||||
return new Promise(async (resolve, reject) => {
|
||||
let volumeDetails = {};
|
||||
@@ -340,18 +301,15 @@ export default class ImportService extends Service {
|
||||
const volumeDetails = await this.broker.call(
|
||||
"comicvine.getVolumes",
|
||||
{
|
||||
volumeURI:
|
||||
matchedResult.volume.api_detail_url,
|
||||
volumeURI: matchedResult.volume.api_detail_url,
|
||||
}
|
||||
);
|
||||
matchedResult.volumeInformation =
|
||||
volumeDetails.results;
|
||||
matchedResult.volumeInformation = volumeDetails.results;
|
||||
Comic.findByIdAndUpdate(
|
||||
comicObjectId,
|
||||
{
|
||||
$set: {
|
||||
"sourcedMetadata.comicvine":
|
||||
matchedResult,
|
||||
"sourcedMetadata.comicvine": matchedResult,
|
||||
},
|
||||
},
|
||||
{ new: true },
|
||||
@@ -382,9 +340,7 @@ export default class ImportService extends Service {
|
||||
}>
|
||||
) {
|
||||
console.log(JSON.stringify(ctx.params, null, 2));
|
||||
const comicObjectId = new ObjectId(
|
||||
ctx.params.comicObjectId
|
||||
);
|
||||
const comicObjectId = new ObjectId(ctx.params.comicObjectId);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
Comic.findByIdAndUpdate(
|
||||
@@ -439,9 +395,7 @@ export default class ImportService extends Service {
|
||||
params: { ids: "array" },
|
||||
handler: async (ctx: Context<{ ids: [string] }>) => {
|
||||
console.log(ctx.params.ids);
|
||||
const queryIds = ctx.params.ids.map(
|
||||
(id) => new ObjectId(id)
|
||||
);
|
||||
const queryIds = ctx.params.ids.map((id) => new ObjectId(id));
|
||||
return await Comic.find({
|
||||
_id: {
|
||||
$in: queryIds,
|
||||
@@ -457,8 +411,7 @@ export default class ImportService extends Service {
|
||||
const volumes = await Comic.aggregate([
|
||||
{
|
||||
$project: {
|
||||
volumeInfo:
|
||||
"$sourcedMetadata.comicvine.volumeInformation",
|
||||
volumeInfo: "$sourcedMetadata.comicvine.volumeInformation",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -504,52 +457,46 @@ export default class ImportService extends Service {
|
||||
const { queryObjects } = ctx.params;
|
||||
// construct the query for ElasticSearch
|
||||
let elasticSearchQuery = {};
|
||||
const elasticSearchQueries = queryObjects.map(
|
||||
(queryObject) => {
|
||||
console.log("Volume: ", queryObject.volumeName);
|
||||
console.log("Issue: ", queryObject.issueName);
|
||||
if (queryObject.issueName === null) {
|
||||
queryObject.issueName = "";
|
||||
}
|
||||
if (queryObject.volumeName === null) {
|
||||
queryObject.volumeName = "";
|
||||
}
|
||||
elasticSearchQuery = {
|
||||
bool: {
|
||||
must: [
|
||||
{
|
||||
match_phrase: {
|
||||
"rawFileDetails.name":
|
||||
queryObject.volumeName,
|
||||
},
|
||||
},
|
||||
{
|
||||
term: {
|
||||
"inferredMetadata.issue.number":
|
||||
parseInt(
|
||||
queryObject.issueNumber,
|
||||
10
|
||||
),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
return [
|
||||
{
|
||||
index: "comics",
|
||||
search_type: "dfs_query_then_fetch",
|
||||
},
|
||||
{
|
||||
query: elasticSearchQuery,
|
||||
},
|
||||
];
|
||||
const elasticSearchQueries = queryObjects.map((queryObject) => {
|
||||
console.log("Volume: ", queryObject.volumeName);
|
||||
console.log("Issue: ", queryObject.issueName);
|
||||
if (queryObject.issueName === null) {
|
||||
queryObject.issueName = "";
|
||||
}
|
||||
);
|
||||
console.log(
|
||||
JSON.stringify(elasticSearchQueries, null, 2)
|
||||
);
|
||||
if (queryObject.volumeName === null) {
|
||||
queryObject.volumeName = "";
|
||||
}
|
||||
elasticSearchQuery = {
|
||||
bool: {
|
||||
must: [
|
||||
{
|
||||
match_phrase: {
|
||||
"rawFileDetails.name": queryObject.volumeName,
|
||||
},
|
||||
},
|
||||
{
|
||||
term: {
|
||||
"inferredMetadata.issue.number": parseInt(
|
||||
queryObject.issueNumber,
|
||||
10
|
||||
),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
return [
|
||||
{
|
||||
index: "comics",
|
||||
search_type: "dfs_query_then_fetch",
|
||||
},
|
||||
{
|
||||
query: elasticSearchQuery,
|
||||
},
|
||||
];
|
||||
});
|
||||
console.log(JSON.stringify(elasticSearchQueries, null, 2));
|
||||
|
||||
return await ctx.broker.call("search.searchComic", {
|
||||
elasticSearchQueries,
|
||||
@@ -562,10 +509,11 @@ export default class ImportService extends Service {
|
||||
rest: "GET /libraryStatistics",
|
||||
params: {},
|
||||
handler: async (ctx: Context<{}>) => {
|
||||
const comicDirectorySize = await getSizeOfDirectory(
|
||||
COMICS_DIRECTORY,
|
||||
[".cbz", ".cbr", ".cb7"]
|
||||
);
|
||||
const comicDirectorySize = await getSizeOfDirectory(COMICS_DIRECTORY, [
|
||||
".cbz",
|
||||
".cbr",
|
||||
".cb7",
|
||||
]);
|
||||
const totalCount = await Comic.countDocuments({});
|
||||
const statistics = await Comic.aggregate([
|
||||
{
|
||||
@@ -574,11 +522,7 @@ export default class ImportService extends Service {
|
||||
{
|
||||
$match: {
|
||||
"rawFileDetails.extension": {
|
||||
$in: [
|
||||
".cbr",
|
||||
".cbz",
|
||||
".cb7",
|
||||
],
|
||||
$in: [".cbr", ".cbz", ".cb7"],
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -592,10 +536,9 @@ export default class ImportService extends Service {
|
||||
issues: [
|
||||
{
|
||||
$match: {
|
||||
"sourcedMetadata.comicvine.volumeInformation":
|
||||
{
|
||||
$gt: {},
|
||||
},
|
||||
"sourcedMetadata.comicvine.volumeInformation": {
|
||||
$gt: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -658,23 +601,16 @@ export default class ImportService extends Service {
|
||||
.drop()
|
||||
.then(async (data) => {
|
||||
console.info(data);
|
||||
const coversFolderDeleteResult =
|
||||
fsExtra.emptyDirSync(
|
||||
path.resolve(
|
||||
`${USERDATA_DIRECTORY}/covers`
|
||||
)
|
||||
);
|
||||
const expandedFolderDeleteResult =
|
||||
fsExtra.emptyDirSync(
|
||||
path.resolve(
|
||||
`${USERDATA_DIRECTORY}/expanded`
|
||||
)
|
||||
);
|
||||
const eSIndicesDeleteResult =
|
||||
await ctx.broker.call(
|
||||
"search.deleteElasticSearchIndices",
|
||||
{}
|
||||
);
|
||||
const coversFolderDeleteResult = fsExtra.emptyDirSync(
|
||||
path.resolve(`${USERDATA_DIRECTORY}/covers`)
|
||||
);
|
||||
const expandedFolderDeleteResult = fsExtra.emptyDirSync(
|
||||
path.resolve(`${USERDATA_DIRECTORY}/expanded`)
|
||||
);
|
||||
const eSIndicesDeleteResult = await ctx.broker.call(
|
||||
"search.deleteElasticSearchIndices",
|
||||
{}
|
||||
);
|
||||
return {
|
||||
data,
|
||||
coversFolderDeleteResult,
|
||||
|
||||
@@ -74,15 +74,14 @@ const errors = [];
|
||||
*/
|
||||
export const extractComicInfoXMLFromRar = async (
|
||||
filePath: string,
|
||||
mimeType: string,
|
||||
mimeType: string
|
||||
): Promise<any> => {
|
||||
try {
|
||||
// Create the target directory
|
||||
const directoryOptions = {
|
||||
mode: 0o2775,
|
||||
};
|
||||
const { fileNameWithoutExtension, extension } =
|
||||
getFileConstituents(filePath);
|
||||
const { fileNameWithoutExtension, extension } = getFileConstituents(filePath);
|
||||
const targetDirectory = `${USERDATA_DIRECTORY}/covers/${sanitize(
|
||||
fileNameWithoutExtension
|
||||
)}`;
|
||||
@@ -93,17 +92,15 @@ export const extractComicInfoXMLFromRar = async (
|
||||
bin: `${UNRAR_BIN_PATH}`, // this will change depending on Docker base OS
|
||||
arguments: ["-v"],
|
||||
});
|
||||
const filesInArchive: [RarFile] = await new Promise(
|
||||
(resolve, reject) => {
|
||||
return archive.list((err, entries) => {
|
||||
if (err) {
|
||||
console.log(`DEBUG: ${JSON.stringify(err, null, 2)}`);
|
||||
reject(err);
|
||||
}
|
||||
resolve(entries);
|
||||
});
|
||||
}
|
||||
);
|
||||
const filesInArchive: [RarFile] = await new Promise((resolve, reject) => {
|
||||
return archive.list((err, entries) => {
|
||||
if (err) {
|
||||
console.log(`DEBUG: ${JSON.stringify(err, null, 2)}`);
|
||||
reject(err);
|
||||
}
|
||||
resolve(entries);
|
||||
});
|
||||
});
|
||||
|
||||
remove(filesInArchive, ({ type }) => type === "Directory");
|
||||
const comicInfoXML = remove(
|
||||
@@ -113,10 +110,7 @@ export const extractComicInfoXMLFromRar = async (
|
||||
|
||||
remove(
|
||||
filesInArchive,
|
||||
({ name }) =>
|
||||
!IMPORT_IMAGE_FILE_FORMATS.includes(
|
||||
path.extname(name).toLowerCase()
|
||||
)
|
||||
({ name }) => !IMPORT_IMAGE_FILE_FORMATS.includes(path.extname(name).toLowerCase())
|
||||
);
|
||||
const files = filesInArchive.sort((a, b) => {
|
||||
if (!isUndefined(a) && !isUndefined(b)) {
|
||||
@@ -129,12 +123,8 @@ export const extractComicInfoXMLFromRar = async (
|
||||
const comicInfoXMLFilePromise = new Promise((resolve, reject) => {
|
||||
let comicinfostring = "";
|
||||
if (!isUndefined(comicInfoXML[0])) {
|
||||
const comicInfoXMLFileName = path.basename(
|
||||
comicInfoXML[0].name
|
||||
);
|
||||
const writeStream = createWriteStream(
|
||||
`${targetDirectory}/${comicInfoXMLFileName}`
|
||||
);
|
||||
const comicInfoXMLFileName = path.basename(comicInfoXML[0].name);
|
||||
const writeStream = createWriteStream(`${targetDirectory}/${comicInfoXMLFileName}`);
|
||||
|
||||
archive.stream(comicInfoXML[0]["name"]).pipe(writeStream);
|
||||
writeStream.on("finish", async () => {
|
||||
@@ -147,11 +137,7 @@ export const extractComicInfoXMLFromRar = async (
|
||||
});
|
||||
readStream.on("error", (error) => reject(error));
|
||||
readStream.on("end", async () => {
|
||||
if (
|
||||
existsSync(
|
||||
`${targetDirectory}/${comicInfoXMLFileName}`
|
||||
)
|
||||
) {
|
||||
if (existsSync(`${targetDirectory}/${comicInfoXMLFileName}`)) {
|
||||
const comicInfoJSON = await convertXMLToJSON(
|
||||
comicinfostring.toString()
|
||||
);
|
||||
@@ -172,34 +158,29 @@ export const extractComicInfoXMLFromRar = async (
|
||||
const sharpStream = sharp().resize(275).toFormat("png");
|
||||
const coverExtractionStream = archive.stream(files[0].name);
|
||||
const resizeStream = coverExtractionStream.pipe(sharpStream);
|
||||
resizeStream.toFile(
|
||||
`${targetDirectory}/${coverFile}`,
|
||||
(err, info) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
checkFileExists(`${targetDirectory}/${coverFile}`).then(
|
||||
(bool) => {
|
||||
console.log(`${coverFile} exists: ${bool}`);
|
||||
// orchestrate result
|
||||
resolve({
|
||||
filePath,
|
||||
name: fileNameWithoutExtension,
|
||||
extension,
|
||||
containedIn: targetDirectory,
|
||||
fileSize: fse.statSync(filePath).size,
|
||||
mimeType,
|
||||
cover: {
|
||||
filePath: path.relative(
|
||||
process.cwd(),
|
||||
`${targetDirectory}/${coverFile}`
|
||||
),
|
||||
},
|
||||
});
|
||||
}
|
||||
);
|
||||
resizeStream.toFile(`${targetDirectory}/${coverFile}`, (err, info) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
);
|
||||
checkFileExists(`${targetDirectory}/${coverFile}`).then((bool) => {
|
||||
console.log(`${coverFile} exists: ${bool}`);
|
||||
// orchestrate result
|
||||
resolve({
|
||||
filePath,
|
||||
name: fileNameWithoutExtension,
|
||||
extension,
|
||||
containedIn: targetDirectory,
|
||||
fileSize: fse.statSync(filePath).size,
|
||||
mimeType,
|
||||
cover: {
|
||||
filePath: path.relative(
|
||||
process.cwd(),
|
||||
`${targetDirectory}/${coverFile}`
|
||||
),
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
return Promise.all([comicInfoXMLFilePromise, coverFilePromise]);
|
||||
@@ -210,15 +191,14 @@ export const extractComicInfoXMLFromRar = async (
|
||||
|
||||
export const extractComicInfoXMLFromZip = async (
|
||||
filePath: string,
|
||||
mimeType: string,
|
||||
mimeType: string
|
||||
): Promise<any> => {
|
||||
try {
|
||||
// Create the target directory
|
||||
const directoryOptions = {
|
||||
mode: 0o2775,
|
||||
};
|
||||
const { fileNameWithoutExtension, extension } =
|
||||
getFileConstituents(filePath);
|
||||
const { fileNameWithoutExtension, extension } = getFileConstituents(filePath);
|
||||
const targetDirectory = `${USERDATA_DIRECTORY}/covers/${sanitize(
|
||||
fileNameWithoutExtension
|
||||
)}`;
|
||||
@@ -237,10 +217,7 @@ export const extractComicInfoXMLFromZip = async (
|
||||
// only allow allowed image formats
|
||||
remove(
|
||||
filesFromArchive.files,
|
||||
({ name }) =>
|
||||
!IMPORT_IMAGE_FILE_FORMATS.includes(
|
||||
path.extname(name).toLowerCase()
|
||||
)
|
||||
({ name }) => !IMPORT_IMAGE_FILE_FORMATS.includes(path.extname(name).toLowerCase())
|
||||
);
|
||||
|
||||
// Natural sort
|
||||
@@ -261,13 +238,7 @@ export const extractComicInfoXMLFromZip = async (
|
||||
extractionTargets.push(filesToWriteToDisk.comicInfoXML);
|
||||
}
|
||||
// Extract the files.
|
||||
await p7zip.extract(
|
||||
filePath,
|
||||
targetDirectory,
|
||||
extractionTargets,
|
||||
"",
|
||||
false
|
||||
);
|
||||
await p7zip.extract(filePath, targetDirectory, extractionTargets, "", false);
|
||||
|
||||
// ComicInfoXML detection, parsing and conversion to JSON
|
||||
// Write ComicInfo.xml to disk
|
||||
@@ -275,26 +246,15 @@ export const extractComicInfoXMLFromZip = async (
|
||||
const comicInfoXMLPromise = new Promise((resolve, reject) => {
|
||||
if (
|
||||
!isNil(filesToWriteToDisk.comicInfoXML) &&
|
||||
existsSync(
|
||||
`${targetDirectory}/${path.basename(
|
||||
filesToWriteToDisk.comicInfoXML
|
||||
)}`
|
||||
)
|
||||
existsSync(`${targetDirectory}/${path.basename(filesToWriteToDisk.comicInfoXML)}`)
|
||||
) {
|
||||
let comicinfoString = "";
|
||||
const comicInfoXMLStream = createReadStream(
|
||||
`${targetDirectory}/${path.basename(
|
||||
filesToWriteToDisk.comicInfoXML
|
||||
)}`
|
||||
);
|
||||
comicInfoXMLStream.on(
|
||||
"data",
|
||||
(data) => (comicinfoString += data)
|
||||
`${targetDirectory}/${path.basename(filesToWriteToDisk.comicInfoXML)}`
|
||||
);
|
||||
comicInfoXMLStream.on("data", (data) => (comicinfoString += data));
|
||||
comicInfoXMLStream.on("end", async () => {
|
||||
const comicInfoJSON = await convertXMLToJSON(
|
||||
comicinfoString.toString()
|
||||
);
|
||||
const comicInfoJSON = await convertXMLToJSON(comicinfoString.toString());
|
||||
resolve({
|
||||
comicInfoJSON: comicInfoJSON.comicinfo,
|
||||
});
|
||||
@@ -314,9 +274,7 @@ export const extractComicInfoXMLFromZip = async (
|
||||
coverStream
|
||||
.pipe(sharpStream)
|
||||
.toFile(
|
||||
`${targetDirectory}/${path.basename(
|
||||
filesToWriteToDisk.coverFile
|
||||
)}`,
|
||||
`${targetDirectory}/${path.basename(filesToWriteToDisk.coverFile)}`,
|
||||
(err, info) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
@@ -365,13 +323,10 @@ export const extractFromArchive = async (filePath: string) => {
|
||||
return Object.assign({}, ...cbrResult);
|
||||
|
||||
default:
|
||||
console.error(
|
||||
"Error inferring filetype for comicinfo.xml extraction."
|
||||
);
|
||||
console.error("Error inferring filetype for comicinfo.xml extraction.");
|
||||
throw new MoleculerError({}, 500, "FILETYPE_INFERENCE_ERROR", {
|
||||
data: { message: "Cannot infer filetype."},
|
||||
data: { message: "Cannot infer filetype." },
|
||||
});
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
@@ -381,10 +336,7 @@ export const extractFromArchive = async (filePath: string) => {
|
||||
* @param {any} options
|
||||
* @returns {Promise} A promise containing the contents of the uncompressed archive.
|
||||
*/
|
||||
export const uncompressEntireArchive = async (
|
||||
filePath: string,
|
||||
options: any
|
||||
) => {
|
||||
export const uncompressEntireArchive = async (filePath: string, options: any) => {
|
||||
const mimeType = await getMimeType(filePath);
|
||||
console.log(`File has the following mime-type: ${mimeType}`);
|
||||
switch (mimeType) {
|
||||
@@ -426,8 +378,7 @@ export const uncompressRarArchive = async (filePath: string, options: any) => {
|
||||
const directoryOptions = {
|
||||
mode: 0o2775,
|
||||
};
|
||||
const { fileNameWithoutExtension, extension } =
|
||||
getFileConstituents(filePath);
|
||||
const { fileNameWithoutExtension, extension } = getFileConstituents(filePath);
|
||||
const targetDirectory = `${USERDATA_DIRECTORY}/expanded/${options.purpose}/${fileNameWithoutExtension}`;
|
||||
await createDirectory(directoryOptions, targetDirectory);
|
||||
|
||||
@@ -464,10 +415,7 @@ export const uncompressRarArchive = async (filePath: string, options: any) => {
|
||||
return await resizeImageDirectory(targetDirectory, options);
|
||||
};
|
||||
|
||||
export const resizeImageDirectory = async (
|
||||
directoryPath: string,
|
||||
options: any
|
||||
) => {
|
||||
export const resizeImageDirectory = async (directoryPath: string, options: any) => {
|
||||
const files = await walkFolder(directoryPath, [
|
||||
".jpg",
|
||||
".jpeg",
|
||||
@@ -495,25 +443,15 @@ export const resizeImage = (directoryPath: string, file: any, options: any) => {
|
||||
const { baseWidth } = options.imageResizeOptions;
|
||||
const sharpResizeInstance = sharp().resize(baseWidth).toFormat("jpg");
|
||||
return new Promise((resolve, reject) => {
|
||||
const resizedStream = createReadStream(
|
||||
`${directoryPath}/${file.name}${file.extension}`
|
||||
);
|
||||
const resizedStream = createReadStream(`${directoryPath}/${file.name}${file.extension}`);
|
||||
if (fse.existsSync(`${directoryPath}/${file.name}${file.extension}`)) {
|
||||
resizedStream
|
||||
.pipe(sharpResizeInstance)
|
||||
.toFile(
|
||||
`${directoryPath}/${file.name}_${baseWidth}px${file.extension}`
|
||||
)
|
||||
.toFile(`${directoryPath}/${file.name}_${baseWidth}px${file.extension}`)
|
||||
.then((data) => {
|
||||
console.log(
|
||||
`Resized image ${JSON.stringify(data, null, 4)}`
|
||||
);
|
||||
fse.unlink(
|
||||
`${directoryPath}/${file.name}${file.extension}`
|
||||
);
|
||||
resolve(
|
||||
`${directoryPath}/${file.name}_${baseWidth}px${file.extension}`
|
||||
);
|
||||
console.log(`Resized image ${JSON.stringify(data, null, 4)}`);
|
||||
fse.unlink(`${directoryPath}/${file.name}${file.extension}`);
|
||||
resolve(`${directoryPath}/${file.name}_${baseWidth}px${file.extension}`);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user