frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

🔧 Refactoring with a new unrar package

Browse Source
This commit is contained in:
Rishi Ghan
2022-03-11 02:11:21 -08:00
parent d9870c7e97
commit cceb0da33f
7 changed files with 201 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
models/comicvine.metadata.model.ts
View File

@@ -66,6 +66,8 @@ const ComicVineMetadataSchema = mongoose.Schema({
_id: false,
aliases: [String],
api_detail_url: String,
has_staff_review: { type: mongoose.Schema.Types.Mixed },
cover_date: String,
date_added: String,
date_last_updated: String,
@@ -84,7 +86,6 @@ const ComicVineMetadataSchema = mongoose.Schema({
image_tags: String,
},
has_staff_review: Boolean,
id: Number,
name: String,
resource_type: String,

27
package-lock.json generated
View File

@@ -16,6 +16,7 @@
"@types/mkdirp": "^1.0.0",
"@types/node": "^13.9.8",
"@types/string-similarity": "^4.0.0",
"@zhangfuxing/unrar": "github:rishighan/unrar",
"axios": "^0.25.0",
"axios-retry": "^3.2.4",
"bree": "^7.1.5",
@@ -45,7 +46,7 @@
"nats": "^1.3.2",
"node-7z": "^3.0.0",
"node-calibre": "^2.1.1",
"node-unrar-js": "^1.0.2",
"node-unrar-js": "^1.0.5",
"sharp": "^0.28.3",
"socket.io": "^4.4.0",
"threetwo-ui-typings": "^1.0.13",
@@ -2164,6 +2165,11 @@
"url": "https://opencollective.com/typescript-eslint"
}
},
"node_modules/@zhangfuxing/unrar": {
"version": "0.6.3",
"resolved": "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1",
"license": "MIT"
},
"node_modules/7zip-bin": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz",
@@ -9453,9 +9459,12 @@
"dev": true
},
"node_modules/node-unrar-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.3.tgz",
"integrity": "sha512-erEhBIhOxKYxxnIhB2CiIU1RQjPN7jjNKbpthWig0VTz3L6AwyORilvTdVzYOkxyTWGY9ZLqHB+EyqmnvvdI5g=="
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.5.tgz",
"integrity": "sha512-e2FfnO1rsbXIYPTHkzA/FV8+xDeuEIyRd5X8VQmWJMC5zeZFV9PX1TROTv2bmukX1kBJ2U8Lzu7LZiwUXByaOQ==",
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/normalize-path": {
"version": "3.0.0",
@@ -16657,6 +16666,10 @@
"eslint-visitor-keys": "^2.0.0"
}
},
"@zhangfuxing/unrar": {
"version": "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1",
"from": "@zhangfuxing/unrar@github:rishighan/unrar"
},
"7zip-bin": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz",
@@ -22114,9 +22127,9 @@
"dev": true
},
"node-unrar-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.3.tgz",
"integrity": "sha512-erEhBIhOxKYxxnIhB2CiIU1RQjPN7jjNKbpthWig0VTz3L6AwyORilvTdVzYOkxyTWGY9ZLqHB+EyqmnvvdI5g=="
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.5.tgz",
"integrity": "sha512-e2FfnO1rsbXIYPTHkzA/FV8+xDeuEIyRd5X8VQmWJMC5zeZFV9PX1TROTv2bmukX1kBJ2U8Lzu7LZiwUXByaOQ=="
},
"normalize-path": {
"version": "3.0.0",

3
package.json
View File

@@ -47,6 +47,7 @@
"@types/mkdirp": "^1.0.0",
"@types/node": "^13.9.8",
"@types/string-similarity": "^4.0.0",
"@zhangfuxing/unrar": "github:rishighan/unrar",
"axios": "^0.25.0",
"axios-retry": "^3.2.4",
"bree": "^7.1.5",
@@ -76,7 +77,7 @@
"nats": "^1.3.2",
"node-7z": "^3.0.0",
"node-calibre": "^2.1.1",
"node-unrar-js": "^1.0.2",
"node-unrar-js": "^1.0.5",
"sharp": "^0.28.3",
"socket.io": "^4.4.0",
"threetwo-ui-typings": "^1.0.13",

5
services/library.service.ts
View File

@@ -53,7 +53,6 @@ import {
IExtractedComicBookCoverFile,
IExtractionOptions,
} from "threetwo-ui-typings";
import { unrarArchive } from "../utils/uncompression.utils";
const ObjectId = require("mongoose").Types.ObjectId;
import fsExtra from "fs-extra";
const through2 = require("through2");
@@ -578,10 +577,6 @@ export default class ImportService extends Service {
}>
) {
console.log(ctx.params);
return await unrarArchive(
ctx.params.filePath,
ctx.params.options
);
},
},
},

83
services/libraryqueue.service.ts
View File

@@ -69,49 +69,51 @@ export default class QueueService extends Service {
console.info("New job received!", job.data);
console.info(`Processing queue...`);
// extract the cover
const result = await extractCoverFromFile2(
job.data.fileObject
);
// const result = await extractCoverFromFile2(
// job.data.fileObject
// );
const {
extension,
fileNameWithExtension,
fileNameWithoutExtension,
} = getFileConstituents(job.data.fileObject.filePath);
const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`;
const foo = await extractFromArchive(job.data.fileObject.filePath, targetDirectory, extension );
// infer any issue-related metadata from the filename
const { inferredIssueDetails } = refineQuery(result.name);
console.log("Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2));
// const {
// extension,
// fileNameWithExtension,
// fileNameWithoutExtension,
// } = getFileConstituents(job.data.fileObject.filePath);
// const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`;
// const foo = await extractFromArchive(job.data.fileObject.filePath, targetDirectory, extension );
// write to mongo
console.log("Writing to mongo...")
const dbImportResult = await this.broker.call(
"library.rawImportToDB",
{
importStatus: {
isImported: true,
tagged: false,
matchedResult: {
score: "0",
},
},
rawFileDetails: result,
inferredMetadata: {
issue: inferredIssueDetails,
},
sourcedMetadata: {
comicInfo: {},
comicvine: {},
},
// since we already have at least 1 copy
// mark it as not wanted by default
acquisition: {
wanted: false,
}
}
);
// const { inferredIssueDetails } = refineQuery(result.name);
// console.log("Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2));
// // // write to mongo
// console.log("Writing to mongo...")
// const dbImportResult = await this.broker.call(
// "library.rawImportToDB",
// {
// importStatus: {
// isImported: true,
// tagged: false,
// matchedResult: {
// score: "0",
// },
// },
// rawFileDetails: result,
// inferredMetadata: {
// issue: inferredIssueDetails,
// },
// sourcedMetadata: {
// comicInfo: {},
// comicvine: {},
// },
// // since we already have at least 1 copy
// // mark it as not wanted by default
// acquisition: {
// wanted: false,
// }
// }
// );
return Promise.resolve({
dbImportResult,
// dbImportResult,
id: job.id,
worker: process.pid,
});
@@ -151,6 +153,7 @@ export default class QueueService extends Service {
console.error(
`An error occured in 'process.import' queue on job id '${job.id}': ${error.message}`
);
console.error(job.data);
}
);
await this.getQueue("process.import").on(

210
utils/uncompression.utils.ts
View File

@@ -31,10 +31,10 @@ SOFTWARE.
* Initial: 2021/05/04 Rishi Ghan
*/
const fse = require("fs-extra");
import { promises as fs } from "fs";
const fse = require("fs-extra");
const Unrar = require("unrar");
import path, { parse } from "path";
import {
IExtractComicBookCoverErrorResponse,
IExtractedComicBookCoverFile,
@@ -49,12 +49,11 @@ import {
walkFolder,
} from "../utils/file.utils";
import { resizeImage } from "./imagetransformation.utils";
import { isNil, isUndefined } from "lodash";
import { each, filter, isNil, isUndefined, remove } from "lodash";
import { convertXMLToJSON } from "./xml.utils";
import sevenBin from "7zip-bin";
import { extract } from "node-7z";
import { extract, list } from "node-7z";
const pathTo7zip = sevenBin.path7za;
const unrar = require("node-unrar-js");
const { Calibre } = require("node-calibre");
import { USERDATA_DIRECTORY, COMICS_DIRECTORY } from "../constants/directories";
@@ -133,72 +132,52 @@ export const extractCoverFromFile2 = async (
}
};
export const unrarArchive = async (
filePath: string,
options: IExtractionOptions
) => {
// create directory
const directoryOptions = {
mode: 0o2775,
};
const fileBuffer = await fse
.readFile(filePath)
.catch((err) => console.error("Failed to read file", err));
try {
console.info("Unrar initiating.");
await fse.ensureDir(options.targetExtractionFolder, directoryOptions);
console.info(`${options.targetExtractionFolder} was created.`);
const extractor = await unrar.createExtractorFromData({
data: fileBuffer,
});
const files = extractor.extract({});
const extractedFiles = [...files.files];
for (const file of extractedFiles) {
console.info(`Attempting to write ${file.fileHeader.name}`);
const fileBuffer = file.extraction;
const fileName = explodePath(file.fileHeader.name).fileName;
// resize image
await resizeImage(
fileBuffer,
path.resolve(options.targetExtractionFolder + "/" + fileName),
200
);
}
// walk the newly created folder and return results
return await walkFolder(options.targetExtractionFolder, [
".jpg",
".png",
".jpeg",
]);
} catch (error) {
console.info(`${error}`);
}
};
export const extractComicInfoXMLFromRar = async (
filePath: string,
fileToExtract: string
) => {
try {
// Read the archive file into a typedArray
const fileBuffer = await fse
.readFile(filePath)
.catch((err) => console.error("Failed to read file", err));
const extractor = await unrar.createExtractorFromData({
data: fileBuffer,
// Create the target directory
const directoryOptions = {
mode: 0o2775,
};
const { fileNameWithoutExtension } = getFileConstituents(filePath);
const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`;
await fse.ensureDir(targetDirectory, directoryOptions);
console.info(`%s was created.`, targetDirectory);
const archive = new Unrar(path.resolve(filePath));
// or
// var archive = new Unrar({
// path: protectedArchivePath,
// arguments: ['-pPassword'],
// bin: pathToUnrarBin // Default: unrar
// });
const loo = [];
archive.list(function (err, entries) {
remove(entries, ({ type }) => type === "Directory");
const comicInfoXML = remove(entries, ({ name }) => name.toLowerCase() === "comicinfo.xml");
const foo = entries.sort((a, b) => {
if (!isUndefined(a) && !isUndefined(b)) {
return a.name
.toLowerCase()
.localeCompare(b.name.toLowerCase());
}
});
loo.push(foo)
console.log(loo[0][0]);
var stream = archive.stream(loo[0][0].name); // name of entry
stream.on('error', console.error);
stream.pipe(require('fs').createWriteStream(`${targetDirectory}/0.jpg`));
if(!isUndefined(comicInfoXML[0]) ) {
console.log(comicInfoXML);
const comicinfoStream = archive.stream(comicInfoXML[0]["name"]);
comicinfoStream.on('error', console.error);
comicinfoStream.pipe(require('fs').createWriteStream(`${targetDirectory}/dhanda.xml`));
}
});
console.info('Unrar initiating.');
const files = extractor.extract({});
const extractedFiles = [...files.files];
console.log(extractedFiles[0]);
for (const file of extractedFiles) {
console.info(`Attempting to write ${file.fileHeader.name}`);
}
// const extracted = extractor.extract({
// files: ({ name }) => name.toLowerCase() === 'comicinfo.xml',
@@ -225,25 +204,88 @@ export const extractComicInfoXMLFromZip = async (
filePath: string,
outputDirectory: string
) => {
const foo = extract(path.resolve(filePath), outputDirectory, {
$cherryPick: ["*.xml"],
$bin: pathTo7zip,
});
for await (const chunk of foo) {
if (chunk.status === "extracted") {
console.log(
`comicinfo.xml detected in ${filePath}, attempting extraction...`
);
const fileContents = await fs.readFile(
path.resolve(`${outputDirectory}/${chunk.file}`),
"utf8"
);
const parsedJSON = await convertXMLToJSON(
Buffer.from(fileContents)
);
console.log(parsedJSON);
return parsedJSON.comicinfo;
}
try {
const listStream = list(path.resolve(filePath), {
$cherryPick: ["*.png", "*.jpg", , "*.jpeg", "*.webp", "*.xml"],
$bin: pathTo7zip,
});
const fileList = [];
listStream
.on("data", (chunk) => fileList.push(chunk))
.on("end", async () => {
// Look for ComicInfo.xml
const comicInfoXML = remove(fileList, (item) =>
!isUndefined(item)
? path.basename(item.file).toLowerCase() ===
"comicinfo.xml"
: undefined
);
// Sort the file list array naturally
const sortedFileList = fileList.sort((a, b) =>
a.file.toLowerCase().localeCompare(b.file.toLowerCase())
);
// Create the target directory
const directoryOptions = {
mode: 0o2775,
};
const { fileNameWithoutExtension } =
getFileConstituents(filePath);
const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`;
await fse.ensureDir(targetDirectory, directoryOptions);
console.info(`%s was created.`, targetDirectory);
// files to write
const filesToWrite = [];
if (
!isUndefined(sortedFileList[0]) &&
!isUndefined(sortedFileList[0].file)
) {
filesToWrite.push(sortedFileList[0].file);
}
// if ComicInfo.xml present, include it in the file list to be written to disk
if (!isUndefined(comicInfoXML[0])) {
console.log(`ComicInfo.xml detected in ${filePath}`);
filesToWrite.push(comicInfoXML[0].file);
}
// Remove nulls, undefined and empty elements from the file list
const filteredFilesToWrite = filesToWrite.filter(
(item) => !isUndefined(item)
);
const extractStream = extract(
`${path.resolve(filePath)}`,
targetDirectory,
{
$cherryPick: filteredFilesToWrite,
$bin: pathTo7zip,
}
);
extractStream.on("data", (data) => {
//do something with the image
console.log("ZIP:", data);
});
});
// for await (const chunk of foo) {
// if (chunk.status === "extracted") {
// console.log(
// `comicinfo.xml detected in ${filePath}, attempting extraction...`
// );
// const fileContents = await fs.readFile(
// path.resolve(`${outputDirectory}/${chunk.file}`),
// "utf8"
// );
// const parsedJSON = await convertXMLToJSON(
// Buffer.from(fileContents)
// );
// console.log(parsedJSON);
// return parsedJSON.comicinfo;
// }
// }
} catch (error) {
throw new Error(error);
}
};

12
yarn.lock
View File

@@ -1102,6 +1102,10 @@
"@typescript-eslint/types" "4.33.0"
"eslint-visitor-keys" "^2.0.0"
"@zhangfuxing/unrar@github:rishighan/unrar":
"resolved" "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1"
"version" "0.6.3"
"7zip-bin@^5.1.1":
"integrity" "sha512-sAP4LldeWNz0lNzmTird3uWfFDWWTeg6V/MsmyyLR9X1idwKBWIgt/ZvinqQldJm3LecKEs1emkbquO6PCiLVQ=="
"resolved" "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz"
@@ -5209,10 +5213,10 @@
"resolved" "https://registry.npmjs.org/node-releases/-/node-releases-2.0.1.tgz"
"version" "2.0.1"
"node-unrar-js@^1.0.2":
"integrity" "sha512-erEhBIhOxKYxxnIhB2CiIU1RQjPN7jjNKbpthWig0VTz3L6AwyORilvTdVzYOkxyTWGY9ZLqHB+EyqmnvvdI5g=="
"resolved" "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.3.tgz"
"version" "1.0.3"
"node-unrar-js@^1.0.5":
"integrity" "sha512-e2FfnO1rsbXIYPTHkzA/FV8+xDeuEIyRd5X8VQmWJMC5zeZFV9PX1TROTv2bmukX1kBJ2U8Lzu7LZiwUXByaOQ=="
"resolved" "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.5.tgz"
"version" "1.0.5"
"nopt@*", "nopt@^5.0.0":
"version" "5.0.0"