diff --git a/models/comicvine.metadata.model.ts b/models/comicvine.metadata.model.ts index 9afbfbf..2133f5b 100644 --- a/models/comicvine.metadata.model.ts +++ b/models/comicvine.metadata.model.ts @@ -66,6 +66,8 @@ const ComicVineMetadataSchema = mongoose.Schema({ _id: false, aliases: [String], api_detail_url: String, + has_staff_review: { type: mongoose.Schema.Types.Mixed }, + cover_date: String, date_added: String, date_last_updated: String, @@ -84,7 +86,6 @@ const ComicVineMetadataSchema = mongoose.Schema({ image_tags: String, }, - has_staff_review: Boolean, id: Number, name: String, resource_type: String, diff --git a/package-lock.json b/package-lock.json index 0b3b302..fedcda0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@types/mkdirp": "^1.0.0", "@types/node": "^13.9.8", "@types/string-similarity": "^4.0.0", + "@zhangfuxing/unrar": "github:rishighan/unrar", "axios": "^0.25.0", "axios-retry": "^3.2.4", "bree": "^7.1.5", @@ -45,7 +46,7 @@ "nats": "^1.3.2", "node-7z": "^3.0.0", "node-calibre": "^2.1.1", - "node-unrar-js": "^1.0.2", + "node-unrar-js": "^1.0.5", "sharp": "^0.28.3", "socket.io": "^4.4.0", "threetwo-ui-typings": "^1.0.13", @@ -2164,6 +2165,11 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@zhangfuxing/unrar": { + "version": "0.6.3", + "resolved": "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1", + "license": "MIT" + }, "node_modules/7zip-bin": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz", @@ -9453,9 +9459,12 @@ "dev": true }, "node_modules/node-unrar-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.3.tgz", - "integrity": "sha512-erEhBIhOxKYxxnIhB2CiIU1RQjPN7jjNKbpthWig0VTz3L6AwyORilvTdVzYOkxyTWGY9ZLqHB+EyqmnvvdI5g==" + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.5.tgz", + "integrity": "sha512-e2FfnO1rsbXIYPTHkzA/FV8+xDeuEIyRd5X8VQmWJMC5zeZFV9PX1TROTv2bmukX1kBJ2U8Lzu7LZiwUXByaOQ==", + "engines": { + "node": ">=10.0.0" + } }, "node_modules/normalize-path": { "version": "3.0.0", @@ -16657,6 +16666,10 @@ "eslint-visitor-keys": "^2.0.0" } }, + "@zhangfuxing/unrar": { + "version": "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1", + "from": "@zhangfuxing/unrar@github:rishighan/unrar" + }, "7zip-bin": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz", @@ -22114,9 +22127,9 @@ "dev": true }, "node-unrar-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.3.tgz", - "integrity": "sha512-erEhBIhOxKYxxnIhB2CiIU1RQjPN7jjNKbpthWig0VTz3L6AwyORilvTdVzYOkxyTWGY9ZLqHB+EyqmnvvdI5g==" + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.5.tgz", + "integrity": "sha512-e2FfnO1rsbXIYPTHkzA/FV8+xDeuEIyRd5X8VQmWJMC5zeZFV9PX1TROTv2bmukX1kBJ2U8Lzu7LZiwUXByaOQ==" }, "normalize-path": { "version": "3.0.0", diff --git a/package.json b/package.json index 93019d7..9cb3bdd 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "@types/mkdirp": "^1.0.0", "@types/node": "^13.9.8", "@types/string-similarity": "^4.0.0", + "@zhangfuxing/unrar": "github:rishighan/unrar", "axios": "^0.25.0", "axios-retry": "^3.2.4", "bree": "^7.1.5", @@ -76,7 +77,7 @@ "nats": "^1.3.2", "node-7z": "^3.0.0", "node-calibre": "^2.1.1", - "node-unrar-js": "^1.0.2", + "node-unrar-js": "^1.0.5", "sharp": "^0.28.3", "socket.io": "^4.4.0", "threetwo-ui-typings": "^1.0.13", diff --git a/services/library.service.ts b/services/library.service.ts index 5a76e86..3de02c5 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -53,7 +53,6 @@ import { IExtractedComicBookCoverFile, IExtractionOptions, } from "threetwo-ui-typings"; -import { unrarArchive } from "../utils/uncompression.utils"; const ObjectId = require("mongoose").Types.ObjectId; import fsExtra from "fs-extra"; const through2 = require("through2"); @@ -578,10 +577,6 @@ export default class ImportService extends Service { }> ) { console.log(ctx.params); - return await unrarArchive( - ctx.params.filePath, - ctx.params.options - ); }, }, }, diff --git a/services/libraryqueue.service.ts b/services/libraryqueue.service.ts index 1b87442..20b12b9 100644 --- a/services/libraryqueue.service.ts +++ b/services/libraryqueue.service.ts @@ -69,49 +69,51 @@ export default class QueueService extends Service { console.info("New job received!", job.data); console.info(`Processing queue...`); // extract the cover - const result = await extractCoverFromFile2( - job.data.fileObject - ); + // const result = await extractCoverFromFile2( + // job.data.fileObject + // ); + + const { + extension, + fileNameWithExtension, + fileNameWithoutExtension, + } = getFileConstituents(job.data.fileObject.filePath); + const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`; + const foo = await extractFromArchive(job.data.fileObject.filePath, targetDirectory, extension ); // infer any issue-related metadata from the filename - const { inferredIssueDetails } = refineQuery(result.name); - console.log("Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2)); - // const { - // extension, - // fileNameWithExtension, - // fileNameWithoutExtension, - // } = getFileConstituents(job.data.fileObject.filePath); - // const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`; - // const foo = await extractFromArchive(job.data.fileObject.filePath, targetDirectory, extension ); - // write to mongo - console.log("Writing to mongo...") - const dbImportResult = await this.broker.call( - "library.rawImportToDB", - { - importStatus: { - isImported: true, - tagged: false, - matchedResult: { - score: "0", - }, - }, - rawFileDetails: result, - inferredMetadata: { - issue: inferredIssueDetails, - }, - sourcedMetadata: { - comicInfo: {}, - comicvine: {}, - }, - // since we already have at least 1 copy - // mark it as not wanted by default - acquisition: { - wanted: false, - } - } - ); + // const { inferredIssueDetails } = refineQuery(result.name); + // console.log("Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2)); + + // // // write to mongo + // console.log("Writing to mongo...") + // const dbImportResult = await this.broker.call( + // "library.rawImportToDB", + // { + // importStatus: { + // isImported: true, + // tagged: false, + // matchedResult: { + // score: "0", + // }, + // }, + // rawFileDetails: result, + // inferredMetadata: { + // issue: inferredIssueDetails, + // }, + // sourcedMetadata: { + // comicInfo: {}, + // comicvine: {}, + // }, + // // since we already have at least 1 copy + // // mark it as not wanted by default + // acquisition: { + // wanted: false, + // } + // } + // ); return Promise.resolve({ - dbImportResult, + // dbImportResult, id: job.id, worker: process.pid, }); @@ -151,6 +153,7 @@ export default class QueueService extends Service { console.error( `An error occured in 'process.import' queue on job id '${job.id}': ${error.message}` ); + console.error(job.data); } ); await this.getQueue("process.import").on( diff --git a/utils/uncompression.utils.ts b/utils/uncompression.utils.ts index 154fc32..e74d74a 100644 --- a/utils/uncompression.utils.ts +++ b/utils/uncompression.utils.ts @@ -31,10 +31,10 @@ SOFTWARE. * Initial: 2021/05/04 Rishi Ghan */ -const fse = require("fs-extra"); import { promises as fs } from "fs"; +const fse = require("fs-extra"); +const Unrar = require("unrar"); import path, { parse } from "path"; - import { IExtractComicBookCoverErrorResponse, IExtractedComicBookCoverFile, @@ -49,12 +49,11 @@ import { walkFolder, } from "../utils/file.utils"; import { resizeImage } from "./imagetransformation.utils"; -import { isNil, isUndefined } from "lodash"; +import { each, filter, isNil, isUndefined, remove } from "lodash"; import { convertXMLToJSON } from "./xml.utils"; import sevenBin from "7zip-bin"; -import { extract } from "node-7z"; +import { extract, list } from "node-7z"; const pathTo7zip = sevenBin.path7za; -const unrar = require("node-unrar-js"); const { Calibre } = require("node-calibre"); import { USERDATA_DIRECTORY, COMICS_DIRECTORY } from "../constants/directories"; @@ -133,72 +132,52 @@ export const extractCoverFromFile2 = async ( } }; -export const unrarArchive = async ( - filePath: string, - options: IExtractionOptions -) => { - // create directory - const directoryOptions = { - mode: 0o2775, - }; - - const fileBuffer = await fse - .readFile(filePath) - .catch((err) => console.error("Failed to read file", err)); - try { - console.info("Unrar initiating."); - await fse.ensureDir(options.targetExtractionFolder, directoryOptions); - console.info(`${options.targetExtractionFolder} was created.`); - - const extractor = await unrar.createExtractorFromData({ - data: fileBuffer, - }); - const files = extractor.extract({}); - const extractedFiles = [...files.files]; - for (const file of extractedFiles) { - console.info(`Attempting to write ${file.fileHeader.name}`); - const fileBuffer = file.extraction; - const fileName = explodePath(file.fileHeader.name).fileName; - // resize image - await resizeImage( - fileBuffer, - path.resolve(options.targetExtractionFolder + "/" + fileName), - 200 - ); - } - // walk the newly created folder and return results - return await walkFolder(options.targetExtractionFolder, [ - ".jpg", - ".png", - ".jpeg", - ]); - } catch (error) { - console.info(`${error}`); - } -}; - export const extractComicInfoXMLFromRar = async ( filePath: string, fileToExtract: string ) => { try { - // Read the archive file into a typedArray - const fileBuffer = await fse - .readFile(filePath) - .catch((err) => console.error("Failed to read file", err)); - const extractor = await unrar.createExtractorFromData({ - data: fileBuffer, + // Create the target directory + const directoryOptions = { + mode: 0o2775, + }; + const { fileNameWithoutExtension } = getFileConstituents(filePath); + const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`; + await fse.ensureDir(targetDirectory, directoryOptions); + console.info(`%s was created.`, targetDirectory); + + const archive = new Unrar(path.resolve(filePath)); + // or + // var archive = new Unrar({ + // path: protectedArchivePath, + // arguments: ['-pPassword'], + // bin: pathToUnrarBin // Default: unrar + // }); + const loo = []; + archive.list(function (err, entries) { + remove(entries, ({ type }) => type === "Directory"); + const comicInfoXML = remove(entries, ({ name }) => name.toLowerCase() === "comicinfo.xml"); + const foo = entries.sort((a, b) => { + if (!isUndefined(a) && !isUndefined(b)) { + return a.name + .toLowerCase() + .localeCompare(b.name.toLowerCase()); + } + }); + loo.push(foo) + console.log(loo[0][0]); + var stream = archive.stream(loo[0][0].name); // name of entry + stream.on('error', console.error); + stream.pipe(require('fs').createWriteStream(`${targetDirectory}/0.jpg`)); + if(!isUndefined(comicInfoXML[0]) ) { + console.log(comicInfoXML); + const comicinfoStream = archive.stream(comicInfoXML[0]["name"]); + comicinfoStream.on('error', console.error); + comicinfoStream.pipe(require('fs').createWriteStream(`${targetDirectory}/dhanda.xml`)); + + } + }); - console.info('Unrar initiating.'); - - - - const files = extractor.extract({}); - const extractedFiles = [...files.files]; - console.log(extractedFiles[0]); - for (const file of extractedFiles) { - console.info(`Attempting to write ${file.fileHeader.name}`); - } // const extracted = extractor.extract({ // files: ({ name }) => name.toLowerCase() === 'comicinfo.xml', @@ -225,25 +204,88 @@ export const extractComicInfoXMLFromZip = async ( filePath: string, outputDirectory: string ) => { - const foo = extract(path.resolve(filePath), outputDirectory, { - $cherryPick: ["*.xml"], - $bin: pathTo7zip, - }); - for await (const chunk of foo) { - if (chunk.status === "extracted") { - console.log( - `comicinfo.xml detected in ${filePath}, attempting extraction...` - ); - const fileContents = await fs.readFile( - path.resolve(`${outputDirectory}/${chunk.file}`), - "utf8" - ); - const parsedJSON = await convertXMLToJSON( - Buffer.from(fileContents) - ); - console.log(parsedJSON); - return parsedJSON.comicinfo; - } + try { + const listStream = list(path.resolve(filePath), { + $cherryPick: ["*.png", "*.jpg", , "*.jpeg", "*.webp", "*.xml"], + $bin: pathTo7zip, + }); + const fileList = []; + listStream + .on("data", (chunk) => fileList.push(chunk)) + .on("end", async () => { + // Look for ComicInfo.xml + const comicInfoXML = remove(fileList, (item) => + !isUndefined(item) + ? path.basename(item.file).toLowerCase() === + "comicinfo.xml" + : undefined + ); + // Sort the file list array naturally + const sortedFileList = fileList.sort((a, b) => + a.file.toLowerCase().localeCompare(b.file.toLowerCase()) + ); + + // Create the target directory + const directoryOptions = { + mode: 0o2775, + }; + const { fileNameWithoutExtension } = + getFileConstituents(filePath); + const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`; + await fse.ensureDir(targetDirectory, directoryOptions); + console.info(`%s was created.`, targetDirectory); + + // files to write + const filesToWrite = []; + if ( + !isUndefined(sortedFileList[0]) && + !isUndefined(sortedFileList[0].file) + ) { + filesToWrite.push(sortedFileList[0].file); + } + + // if ComicInfo.xml present, include it in the file list to be written to disk + if (!isUndefined(comicInfoXML[0])) { + console.log(`ComicInfo.xml detected in ${filePath}`); + filesToWrite.push(comicInfoXML[0].file); + } + + // Remove nulls, undefined and empty elements from the file list + const filteredFilesToWrite = filesToWrite.filter( + (item) => !isUndefined(item) + ); + const extractStream = extract( + `${path.resolve(filePath)}`, + targetDirectory, + { + $cherryPick: filteredFilesToWrite, + $bin: pathTo7zip, + } + ); + extractStream.on("data", (data) => { + //do something with the image + console.log("ZIP:", data); + }); + }); + + // for await (const chunk of foo) { + // if (chunk.status === "extracted") { + // console.log( + // `comicinfo.xml detected in ${filePath}, attempting extraction...` + // ); + // const fileContents = await fs.readFile( + // path.resolve(`${outputDirectory}/${chunk.file}`), + // "utf8" + // ); + // const parsedJSON = await convertXMLToJSON( + // Buffer.from(fileContents) + // ); + // console.log(parsedJSON); + // return parsedJSON.comicinfo; + // } + // } + } catch (error) { + throw new Error(error); } }; diff --git a/yarn.lock b/yarn.lock index 99208e4..53071d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1102,6 +1102,10 @@ "@typescript-eslint/types" "4.33.0" "eslint-visitor-keys" "^2.0.0" +"@zhangfuxing/unrar@github:rishighan/unrar": + "resolved" "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1" + "version" "0.6.3" + "7zip-bin@^5.1.1": "integrity" "sha512-sAP4LldeWNz0lNzmTird3uWfFDWWTeg6V/MsmyyLR9X1idwKBWIgt/ZvinqQldJm3LecKEs1emkbquO6PCiLVQ==" "resolved" "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz" @@ -5209,10 +5213,10 @@ "resolved" "https://registry.npmjs.org/node-releases/-/node-releases-2.0.1.tgz" "version" "2.0.1" -"node-unrar-js@^1.0.2": - "integrity" "sha512-erEhBIhOxKYxxnIhB2CiIU1RQjPN7jjNKbpthWig0VTz3L6AwyORilvTdVzYOkxyTWGY9ZLqHB+EyqmnvvdI5g==" - "resolved" "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.3.tgz" - "version" "1.0.3" +"node-unrar-js@^1.0.5": + "integrity" "sha512-e2FfnO1rsbXIYPTHkzA/FV8+xDeuEIyRd5X8VQmWJMC5zeZFV9PX1TROTv2bmukX1kBJ2U8Lzu7LZiwUXByaOQ==" + "resolved" "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-1.0.5.tgz" + "version" "1.0.5" "nopt@*", "nopt@^5.0.0": "version" "5.0.0"