From 336735ef039e5e3476b3259e36ec2f8c656ea020 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Mon, 7 Mar 2022 21:09:58 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20Refactoring=20to=20move=20away?= =?UTF-8?q?=20from=20calibre=20for=20uncompression=20and=20extraction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/comic.model.ts | 3 +- models/comicvine.metadata.model.ts | 14 +++++++++- services/library.service.ts | 5 ++-- services/libraryqueue.service.ts | 10 +++++-- services/search.service.ts | 7 ++--- utils/uncompression.utils.ts | 44 ++++++++++++++++++------------ 6 files changed, 56 insertions(+), 27 deletions(-) diff --git a/models/comic.model.ts b/models/comic.model.ts index da8f65d..d1e281f 100644 --- a/models/comic.model.ts +++ b/models/comic.model.ts @@ -57,12 +57,13 @@ const ComicSchema = mongoose.Schema( shortboxed: {}, gcd: {}, }, - rawFileDetails: { type: RawFileDetailsSchema, es_indexed: true }, + rawFileDetails: { type: RawFileDetailsSchema, es_indexed: true, default: {} }, inferredMetadata: { issue: { name: { type: String, es_indexed: true }, number: { type: Number, + es_indexed: true, required: false, default: 0, }, diff --git a/models/comicvine.metadata.model.ts b/models/comicvine.metadata.model.ts index 0955e52..9afbfbf 100644 --- a/models/comicvine.metadata.model.ts +++ b/models/comicvine.metadata.model.ts @@ -39,7 +39,6 @@ const VolumeInformation = mongoose.Schema({ original_url: String, image_tags: String, }, - issues: [ { api_detail_url: String, @@ -72,6 +71,19 @@ const ComicVineMetadataSchema = mongoose.Schema({ date_last_updated: String, deck: String, description: String, + image: { + icon_url: String, + medium_url: String, + screen_url: String, + screen_large_url: String, + small_url: String, + super_url: String, + thumb_url: String, + tiny_url: String, + original_url: String, + image_tags: String, + }, + has_staff_review: Boolean, id: Number, name: String, diff --git a/services/library.service.ts b/services/library.service.ts index 842e120..5a76e86 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -124,7 +124,8 @@ export default class ImportService extends Service { let comicExists = await Comic.exists({ "rawFileDetails.name": `${path.basename( item.path, - path.extname(item.path) + path.extname(item.path), + )}`, }); if (!comicExists) { @@ -171,7 +172,7 @@ export default class ImportService extends Service { ) { let volumeDetails; const comicMetadata = ctx.params; - + console.log(comicMetadata); // When an issue is added from the search CV feature if ( comicMetadata.sourcedMetadata.comicvine && diff --git a/services/libraryqueue.service.ts b/services/libraryqueue.service.ts index 5217efa..1b87442 100644 --- a/services/libraryqueue.service.ts +++ b/services/libraryqueue.service.ts @@ -46,7 +46,7 @@ import BullMQMixin from "moleculer-bull"; import { SandboxedJob } from "moleculer-bull"; import { DbMixin } from "../mixins/db.mixin"; import Comic from "../models/comic.model"; -import { extractComicInfoXMLFromArchive, extractCoverFromFile2 } from "../utils/uncompression.utils"; +import { extractFromArchive, extractCoverFromFile2 } from "../utils/uncompression.utils"; import { refineQuery } from "filename-parser"; import { io } from "./api.service"; import { getFileConstituents } from "../utils/file.utils"; @@ -76,7 +76,13 @@ export default class QueueService extends Service { // infer any issue-related metadata from the filename const { inferredIssueDetails } = refineQuery(result.name); console.log("Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2)); - + // const { + // extension, + // fileNameWithExtension, + // fileNameWithoutExtension, + // } = getFileConstituents(job.data.fileObject.filePath); + // const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`; + // const foo = await extractFromArchive(job.data.fileObject.filePath, targetDirectory, extension ); // write to mongo console.log("Writing to mongo...") const dbImportResult = await this.broker.call( diff --git a/services/search.service.ts b/services/search.service.ts index e25f5ee..dbbb57a 100644 --- a/services/search.service.ts +++ b/services/search.service.ts @@ -51,12 +51,11 @@ export default class SettingsService extends Service { const { body } = await eSClient.msearch({ body: queries, }); - - body.results.forEach((match) => { - console.log(match.hits.hits); + body.responses.forEach((match) => { + console.log(match.hits); }); - return body.results; + return body.responses; }, }, issue: { diff --git a/utils/uncompression.utils.ts b/utils/uncompression.utils.ts index 57afd73..154fc32 100644 --- a/utils/uncompression.utils.ts +++ b/utils/uncompression.utils.ts @@ -189,23 +189,33 @@ export const extractComicInfoXMLFromRar = async ( const extractor = await unrar.createExtractorFromData({ data: fileBuffer, }); + console.info('Unrar initiating.'); - const extracted = extractor.extract({ - files: ({ name }) => name.toLowerCase() === 'comicinfo.xml', - }); - const files = [...extracted.files]; //load the files - if (!isUndefined(files[0])) { - console.log( - `comicinfo.xml detected in ${filePath}, attempting extraction...` - ); - const fileContents = String.fromCharCode.apply( - null, - files[0].extraction - ); - const parsedJSON = await convertXMLToJSON(fileContents); - console.log(parsedJSON); - return parsedJSON.comicinfo; - } + + + const files = extractor.extract({}); + const extractedFiles = [...files.files]; + console.log(extractedFiles[0]); + for (const file of extractedFiles) { + console.info(`Attempting to write ${file.fileHeader.name}`); + } + + // const extracted = extractor.extract({ + // files: ({ name }) => name.toLowerCase() === 'comicinfo.xml', + // }); + // const files = [...extracted.files]; //load the files + // if (!isUndefined(files[0])) { + // console.log( + // `comicinfo.xml detected in ${filePath}, attempting extraction...` + // ); + // const fileContents = String.fromCharCode.apply( + // null, + // files[0].extraction + // ); + // const parsedJSON = await convertXMLToJSON(fileContents); + // console.log(parsedJSON); + // return parsedJSON.comicinfo; + // } } catch (error) { throw new Error(error); } @@ -237,7 +247,7 @@ export const extractComicInfoXMLFromZip = async ( } }; -export const extractComicInfoXMLFromArchive = async ( +export const extractFromArchive = async ( filePath: string, outputDirectory: string, extension: string