From 1a2770aa4044e1323aecb2aa5c047de2cd116ad8 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Sun, 13 Mar 2022 10:46:22 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20Refactored=207zip=20method=20to?= =?UTF-8?q?=20ignore=20dotfiles=20and=20directories?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 10 -- package.json | 1 - utils/uncompression.utils.ts | 172 +++++++++++++++++------------------ utils/xml.utils.ts | 1 - yarn.lock | 4 - 5 files changed, 81 insertions(+), 107 deletions(-) diff --git a/package-lock.json b/package-lock.json index fedcda0..35b531e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,6 @@ "@types/mkdirp": "^1.0.0", "@types/node": "^13.9.8", "@types/string-similarity": "^4.0.0", - "@zhangfuxing/unrar": "github:rishighan/unrar", "axios": "^0.25.0", "axios-retry": "^3.2.4", "bree": "^7.1.5", @@ -2165,11 +2164,6 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@zhangfuxing/unrar": { - "version": "0.6.3", - "resolved": "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1", - "license": "MIT" - }, "node_modules/7zip-bin": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz", @@ -16666,10 +16660,6 @@ "eslint-visitor-keys": "^2.0.0" } }, - "@zhangfuxing/unrar": { - "version": "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1", - "from": "@zhangfuxing/unrar@github:rishighan/unrar" - }, "7zip-bin": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz", diff --git a/package.json b/package.json index 9cb3bdd..7ba543d 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,6 @@ "@types/mkdirp": "^1.0.0", "@types/node": "^13.9.8", "@types/string-similarity": "^4.0.0", - "@zhangfuxing/unrar": "github:rishighan/unrar", "axios": "^0.25.0", "axios-retry": "^3.2.4", "bree": "^7.1.5", diff --git a/utils/uncompression.utils.ts b/utils/uncompression.utils.ts index de3f10f..7eecb56 100644 --- a/utils/uncompression.utils.ts +++ b/utils/uncompression.utils.ts @@ -31,10 +31,11 @@ SOFTWARE. * Initial: 2021/05/04 Rishi Ghan */ -import { promises as fs } from "fs"; +import { createWriteStream, createReadStream, promises as fs } from "fs"; const fse = require("fs-extra"); const Unrar = require("unrar"); import path, { parse } from "path"; + import { IExtractComicBookCoverErrorResponse, IExtractedComicBookCoverFile, @@ -49,7 +50,7 @@ import { walkFolder, } from "../utils/file.utils"; import { resizeImage } from "./imagetransformation.utils"; -import { each, filter, isNil, isUndefined, remove } from "lodash"; +import { isNil, isUndefined, remove } from "lodash"; import { convertXMLToJSON } from "./xml.utils"; import sevenBin from "7zip-bin"; import { extract, list } from "node-7z"; @@ -146,14 +147,11 @@ export const extractComicInfoXMLFromRar = async ( await fse.ensureDir(targetDirectory, directoryOptions); console.info(`%s was created.`, targetDirectory); - const archive = new Unrar(path.resolve(filePath)); - // or - // var archive = new Unrar({ - // path: protectedArchivePath, - // arguments: ['-pPassword'], - // bin: pathToUnrarBin // Default: unrar - // }); - archive.list(function (err, entries) { + const archive = new Unrar({ + path: path.resolve(filePath), + bin: `/usr/local/bin/unrar`, // this will change depending on Docker base OS + }); + archive.list(async (err, entries) => { remove(entries, ({ type }) => type === "Directory"); const comicInfoXML = remove( entries, @@ -166,55 +164,43 @@ export const extractComicInfoXMLFromRar = async ( .localeCompare(b.name.toLowerCase()); } }); - const sharpStream = sharp().resize(200, 200); + // Cover image extraction and resizing + const sharpStream = sharp().resize(275); + let comicInfoString = ""; const coverImageStream = archive .stream(files[0].name) .on("error", console.error) - // .pipe( - // require("fs").createWriteStream(`${targetDirectory}/0.jpg`) - // ); .pipe(sharpStream) - .toFile(`${targetDirectory}/0.jpg`, (err, info) => { + .toFile(`${targetDirectory}/${files[0].name}`, (err, info) => { if (err) { console.log("Failed to resize image:"); console.log(err); return err; } - console.log( "Image file resized with the following parameters: %o", info ); return info; }); + // ComicInfo.xml extraction and parsing to JSON if (!isUndefined(comicInfoXML[0])) { - console.log(comicInfoXML); const comicinfoStream = archive.stream(comicInfoXML[0]["name"]); comicinfoStream.on("error", console.error); comicinfoStream.pipe( - require("fs").createWriteStream( - `${targetDirectory}/dhanda.xml` + createWriteStream( + `${targetDirectory}/${comicInfoXML[0]["name"]}` ) ); + comicinfoStream.on("data", (data) => { + comicInfoString += data; + }); + comicinfoStream.on("end", async (data) => { + const foo = await convertXMLToJSON(comicInfoString); + console.log(foo); + }); } }); - - // const extracted = extractor.extract({ - // files: ({ name }) => name.toLowerCase() === 'comicinfo.xml', - // }); - // const files = [...extracted.files]; //load the files - // if (!isUndefined(files[0])) { - // console.log( - // `comicinfo.xml detected in ${filePath}, attempting extraction...` - // ); - // const fileContents = String.fromCharCode.apply( - // null, - // files[0].extraction - // ); - // const parsedJSON = await convertXMLToJSON(fileContents); - // console.log(parsedJSON); - // return parsedJSON.comicinfo; - // } } catch (error) { throw new Error(error); } @@ -225,68 +211,72 @@ export const extractComicInfoXMLFromZip = async ( outputDirectory: string ) => { try { + const fileList = []; const listStream = list(path.resolve(filePath), { - $cherryPick: ["*.png", "*.jpg", , "*.jpeg", "*.webp", "*.xml"], $bin: pathTo7zip, }); - const fileList = []; - listStream - .on("data", (chunk) => fileList.push(chunk)) - .on("end", async () => { - // Look for ComicInfo.xml - const comicInfoXML = remove(fileList, (item) => - !isUndefined(item) - ? path.basename(item.file).toLowerCase() === - "comicinfo.xml" - : undefined - ); - // Sort the file list array naturally - const sortedFileList = fileList.sort((a, b) => - a.file.toLowerCase().localeCompare(b.file.toLowerCase()) - ); + listStream.on("data", (chunk) => fileList.push(chunk)); + listStream.on("end", async () => { + // Remove dotfiles and directories + remove( + fileList, + (item) => + item.attributes === "D...." || + !isNil(item.file.match(/(?:^|[\\\/])(\.(?!\.)[^\\\/]+)$/g)) + ); + // Look for ComicInfo.xml, if present, + // a. remove it from the fileList + const comicInfoXML = remove(fileList, (item) => + !isUndefined(item) + ? path.basename(item.file).toLowerCase() === "comicinfo.xml" + : undefined + ); + // Sort the file list array naturally + const sortedFileList = fileList.sort((a, b) => + a.file.toLowerCase().localeCompare(b.file.toLowerCase()) + ); - // Create the target directory - const directoryOptions = { - mode: 0o2775, - }; - const { fileNameWithoutExtension } = - getFileConstituents(filePath); - const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`; - await fse.ensureDir(targetDirectory, directoryOptions); - console.info(`%s was created.`, targetDirectory); + // Create the target directory + const directoryOptions = { + mode: 0o2775, + }; + const { fileNameWithoutExtension } = getFileConstituents(filePath); + const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`; + await fse.ensureDir(targetDirectory, directoryOptions); + console.info(`%s was created.`, targetDirectory); - // files to write - const filesToWrite = []; - if ( - !isUndefined(sortedFileList[0]) && - !isUndefined(sortedFileList[0].file) - ) { - filesToWrite.push(sortedFileList[0].file); + // Prep the shortlist + const filesToWrite = []; + if ( + !isUndefined(sortedFileList[0]) && + !isUndefined(sortedFileList[0].file) + ) { + filesToWrite.push(sortedFileList[0].file); + } + // b. if ComicInfo.xml present, include it in the file list to be written to disk + if (!isUndefined(comicInfoXML[0])) { + console.log(`ComicInfo.xml detected in ${filePath}`); + filesToWrite.push(comicInfoXML[0].file); + } + + // Remove nulls, undefined and empty elements from the file list + const filteredFilesToWrite = filesToWrite.filter( + (item) => !isUndefined(item) + ); + console.log(filesToWrite); + const extractStream = extract( + `${path.resolve(filePath)}`, + targetDirectory, + { + $cherryPick: [...filteredFilesToWrite], + $bin: pathTo7zip, } - - // if ComicInfo.xml present, include it in the file list to be written to disk - if (!isUndefined(comicInfoXML[0])) { - console.log(`ComicInfo.xml detected in ${filePath}`); - filesToWrite.push(comicInfoXML[0].file); - } - - // Remove nulls, undefined and empty elements from the file list - const filteredFilesToWrite = filesToWrite.filter( - (item) => !isUndefined(item) - ); - const extractStream = extract( - `${path.resolve(filePath)}`, - targetDirectory, - { - $cherryPick: filteredFilesToWrite, - $bin: pathTo7zip, - } - ); - extractStream.on("data", (data) => { - //do something with the image - console.log("ZIP:", data); - }); + ); + extractStream.on("data", (data) => { + //do something with the image + // console.log("ZIP:", data); }); + }); // for await (const chunk of foo) { // if (chunk.status === "extracted") { diff --git a/utils/xml.utils.ts b/utils/xml.utils.ts index 5826aca..cdbe0a3 100644 --- a/utils/xml.utils.ts +++ b/utils/xml.utils.ts @@ -27,7 +27,6 @@ SOFTWARE. */ import xml2js from "xml2js"; -import fs from "fs"; export const convertXMLToJSON = (xmlPayload) => { const parser = new xml2js.Parser({ diff --git a/yarn.lock b/yarn.lock index 53071d4..aa44a40 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1102,10 +1102,6 @@ "@typescript-eslint/types" "4.33.0" "eslint-visitor-keys" "^2.0.0" -"@zhangfuxing/unrar@github:rishighan/unrar": - "resolved" "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1" - "version" "0.6.3" - "7zip-bin@^5.1.1": "integrity" "sha512-sAP4LldeWNz0lNzmTird3uWfFDWWTeg6V/MsmyyLR9X1idwKBWIgt/ZvinqQldJm3LecKEs1emkbquO6PCiLVQ==" "resolved" "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz"