frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

🔧 Refactored 7zip method to ignore dotfiles and directories

Browse Source
This commit is contained in:
Rishi Ghan
2022-03-13 10:46:22 -07:00
parent 31f307a753
commit 1a2770aa40
5 changed files with 81 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
package-lock.json generated
View File

@@ -16,7 +16,6 @@
"@types/mkdirp": "^1.0.0", "@types/mkdirp": "^1.0.0",
"@types/node": "^13.9.8", "@types/node": "^13.9.8",
"@types/string-similarity": "^4.0.0", "@types/string-similarity": "^4.0.0",
"@zhangfuxing/unrar": "github:rishighan/unrar",
"axios": "^0.25.0", "axios": "^0.25.0",
"axios-retry": "^3.2.4", "axios-retry": "^3.2.4",
"bree": "^7.1.5", "bree": "^7.1.5",
@@ -2165,11 +2164,6 @@
"url": "https://opencollective.com/typescript-eslint" "url": "https://opencollective.com/typescript-eslint"
} }
}, },
"node_modules/@zhangfuxing/unrar": {
"version": "0.6.3",
"resolved": "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1",
"license": "MIT"
},
"node_modules/7zip-bin": { "node_modules/7zip-bin": {
"version": "5.1.1", "version": "5.1.1",
"resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz", "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz",
@@ -16666,10 +16660,6 @@
"eslint-visitor-keys": "^2.0.0" "eslint-visitor-keys": "^2.0.0"
} }
}, },
"@zhangfuxing/unrar": {
"version": "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1",
"from": "@zhangfuxing/unrar@github:rishighan/unrar"
},
"7zip-bin": { "7zip-bin": {
"version": "5.1.1", "version": "5.1.1",
"resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz", "resolved": "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz",

1
package.json
View File

@@ -47,7 +47,6 @@
"@types/mkdirp": "^1.0.0", "@types/mkdirp": "^1.0.0",
"@types/node": "^13.9.8", "@types/node": "^13.9.8",
"@types/string-similarity": "^4.0.0", "@types/string-similarity": "^4.0.0",
"@zhangfuxing/unrar": "github:rishighan/unrar",
"axios": "^0.25.0", "axios": "^0.25.0",
"axios-retry": "^3.2.4", "axios-retry": "^3.2.4",
"bree": "^7.1.5", "bree": "^7.1.5",

172
utils/uncompression.utils.ts
View File

@@ -31,10 +31,11 @@ SOFTWARE.
* Initial: 2021/05/04 Rishi Ghan * Initial: 2021/05/04 Rishi Ghan
*/ */
import { promises as fs } from "fs"; import { createWriteStream, createReadStream, promises as fs } from "fs";
const fse = require("fs-extra"); const fse = require("fs-extra");
const Unrar = require("unrar"); const Unrar = require("unrar");
import path, { parse } from "path"; import path, { parse } from "path";
import { import {
IExtractComicBookCoverErrorResponse, IExtractComicBookCoverErrorResponse,
IExtractedComicBookCoverFile, IExtractedComicBookCoverFile,
@@ -49,7 +50,7 @@ import {
walkFolder, walkFolder,
} from "../utils/file.utils"; } from "../utils/file.utils";
import { resizeImage } from "./imagetransformation.utils"; import { resizeImage } from "./imagetransformation.utils";
import { each, filter, isNil, isUndefined, remove } from "lodash"; import { isNil, isUndefined, remove } from "lodash";
import { convertXMLToJSON } from "./xml.utils"; import { convertXMLToJSON } from "./xml.utils";
import sevenBin from "7zip-bin"; import sevenBin from "7zip-bin";
import { extract, list } from "node-7z"; import { extract, list } from "node-7z";
@@ -146,14 +147,11 @@ export const extractComicInfoXMLFromRar = async (
await fse.ensureDir(targetDirectory, directoryOptions); await fse.ensureDir(targetDirectory, directoryOptions);
console.info(`%s was created.`, targetDirectory); console.info(`%s was created.`, targetDirectory);
const archive = new Unrar(path.resolve(filePath)); const archive = new Unrar({
// or path: path.resolve(filePath),
// var archive = new Unrar({ bin: `/usr/local/bin/unrar`, // this will change depending on Docker base OS
// path: protectedArchivePath, });
// arguments: ['-pPassword'], archive.list(async (err, entries) => {
// bin: pathToUnrarBin // Default: unrar
// });
archive.list(function (err, entries) {
remove(entries, ({ type }) => type === "Directory"); remove(entries, ({ type }) => type === "Directory");
const comicInfoXML = remove( const comicInfoXML = remove(
entries, entries,
@@ -166,55 +164,43 @@ export const extractComicInfoXMLFromRar = async (
.localeCompare(b.name.toLowerCase()); .localeCompare(b.name.toLowerCase());
} }
}); });
const sharpStream = sharp().resize(200, 200); // Cover image extraction and resizing
const sharpStream = sharp().resize(275);
let comicInfoString = "";
const coverImageStream = archive const coverImageStream = archive
.stream(files[0].name) .stream(files[0].name)
.on("error", console.error) .on("error", console.error)
// .pipe(
// require("fs").createWriteStream(`${targetDirectory}/0.jpg`)
// );
.pipe(sharpStream) .pipe(sharpStream)
.toFile(`${targetDirectory}/0.jpg`, (err, info) => { .toFile(`${targetDirectory}/${files[0].name}`, (err, info) => {
if (err) { if (err) {
console.log("Failed to resize image:"); console.log("Failed to resize image:");
console.log(err); console.log(err);
return err; return err;
} }
console.log( console.log(
"Image file resized with the following parameters: %o", "Image file resized with the following parameters: %o",
info info
); );
return info; return info;
}); });
// ComicInfo.xml extraction and parsing to JSON
if (!isUndefined(comicInfoXML[0])) { if (!isUndefined(comicInfoXML[0])) {
console.log(comicInfoXML);
const comicinfoStream = archive.stream(comicInfoXML[0]["name"]); const comicinfoStream = archive.stream(comicInfoXML[0]["name"]);
comicinfoStream.on("error", console.error); comicinfoStream.on("error", console.error);
comicinfoStream.pipe( comicinfoStream.pipe(
require("fs").createWriteStream( createWriteStream(
`${targetDirectory}/dhanda.xml` `${targetDirectory}/${comicInfoXML[0]["name"]}`
) )
); );
comicinfoStream.on("data", (data) => {
comicInfoString += data;
});
comicinfoStream.on("end", async (data) => {
const foo = await convertXMLToJSON(comicInfoString);
console.log(foo);
});
} }
}); });
// const extracted = extractor.extract({
// files: ({ name }) => name.toLowerCase() === 'comicinfo.xml',
// });
// const files = [...extracted.files]; //load the files
// if (!isUndefined(files[0])) {
// console.log(
// `comicinfo.xml detected in ${filePath}, attempting extraction...`
// );
// const fileContents = String.fromCharCode.apply(
// null,
// files[0].extraction
// );
// const parsedJSON = await convertXMLToJSON(fileContents);
// console.log(parsedJSON);
// return parsedJSON.comicinfo;
// }
} catch (error) { } catch (error) {
throw new Error(error); throw new Error(error);
} }
@@ -225,68 +211,72 @@ export const extractComicInfoXMLFromZip = async (
outputDirectory: string outputDirectory: string
) => { ) => {
try { try {
const fileList = [];
const listStream = list(path.resolve(filePath), { const listStream = list(path.resolve(filePath), {
$cherryPick: ["*.png", "*.jpg", , "*.jpeg", "*.webp", "*.xml"],
$bin: pathTo7zip, $bin: pathTo7zip,
}); });
const fileList = []; listStream.on("data", (chunk) => fileList.push(chunk));
listStream listStream.on("end", async () => {
.on("data", (chunk) => fileList.push(chunk)) // Remove dotfiles and directories
.on("end", async () => { remove(
// Look for ComicInfo.xml fileList,
const comicInfoXML = remove(fileList, (item) => (item) =>
!isUndefined(item) item.attributes === "D...." ||
? path.basename(item.file).toLowerCase() === !isNil(item.file.match(/(?:^|[\\\/])(\.(?!\.)[^\\\/]+)$/g))
"comicinfo.xml" );
: undefined // Look for ComicInfo.xml, if present,
); // a. remove it from the fileList
// Sort the file list array naturally const comicInfoXML = remove(fileList, (item) =>
const sortedFileList = fileList.sort((a, b) => !isUndefined(item)
a.file.toLowerCase().localeCompare(b.file.toLowerCase()) ? path.basename(item.file).toLowerCase() === "comicinfo.xml"
); : undefined
);
// Sort the file list array naturally
const sortedFileList = fileList.sort((a, b) =>
a.file.toLowerCase().localeCompare(b.file.toLowerCase())
);
// Create the target directory // Create the target directory
const directoryOptions = { const directoryOptions = {
mode: 0o2775, mode: 0o2775,
}; };
const { fileNameWithoutExtension } = const { fileNameWithoutExtension } = getFileConstituents(filePath);
getFileConstituents(filePath); const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`;
const targetDirectory = `${USERDATA_DIRECTORY}/covers/${fileNameWithoutExtension}`; await fse.ensureDir(targetDirectory, directoryOptions);
await fse.ensureDir(targetDirectory, directoryOptions); console.info(`%s was created.`, targetDirectory);
console.info(`%s was created.`, targetDirectory);
// files to write // Prep the shortlist
const filesToWrite = []; const filesToWrite = [];
if ( if (
!isUndefined(sortedFileList[0]) && !isUndefined(sortedFileList[0]) &&
!isUndefined(sortedFileList[0].file) !isUndefined(sortedFileList[0].file)
) { ) {
filesToWrite.push(sortedFileList[0].file); filesToWrite.push(sortedFileList[0].file);
}
// b. if ComicInfo.xml present, include it in the file list to be written to disk
if (!isUndefined(comicInfoXML[0])) {
console.log(`ComicInfo.xml detected in ${filePath}`);
filesToWrite.push(comicInfoXML[0].file);
}
// Remove nulls, undefined and empty elements from the file list
const filteredFilesToWrite = filesToWrite.filter(
(item) => !isUndefined(item)
);
console.log(filesToWrite);
const extractStream = extract(
`${path.resolve(filePath)}`,
targetDirectory,
{
$cherryPick: [...filteredFilesToWrite],
$bin: pathTo7zip,
} }
);
// if ComicInfo.xml present, include it in the file list to be written to disk extractStream.on("data", (data) => {
if (!isUndefined(comicInfoXML[0])) { //do something with the image
console.log(`ComicInfo.xml detected in ${filePath}`); // console.log("ZIP:", data);
filesToWrite.push(comicInfoXML[0].file);
}
// Remove nulls, undefined and empty elements from the file list
const filteredFilesToWrite = filesToWrite.filter(
(item) => !isUndefined(item)
);
const extractStream = extract(
`${path.resolve(filePath)}`,
targetDirectory,
{
$cherryPick: filteredFilesToWrite,
$bin: pathTo7zip,
}
);
extractStream.on("data", (data) => {
//do something with the image
console.log("ZIP:", data);
});
}); });
});
// for await (const chunk of foo) { // for await (const chunk of foo) {
// if (chunk.status === "extracted") { // if (chunk.status === "extracted") {

1
utils/xml.utils.ts
View File

@@ -27,7 +27,6 @@ SOFTWARE.
*/ */
import xml2js from "xml2js"; import xml2js from "xml2js";
import fs from "fs";
export const convertXMLToJSON = (xmlPayload) => { export const convertXMLToJSON = (xmlPayload) => {
const parser = new xml2js.Parser({ const parser = new xml2js.Parser({

4
yarn.lock
View File

@@ -1102,10 +1102,6 @@
"@typescript-eslint/types" "4.33.0" "@typescript-eslint/types" "4.33.0"
"eslint-visitor-keys" "^2.0.0" "eslint-visitor-keys" "^2.0.0"
"@zhangfuxing/unrar@github:rishighan/unrar":
"resolved" "git+ssh://git@github.com/rishighan/unrar.git#e278e29d82d00827a016d34320373cbca1d099f1"
"version" "0.6.3"
"7zip-bin@^5.1.1": "7zip-bin@^5.1.1":
"integrity" "sha512-sAP4LldeWNz0lNzmTird3uWfFDWWTeg6V/MsmyyLR9X1idwKBWIgt/ZvinqQldJm3LecKEs1emkbquO6PCiLVQ==" "integrity" "sha512-sAP4LldeWNz0lNzmTird3uWfFDWWTeg6V/MsmyyLR9X1idwKBWIgt/ZvinqQldJm3LecKEs1emkbquO6PCiLVQ=="
"resolved" "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz" "resolved" "https://registry.npmjs.org/7zip-bin/-/7zip-bin-5.1.1.tgz"