frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

🪛 Data transferring over sockets!

Browse Source
This commit is contained in:
Rishi Ghan
2021-05-27 15:50:10 -07:00
parent ef4e1f75b0
commit 0aea022208
5 changed files with 268 additions and 448 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
package-lock.json generated
View File

@@ -1099,15 +1099,6 @@
"integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==", "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==",
"dev": true "dev": true
}, },
"@types/oboe": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@types/oboe/-/oboe-2.1.0.tgz",
"integrity": "sha512-2F95dk3QRgauL9gTUrydXkmnHA1tSCMzqwlSwXlfuBW8Y8F4IfgHecdTf7x4UX9sn3dpNKpfk297r+f/YHJXtA==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/pino": { "@types/pino": {
"version": "6.3.8", "version": "6.3.8",
"resolved": "https://registry.npmjs.org/@types/pino/-/pino-6.3.8.tgz", "resolved": "https://registry.npmjs.org/@types/pino/-/pino-6.3.8.tgz",
@@ -1507,14 +1498,6 @@
"integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==", "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==",
"dev": true "dev": true
}, },
"axios": {
"version": "0.21.1",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
"integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==",
"requires": {
"follow-redirects": "^1.10.0"
}
},
"babel-jest": { "babel-jest": {
"version": "25.5.1", "version": "25.5.1",
"resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-25.5.1.tgz", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-25.5.1.tgz",
@@ -3504,11 +3487,6 @@
"resolved": "https://registry.npmjs.org/fn-args/-/fn-args-5.0.0.tgz", "resolved": "https://registry.npmjs.org/fn-args/-/fn-args-5.0.0.tgz",
"integrity": "sha512-CtbfI3oFFc3nbdIoHycrfbrxiGgxXBXXuyOl49h47JawM1mYrqpiRqnH5CB2mBatdXvHHOUO6a+RiAuuvKt0lw==" "integrity": "sha512-CtbfI3oFFc3nbdIoHycrfbrxiGgxXBXXuyOl49h47JawM1mYrqpiRqnH5CB2mBatdXvHHOUO6a+RiAuuvKt0lw=="
}, },
"follow-redirects": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
"integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg=="
},
"for-in": { "for-in": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
@@ -3840,19 +3818,6 @@
} }
} }
}, },
"highland": {
"version": "2.13.5",
"resolved": "https://registry.npmjs.org/highland/-/highland-2.13.5.tgz",
"integrity": "sha512-dn2flPapIIAa4BtkB2ahjshg8iSJtrJtdhEb9/oiOrS5HMQTR/GuhFpqJ+11YBdtnl3AwWKvbZd1Uxr8uAmA7A==",
"requires": {
"util-deprecate": "^1.0.2"
}
},
"highland-json": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/highland-json/-/highland-json-1.4.1.tgz",
"integrity": "sha512-rZDAxg5gfSQVnuQ6LHnrBzouxt9WNM4uNiFAp8X+LDuz1TiO63rnf5wPnDWkJLzEnbSsrN7ks3lIIibmbNuKmg=="
},
"hosted-git-info": { "hosted-git-info": {
"version": "2.8.9", "version": "2.8.9",
"resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
@@ -3893,11 +3858,6 @@
} }
} }
}, },
"http-https": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/http-https/-/http-https-1.0.0.tgz",
"integrity": "sha1-L5CN1fHbQGjAWM1ubUzjkskTOJs="
},
"http-signature": { "http-signature": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -5702,19 +5662,6 @@
"integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=",
"dev": true "dev": true
}, },
"json-stream-stringify": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/json-stream-stringify/-/json-stream-stringify-2.0.3.tgz",
"integrity": "sha512-Ry+1rZE1YVKlCMG1emCiReP/OfZrFrEGZn6TC7NPpIGO9NXf0KEqqBL0flgt2J59EiRPv+CKi7S7v31RAHrdXw=="
},
"json-streamify": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/json-streamify/-/json-streamify-0.1.4.tgz",
"integrity": "sha1-HBgHSnAJv3vS0hRDzEJbmGTNCJ0=",
"requires": {
"traverse": ">=0.2.6"
}
},
"json-stringify-safe": { "json-stringify-safe": {
"version": "5.0.1", "version": "5.0.1",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
@@ -6725,14 +6672,6 @@
"has": "^1.0.3" "has": "^1.0.3"
} }
}, },
"oboe": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/oboe/-/oboe-2.1.5.tgz",
"integrity": "sha1-VVQoTFQ6ImbXo48X4HOCH73jk80=",
"requires": {
"http-https": "^1.0.0"
}
},
"on-finished": { "on-finished": {
"version": "2.3.0", "version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@@ -8542,19 +8481,6 @@
"integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=",
"dev": true "dev": true
}, },
"stream-chain": {
"version": "2.2.4",
"resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.4.tgz",
"integrity": "sha512-9lsl3YM53V5N/I1C2uJtc3Kavyi3kNYN83VkKb/bMWRk7D9imiFyUPYa0PoZbLohSVOX1mYE9YsmwObZUsth6Q=="
},
"stream-json": {
"version": "1.7.1",
"resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.7.1.tgz",
"integrity": "sha512-I7g0IDqvdJXbJ279/D3ZoTx0VMhmKnEF7u38CffeWdF8bfpMPsLo+5fWnkNjO2GU/JjWaRjdH+zmH03q+XGXFw==",
"requires": {
"stream-chain": "^2.2.3"
}
},
"streamsearch": { "streamsearch": {
"version": "0.1.2", "version": "0.1.2",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
@@ -8821,26 +8747,6 @@
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
}, },
"through2": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz",
"integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==",
"requires": {
"readable-stream": "3"
},
"dependencies": {
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"timers-ext": { "timers-ext": {
"version": "0.1.7", "version": "0.1.7",
"resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz",

10
package.json
View File

@@ -18,7 +18,6 @@
"author": "", "author": "",
"devDependencies": { "devDependencies": {
"@types/lodash": "^4.14.168", "@types/lodash": "^4.14.168",
"@types/oboe": "^2.1.0",
"@types/unzipper": "^0.10.3", "@types/unzipper": "^0.10.3",
"@typescript-eslint/eslint-plugin": "^2.26.0", "@typescript-eslint/eslint-plugin": "^2.26.0",
"@typescript-eslint/parser": "^2.26.0", "@typescript-eslint/parser": "^2.26.0",
@@ -38,12 +37,7 @@
"@types/node": "^13.9.8", "@types/node": "^13.9.8",
"@types/pino": "^6.3.8", "@types/pino": "^6.3.8",
"JSONStream": "^1.3.5", "JSONStream": "^1.3.5",
"axios": "^0.21.1",
"fs-extra": "^10.0.0", "fs-extra": "^10.0.0",
"highland": "^2.13.5",
"highland-json": "^1.4.1",
"json-stream-stringify": "^2.0.3",
"json-streamify": "^0.1.4",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"moleculer": "^0.14.0", "moleculer": "^0.14.0",
"moleculer-db": "^0.8.4", "moleculer-db": "^0.8.4",
@@ -54,15 +48,11 @@
"mongoose-paginate": "^5.0.3", "mongoose-paginate": "^5.0.3",
"nats": "^1.3.2", "nats": "^1.3.2",
"node-unrar-js": "^1.0.2", "node-unrar-js": "^1.0.2",
"oboe": "^2.1.5",
"pino": "^6.11.3", "pino": "^6.11.3",
"pino-pretty": "^4.7.1", "pino-pretty": "^4.7.1",
"sharp": "^0.28.1", "sharp": "^0.28.1",
"socket.io": "^4.1.1", "socket.io": "^4.1.1",
"socket.io-stream": "^0.9.1", "socket.io-stream": "^0.9.1",
"stream-chain": "^2.2.4",
"stream-json": "^1.7.1",
"through2": "^4.0.2",
"typescript": "^3.8.3", "typescript": "^3.8.3",
"unzipper": "^0.10.11" "unzipper": "^0.10.11"
}, },

43
services/api.service.ts
View File

@@ -1,16 +1,8 @@
import { IncomingMessage } from "http";
import fs from "fs";
import path from "path";
import { Service, ServiceBroker, Context } from "moleculer"; import { Service, ServiceBroker, Context } from "moleculer";
import ApiGateway from "moleculer-web"; import ApiGateway from "moleculer-web";
import { getCovers, extractArchive } from "../utils/uncompression.utils"; import { extractArchive } from "../utils/uncompression.utils";
import { map, flatten } from "lodash"; import { map } from "lodash";
import JSONStream from "JSONStream";
const IO = require("socket.io")(); const IO = require("socket.io")();
const ss = require("socket.io-stream");
const JsonStreamStringify = require("json-stream-stringify");
import axios from "axios";
const { Writable, Readable } = require("stream");
export default class ApiService extends Service { export default class ApiService extends Service {
public constructor(broker: ServiceBroker) { public constructor(broker: ServiceBroker) {
@@ -34,12 +26,7 @@ export default class ApiService extends Service {
mergeParams: true, mergeParams: true,
autoAliases: true, autoAliases: true,
aliases: { aliases: {},
async "POST getComicCovers"(req, res) {
const { extractionOptions, walkedFolders } =
req.body;
},
},
// Calling options. More info: https://moleculer.services/docs/0.14/moleculer-web.html#Calling-options // Calling options. More info: https://moleculer.services/docs/0.14/moleculer-web.html#Calling-options
callingOptions: {}, callingOptions: {},
@@ -105,25 +92,20 @@ export default class ApiService extends Service {
params params
); );
const { extractionOptions, walkedFolders } = params; const { extractionOptions, walkedFolders } = params;
const stream = ss.createStream();
switch (extractionOptions.extractionMode) { switch (extractionOptions.extractionMode) {
case "bulk": case "bulk":
map(walkedFolders, async (folder, idx) => { map(walkedFolders, async (folder, idx) => {
let foo = await extractArchive( let comicBookCoverMetadata =
extractionOptions, await extractArchive(
folder extractionOptions,
); folder
);
let fo = new JsonStreamStringify({
foo,
});
client.emit("comicBookCoverMetadata", { client.emit("comicBookCoverMetadata", {
data: foo, data: comicBookCoverMetadata,
status: "Done!", status: "Done!",
}); });
}); });
// res.end();
case "single": case "single":
return await extractArchive( return await extractArchive(
@@ -141,13 +123,6 @@ export default class ApiService extends Service {
data: `${extractionOptions}`, data: `${extractionOptions}`,
}; };
} }
// this.broker
// .call("import." + action, params, opts)
// .then((resp) => {
// // client.emit("comicBookCoverMetadata", resp);
// })
// .catch((err) => this.logger.error(err));
} }
); );

78
services/import.service.ts
View File

@@ -1,33 +1,8 @@
"use strict"; "use strict";
import { Context, Service, ServiceBroker, ServiceSchema } from "moleculer"; import { Context, Service, ServiceBroker, ServiceSchema } from "moleculer";
import fs from "fs";
import { DbMixin } from "../mixins/db.mixin"; import { DbMixin } from "../mixins/db.mixin";
import Comic from "../models/comic.model"; import Comic from "../models/comic.model";
import { map, flatten, isUndefined } from "lodash"; import { walkFolder } from "../utils/uncompression.utils";
import {
extractArchive,
getCovers,
walkFolder,
} from "../utils/uncompression.utils";
import {
IExtractionOptions,
IExtractedComicBookCoverFile,
IFolderData,
} from "../interfaces/folder.interface";
import axios from "axios";
import { Readable } from "stream";
import through2 from "through2";
import oboe from "oboe";
import H from "highland";
import { stringify } from "highland-json";
const JsonStreamStringify = require("json-stream-stringify");
const IO = require("socket.io")();
const { chain } = require("stream-chain");
const { parser } = require("stream-json");
const { pick } = require("stream-json/filters/Pick");
const { ignore } = require("stream-json/filters/Ignore");
const { streamValues } = require("stream-json/streamers/StreamValues");
const StreamArray = require("stream-json/streamers/StreamArray");
export default class ProductsService extends Service { export default class ProductsService extends Service {
// @ts-ignore // @ts-ignore
@@ -66,57 +41,6 @@ export default class ProductsService extends Service {
); );
}, },
}, },
// getComicCovers: {
// rest: "POST /getComicCovers",
// params: {
// extractionOptions: "object",
// walkedFolders: "array",
// },
// async handler(
// ctx: Context<{
// extractionOptions: IExtractionOptions;
// walkedFolders: IFolderData[];
// }>
// ) {
// switch (
// ctx.params.extractionOptions.extractionMode
// ) {
// case "bulk":
// map(
// ctx.params.walkedFolders,
// async (folder, idx) => {
// let foo = await extractArchive(
// ctx.params
// .extractionOptions,
// folder
// );
// // console.log("levar", foo);
// let jsonStream =
// new JsonStreamStringify({
// foo,
// });
// return jsonStream;
// }
// );
//
// case "single":
// return await extractArchive(
// ctx.params.extractionOptions,
// ctx.params.walkedFolders[0]
// );
// default:
// console.log(
// "Unknown extraction mode selected."
// );
// return {
// message:
// "Unknown extraction mode selected.",
// errorCode: "90",
// data: `${ctx.params.extractionOptions}`,
// };
// }
// },
// },
}, },
methods: {}, methods: {},
}, },

491
utils/uncompression.utils.ts
View File

@@ -37,11 +37,11 @@ import { default as unzipper } from "unzipper";
import _ from "lodash"; import _ from "lodash";
import { each, isEmpty, map, remove, indexOf } from "lodash"; import { each, isEmpty, map, remove, indexOf } from "lodash";
import { import {
IExplodedPathResponse, IExplodedPathResponse,
IExtractComicBookCoverErrorResponse, IExtractComicBookCoverErrorResponse,
IExtractedComicBookCoverFile, IExtractedComicBookCoverFile,
IExtractionOptions, IExtractionOptions,
IFolderData, IFolderData,
} from "../interfaces/folder.interface"; } from "../interfaces/folder.interface";
import { logger } from "./logger.utils"; import { logger } from "./logger.utils";
const { writeFile, readFile } = require("fs").promises; const { writeFile, readFile } = require("fs").promises;
@@ -50,274 +50,299 @@ const unrarer = require("node-unrar-js");
const Walk = require("@root/walk"); const Walk = require("@root/walk");
const fse = require("fs-extra"); const fse = require("fs-extra");
export const unrar = async ( export const unrar = async (
extractionOptions: IExtractionOptions, extractionOptions: IExtractionOptions,
walkedFolder: IFolderData, walkedFolder: IFolderData
): Promise< ): Promise<
| IExtractedComicBookCoverFile | IExtractedComicBookCoverFile
| IExtractedComicBookCoverFile[] | IExtractedComicBookCoverFile[]
| IExtractComicBookCoverErrorResponse | IExtractComicBookCoverErrorResponse
> => { > => {
const paths = constructPaths(extractionOptions, walkedFolder); const paths = constructPaths(extractionOptions, walkedFolder);
const directoryOptions = { const directoryOptions = {
mode: 0o2775, mode: 0o2775,
}; };
const fileBuffer = await readFile(paths.inputFilePath).catch(err => const fileBuffer = await readFile(paths.inputFilePath).catch((err) =>
console.error("Failed to read file", err), console.error("Failed to read file", err)
); );
try { try {
await fse.ensureDir(paths.targetPath, directoryOptions); await fse.ensureDir(paths.targetPath, directoryOptions);
logger.info(`${paths.targetPath} was created.`); logger.info(`${paths.targetPath} was created.`);
} catch (error) { } catch (error) {
logger.error(`${error}: Couldn't create directory.`); logger.error(`${error}: Couldn't create directory.`);
} }
const extractor = await unrarer.createExtractorFromData({ data: fileBuffer }); const extractor = await unrarer.createExtractorFromData({
data: fileBuffer,
});
switch (extractionOptions.extractTarget) { switch (extractionOptions.extractTarget) {
case "cover": case "cover":
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
try { try {
let fileNameToExtract = ""; let fileNameToExtract = "";
const list = extractor.getFileList(); const list = extractor.getFileList();
const fileHeaders = [...list.fileHeaders]; const fileHeaders = [...list.fileHeaders];
each(fileHeaders, async fileHeader => { each(fileHeaders, async (fileHeader) => {
const fileName = explodePath(fileHeader.name).fileName; const fileName = explodePath(fileHeader.name).fileName;
if ( if (
fileName !== "" && fileName !== "" &&
fileHeader.flags.directory === false && fileHeader.flags.directory === false &&
isEmpty(fileNameToExtract) isEmpty(fileNameToExtract)
) { ) {
logger.info(`Attempting to write ${fileHeader.name}`); logger.info(
fileNameToExtract = fileHeader.name; `Attempting to write ${fileHeader.name}`
const file = extractor.extract({ files: [fileHeader.name] }); );
const extractedFile = [...file.files][0]; fileNameToExtract = fileHeader.name;
const fileArrayBuffer = extractedFile.extraction; const file = extractor.extract({
await writeFile( files: [fileHeader.name],
paths.targetPath + "/" + fileName, });
fileArrayBuffer, const extractedFile = [...file.files][0];
); const fileArrayBuffer = extractedFile.extraction;
resolve({ await writeFile(
name: `${fileName}`, paths.targetPath + "/" + fileName,
path: paths.targetPath, fileArrayBuffer
fileSize: fileHeader.packSize, );
}); resolve({
} name: `${fileName}`,
}); path: paths.targetPath,
} catch (error) { fileSize: fileHeader.packSize,
logger.error(`${error}: Couldn't write file.`); });
reject(error); }
} });
}); } catch (error) {
logger.error(`${error}: Couldn't write file.`);
reject(error);
}
});
case "all": case "all":
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
try { try {
const files = extractor.extract({}); const files = extractor.extract({});
const extractedFiles = [...files.files]; const extractedFiles = [...files.files];
const comicBookCoverFiles: IExtractedComicBookCoverFile[] = []; const comicBookCoverFiles: IExtractedComicBookCoverFile[] =
for (const file of extractedFiles) { [];
logger.info(`Attempting to write ${file.fileHeader.name}`); for (const file of extractedFiles) {
const fileBuffer = file.extraction; logger.info(
const fileName = explodePath(file.fileHeader.name).fileName; `Attempting to write ${file.fileHeader.name}`
);
const fileBuffer = file.extraction;
const fileName = explodePath(
file.fileHeader.name
).fileName;
if (fileName !== "" && file.fileHeader.flags.directory === false) { if (
await writeFile(paths.targetPath + "/" + fileName, fileBuffer); fileName !== "" &&
} file.fileHeader.flags.directory === false
comicBookCoverFiles.push({ ) {
name: `${file.fileHeader.name}`, await writeFile(
path: paths.targetPath, paths.targetPath + "/" + fileName,
fileSize: file.fileHeader.packSize, fileBuffer
}); );
} }
resolve(_.flatten(comicBookCoverFiles)); comicBookCoverFiles.push({
} catch (error) { name: `${file.fileHeader.name}`,
resolve({ path: paths.targetPath,
message: `${error}`, fileSize: file.fileHeader.packSize,
errorCode: "500", });
data: walkedFolder.name, }
}); resolve(_.flatten(comicBookCoverFiles));
} } catch (error) {
}); resolve({
message: `${error}`,
errorCode: "500",
data: walkedFolder.name,
});
}
});
default: default:
return { return {
message: "File format not supported, yet.", message: "File format not supported, yet.",
errorCode: "90", errorCode: "90",
data: "asda", data: "asda",
}; };
} }
}; };
export const unzip = async ( export const unzip = async (
extractionOptions: IExtractionOptions, extractionOptions: IExtractionOptions,
walkedFolder: IFolderData, walkedFolder: IFolderData
): Promise< ): Promise<
| IExtractedComicBookCoverFile[] | IExtractedComicBookCoverFile[]
| IExtractedComicBookCoverFile | IExtractedComicBookCoverFile
| IExtractComicBookCoverErrorResponse | IExtractComicBookCoverErrorResponse
> => { > => {
const directoryOptions = { const directoryOptions = {
mode: 0o2775, mode: 0o2775,
}; };
const paths = constructPaths(extractionOptions, walkedFolder); const paths = constructPaths(extractionOptions, walkedFolder);
try { try {
await fse.ensureDir(paths.targetPath, directoryOptions); await fse.ensureDir(paths.targetPath, directoryOptions);
logger.info(`${paths.targetPath} was created or already exists.`); logger.info(`${paths.targetPath} was created or already exists.`);
} catch (error) { } catch (error) {
logger.error(`${error} Couldn't create directory.`); logger.error(`${error} Couldn't create directory.`);
} }
const extractedFiles: IExtractedComicBookCoverFile[] = []; const extractedFiles: IExtractedComicBookCoverFile[] = [];
const zip = createReadStream(paths.inputFilePath).pipe( const zip = createReadStream(paths.inputFilePath).pipe(
unzipper.Parse({ forceStream: true }), unzipper.Parse({ forceStream: true })
); );
for await (const entry of zip) { for await (const entry of zip) {
const fileName = explodePath(entry.path).fileName; const fileName = explodePath(entry.path).fileName;
const size = entry.vars.uncompressedSize; const size = entry.vars.uncompressedSize;
if ( if (
extractedFiles.length === 1 && extractedFiles.length === 1 &&
extractionOptions.extractTarget === "cover" extractionOptions.extractTarget === "cover"
) { ) {
break; break;
} }
if (fileName !== "" && entry.type !== "Directory") { if (fileName !== "" && entry.type !== "Directory") {
logger.info(`Attempting to write ${fileName}`); logger.info(`Attempting to write ${fileName}`);
entry.pipe(createWriteStream(paths.targetPath + "/" + fileName)); entry.pipe(createWriteStream(paths.targetPath + "/" + fileName));
extractedFiles.push({ extractedFiles.push({
name: fileName, name: fileName,
fileSize: size, fileSize: size,
path: paths.targetPath, path: paths.targetPath,
}); });
} }
entry.autodrain(); entry.autodrain();
} }
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
logger.info(""); logger.info("");
resolve(_.flatten(extractedFiles)); resolve(_.flatten(extractedFiles));
}); });
}; };
export const extractArchive = async ( export const extractArchive = async (
extractionOptions: IExtractionOptions, extractionOptions: IExtractionOptions,
walkedFolder: IFolderData, walkedFolder: IFolderData
): Promise< ): Promise<
| IExtractedComicBookCoverFile | IExtractedComicBookCoverFile
| IExtractedComicBookCoverFile[] | IExtractedComicBookCoverFile[]
| IExtractComicBookCoverErrorResponse | IExtractComicBookCoverErrorResponse
> => { > => {
switch (walkedFolder.extension) { switch (walkedFolder.extension) {
case ".cbz": case ".cbz":
return await unzip(extractionOptions, walkedFolder); return await unzip(extractionOptions, walkedFolder);
case ".cbr": case ".cbr":
return await unrar(extractionOptions, walkedFolder); return await unrar(extractionOptions, walkedFolder);
default: default:
return { return {
message: "File format not supported, yet.", message: "File format not supported, yet.",
errorCode: "90", errorCode: "90",
data: `${extractionOptions}`, data: `${extractionOptions}`,
}; };
} }
}; };
export const getCovers = async ( export const getCovers = async (
options: IExtractionOptions, options: IExtractionOptions,
walkedFolders: IFolderData[], walkedFolders: IFolderData[]
): Promise< ): Promise<
| IExtractedComicBookCoverFile | IExtractedComicBookCoverFile
| IExtractComicBookCoverErrorResponse | IExtractComicBookCoverErrorResponse
| IExtractedComicBookCoverFile[] | IExtractedComicBookCoverFile[]
| ( | (
| IExtractedComicBookCoverFile | IExtractedComicBookCoverFile
| IExtractComicBookCoverErrorResponse | IExtractComicBookCoverErrorResponse
| IExtractedComicBookCoverFile[] | IExtractedComicBookCoverFile[]
)[] )[]
| IExtractComicBookCoverErrorResponse | IExtractComicBookCoverErrorResponse
> => { > => {
switch (options.extractionMode) { switch (options.extractionMode) {
case "bulk": case "bulk":
const extractedDataPromises = map(walkedFolders, async folder => await extractArchive(options, folder)); const extractedDataPromises = map(
return Promise.all(extractedDataPromises).then(data => _.flatten(data)); walkedFolders,
case "single": async (folder) => await extractArchive(options, folder)
return await extractArchive(options, walkedFolders[0]); );
default: return Promise.all(extractedDataPromises).then((data) =>
logger.error("Unknown extraction mode selected."); _.flatten(data)
return { );
message: "Unknown extraction mode selected.", case "single":
errorCode: "90", return await extractArchive(options, walkedFolders[0]);
data: `${options}`, default:
}; logger.error("Unknown extraction mode selected.");
} return {
message: "Unknown extraction mode selected.",
errorCode: "90",
data: `${options}`,
};
}
}; };
export const walkFolder = async (folder: string): Promise<IFolderData[]> => { export const walkFolder = async (folder: string): Promise<IFolderData[]> => {
const result: IFolderData[] = []; const result: IFolderData[] = [];
let walkResult: IFolderData = { let walkResult: IFolderData = {
name: "", name: "",
extension: "", extension: "",
containedIn: "", containedIn: "",
isFile: false, isFile: false,
isLink: true, isLink: true,
}; };
const walk = Walk.create({ sort: filterOutDotFiles }); const walk = Walk.create({ sort: filterOutDotFiles });
await walk(folder, async (err, pathname, dirent) => { await walk(folder, async (err, pathname, dirent) => {
if (err) { if (err) {
logger.error("Failed to lstat directory", { error: err }); logger.error("Failed to lstat directory", { error: err });
return false; return false;
} }
if ([".cbz", ".cbr"].includes(path.extname(dirent.name))) { if ([".cbz", ".cbr"].includes(path.extname(dirent.name))) {
walkResult = { walkResult = {
name: path.basename(dirent.name, path.extname(dirent.name)), name: path.basename(dirent.name, path.extname(dirent.name)),
extension: path.extname(dirent.name), extension: path.extname(dirent.name),
containedIn: path.dirname(pathname), containedIn: path.dirname(pathname),
isFile: dirent.isFile(), isFile: dirent.isFile(),
isLink: dirent.isSymbolicLink(), isLink: dirent.isSymbolicLink(),
}; };
logger.info( logger.info(
`Scanned ${dirent.name} contained in ${path.dirname(pathname)}`, `Scanned ${dirent.name} contained in ${path.dirname(pathname)}`
); );
result.push(walkResult); result.push(walkResult);
} }
}); });
return result; return result;
}; };
export const explodePath = (filePath: string): IExplodedPathResponse => { export const explodePath = (filePath: string): IExplodedPathResponse => {
const exploded = filePath.split("/"); const exploded = filePath.split("/");
const fileName = remove(exploded, item => indexOf(exploded, item) === exploded.length - 1).join(""); const fileName = remove(
exploded,
(item) => indexOf(exploded, item) === exploded.length - 1
).join("");
return { return {
exploded, exploded,
fileName, fileName,
}; };
}; };
const constructPaths = ( const constructPaths = (
extractionOptions: IExtractionOptions, extractionOptions: IExtractionOptions,
walkedFolder: IFolderData, walkedFolder: IFolderData
) => ({ ) => ({
targetPath: targetPath:
extractionOptions.targetExtractionFolder + "/" + walkedFolder.name, extractionOptions.targetExtractionFolder + "/" + walkedFolder.name,
inputFilePath: inputFilePath:
walkedFolder.containedIn + walkedFolder.containedIn +
"/" + "/" +
walkedFolder.name + walkedFolder.name +
walkedFolder.extension, walkedFolder.extension,
}); });
export const extractMetadataFromImage = async ( export const extractMetadataFromImage = async (
imageFilePath: string, imageFilePath: string
): Promise<unknown> => { ): Promise<unknown> => {
const image = await sharp(imageFilePath) const image = await sharp(imageFilePath)
.metadata() .metadata()
.then(function(metadata) { .then(function (metadata) {
return metadata; return metadata;
}); });
return image; return image;
}; };
const filterOutDotFiles = entities => entities.filter(ent => !ent.name.startsWith(".")); const filterOutDotFiles = (entities) =>
entities.filter((ent) => !ent.name.startsWith("."));