🪛 Data transferring over sockets!

package-lock.json

@@ -1099,15 +1099,6 @@
       "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==",
       "dev": true
     },
-    "@types/oboe": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@types/oboe/-/oboe-2.1.0.tgz",
-      "integrity": "sha512-2F95dk3QRgauL9gTUrydXkmnHA1tSCMzqwlSwXlfuBW8Y8F4IfgHecdTf7x4UX9sn3dpNKpfk297r+f/YHJXtA==",
-      "dev": true,
-      "requires": {
-        "@types/node": "*"
-      }
-    },
     "@types/pino": {
       "version": "6.3.8",
       "resolved": "https://registry.npmjs.org/@types/pino/-/pino-6.3.8.tgz",
@@ -1507,14 +1498,6 @@
       "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==",
       "dev": true
     },
-    "axios": {
-      "version": "0.21.1",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
-      "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==",
-      "requires": {
-        "follow-redirects": "^1.10.0"
-      }
-    },
     "babel-jest": {
       "version": "25.5.1",
       "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-25.5.1.tgz",
@@ -3504,11 +3487,6 @@
       "resolved": "https://registry.npmjs.org/fn-args/-/fn-args-5.0.0.tgz",
       "integrity": "sha512-CtbfI3oFFc3nbdIoHycrfbrxiGgxXBXXuyOl49h47JawM1mYrqpiRqnH5CB2mBatdXvHHOUO6a+RiAuuvKt0lw=="
     },
-    "follow-redirects": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz",
-      "integrity": "sha512-HWqDgT7ZEkqRzBvc2s64vSZ/hfOceEol3ac/7tKwzuvEyWx3/4UegXh5oBOIotkGsObyk3xznnSRVADBgWSQVg=="
-    },
     "for-in": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
@@ -3840,19 +3818,6 @@
         }
       }
     },
-    "highland": {
-      "version": "2.13.5",
-      "resolved": "https://registry.npmjs.org/highland/-/highland-2.13.5.tgz",
-      "integrity": "sha512-dn2flPapIIAa4BtkB2ahjshg8iSJtrJtdhEb9/oiOrS5HMQTR/GuhFpqJ+11YBdtnl3AwWKvbZd1Uxr8uAmA7A==",
-      "requires": {
-        "util-deprecate": "^1.0.2"
-      }
-    },
-    "highland-json": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/highland-json/-/highland-json-1.4.1.tgz",
-      "integrity": "sha512-rZDAxg5gfSQVnuQ6LHnrBzouxt9WNM4uNiFAp8X+LDuz1TiO63rnf5wPnDWkJLzEnbSsrN7ks3lIIibmbNuKmg=="
-    },
     "hosted-git-info": {
       "version": "2.8.9",
       "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
@@ -3893,11 +3858,6 @@
         }
       }
     },
-    "http-https": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/http-https/-/http-https-1.0.0.tgz",
-      "integrity": "sha1-L5CN1fHbQGjAWM1ubUzjkskTOJs="
-    },
     "http-signature": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -5702,19 +5662,6 @@
       "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=",
       "dev": true
     },
-    "json-stream-stringify": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/json-stream-stringify/-/json-stream-stringify-2.0.3.tgz",
-      "integrity": "sha512-Ry+1rZE1YVKlCMG1emCiReP/OfZrFrEGZn6TC7NPpIGO9NXf0KEqqBL0flgt2J59EiRPv+CKi7S7v31RAHrdXw=="
-    },
-    "json-streamify": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/json-streamify/-/json-streamify-0.1.4.tgz",
-      "integrity": "sha1-HBgHSnAJv3vS0hRDzEJbmGTNCJ0=",
-      "requires": {
-        "traverse": ">=0.2.6"
-      }
-    },
     "json-stringify-safe": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
@@ -6725,14 +6672,6 @@
         "has": "^1.0.3"
       }
     },
-    "oboe": {
-      "version": "2.1.5",
-      "resolved": "https://registry.npmjs.org/oboe/-/oboe-2.1.5.tgz",
-      "integrity": "sha1-VVQoTFQ6ImbXo48X4HOCH73jk80=",
-      "requires": {
-        "http-https": "^1.0.0"
-      }
-    },
     "on-finished": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@@ -8542,19 +8481,6 @@
       "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=",
       "dev": true
     },
-    "stream-chain": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.4.tgz",
-      "integrity": "sha512-9lsl3YM53V5N/I1C2uJtc3Kavyi3kNYN83VkKb/bMWRk7D9imiFyUPYa0PoZbLohSVOX1mYE9YsmwObZUsth6Q=="
-    },
-    "stream-json": {
-      "version": "1.7.1",
-      "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.7.1.tgz",
-      "integrity": "sha512-I7g0IDqvdJXbJ279/D3ZoTx0VMhmKnEF7u38CffeWdF8bfpMPsLo+5fWnkNjO2GU/JjWaRjdH+zmH03q+XGXFw==",
-      "requires": {
-        "stream-chain": "^2.2.3"
-      }
-    },
     "streamsearch": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
@@ -8821,26 +8747,6 @@
       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
       "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
     },
-    "through2": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz",
-      "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==",
-      "requires": {
-        "readable-stream": "3"
-      },
-      "dependencies": {
-        "readable-stream": {
-          "version": "3.6.0",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
-          "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
-          "requires": {
-            "inherits": "^2.0.3",
-            "string_decoder": "^1.1.1",
-            "util-deprecate": "^1.0.1"
-          }
-        }
-      }
-    },
     "timers-ext": {
       "version": "0.1.7",
       "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz",

package.json

@@ -18,7 +18,6 @@
   "author": "",
   "devDependencies": {
     "@types/lodash": "^4.14.168",
-    "@types/oboe": "^2.1.0",
     "@types/unzipper": "^0.10.3",
     "@typescript-eslint/eslint-plugin": "^2.26.0",
     "@typescript-eslint/parser": "^2.26.0",
@@ -38,12 +37,7 @@
     "@types/node": "^13.9.8",
     "@types/pino": "^6.3.8",
     "JSONStream": "^1.3.5",
-    "axios": "^0.21.1",
     "fs-extra": "^10.0.0",
-    "highland": "^2.13.5",
-    "highland-json": "^1.4.1",
-    "json-stream-stringify": "^2.0.3",
-    "json-streamify": "^0.1.4",
     "lodash": "^4.17.21",
     "moleculer": "^0.14.0",
     "moleculer-db": "^0.8.4",
@@ -54,15 +48,11 @@
     "mongoose-paginate": "^5.0.3",
     "nats": "^1.3.2",
     "node-unrar-js": "^1.0.2",
-    "oboe": "^2.1.5",
     "pino": "^6.11.3",
     "pino-pretty": "^4.7.1",
     "sharp": "^0.28.1",
     "socket.io": "^4.1.1",
     "socket.io-stream": "^0.9.1",
-    "stream-chain": "^2.2.4",
-    "stream-json": "^1.7.1",
-    "through2": "^4.0.2",
     "typescript": "^3.8.3",
     "unzipper": "^0.10.11"
   },

services/api.service.ts

@@ -1,16 +1,8 @@
-import { IncomingMessage } from "http";
-import fs from "fs";
-import path from "path";
 import { Service, ServiceBroker, Context } from "moleculer";
 import ApiGateway from "moleculer-web";
-import { getCovers, extractArchive } from "../utils/uncompression.utils";
+import { extractArchive } from "../utils/uncompression.utils";
-import { map, flatten } from "lodash";
+import { map } from "lodash";
-import JSONStream from "JSONStream";
 const IO = require("socket.io")();
-const ss = require("socket.io-stream");
-const JsonStreamStringify = require("json-stream-stringify");
-import axios from "axios";
-const { Writable, Readable } = require("stream");
 
 export default class ApiService extends Service {
 	public constructor(broker: ServiceBroker) {
@@ -34,12 +26,7 @@ export default class ApiService extends Service {
 						mergeParams: true,
 						autoAliases: true,
 
-						aliases: {
+						aliases: {},
-							async "POST getComicCovers"(req, res) {
-								const { extractionOptions, walkedFolders } =
-									req.body;
-							},
-						},
 
 						// Calling options. More info: https://moleculer.services/docs/0.14/moleculer-web.html#Calling-options
 						callingOptions: {},
@@ -105,25 +92,20 @@ export default class ApiService extends Service {
 								params
 							);
 							const { extractionOptions, walkedFolders } = params;
-							const stream = ss.createStream();
 							switch (extractionOptions.extractionMode) {
 								case "bulk":
 									map(walkedFolders, async (folder, idx) => {
-										let foo = await extractArchive(
+										let comicBookCoverMetadata =
+											await extractArchive(
 												extractionOptions,
 												folder
 											);
 
-										let fo = new JsonStreamStringify({
-											foo,
-										});
-
 										client.emit("comicBookCoverMetadata", {
-											data: foo,
+											data: comicBookCoverMetadata,
 											status: "Done!",
 										});
 									});
-								// res.end();
 
 								case "single":
 									return await extractArchive(
@@ -141,13 +123,6 @@ export default class ApiService extends Service {
 										data: `${extractionOptions}`,
 									};
 							}
-
-							// this.broker
-							// 	.call("import." + action, params, opts)
-							// 	.then((resp) => {
-							// 		// client.emit("comicBookCoverMetadata", resp);
-							// 	})
-							// 	.catch((err) => this.logger.error(err));
 						}
 					);
 

services/import.service.ts

@@ -1,33 +1,8 @@
 "use strict";
 import { Context, Service, ServiceBroker, ServiceSchema } from "moleculer";
-import fs from "fs";
 import { DbMixin } from "../mixins/db.mixin";
 import Comic from "../models/comic.model";
-import { map, flatten, isUndefined } from "lodash";
+import { walkFolder } from "../utils/uncompression.utils";
-import {
-	extractArchive,
-	getCovers,
-	walkFolder,
-} from "../utils/uncompression.utils";
-import {
-	IExtractionOptions,
-	IExtractedComicBookCoverFile,
-	IFolderData,
-} from "../interfaces/folder.interface";
-import axios from "axios";
-import { Readable } from "stream";
-import through2 from "through2";
-import oboe from "oboe";
-import H from "highland";
-import { stringify } from "highland-json";
-const JsonStreamStringify = require("json-stream-stringify");
-const IO = require("socket.io")();
-const { chain } = require("stream-chain");
-const { parser } = require("stream-json");
-const { pick } = require("stream-json/filters/Pick");
-const { ignore } = require("stream-json/filters/Ignore");
-const { streamValues } = require("stream-json/streamers/StreamValues");
-const StreamArray = require("stream-json/streamers/StreamArray");
 
 export default class ProductsService extends Service {
 	// @ts-ignore
@@ -66,57 +41,6 @@ export default class ProductsService extends Service {
 								);
 							},
 						},
-						// 						getComicCovers: {
-						// 							rest: "POST /getComicCovers",
-						// 							params: {
-						// 								extractionOptions: "object",
-						// 								walkedFolders: "array",
-						// 							},
-						// 							async handler(
-						// 								ctx: Context<{
-						// 									extractionOptions: IExtractionOptions;
-						// 									walkedFolders: IFolderData[];
-						// 								}>
-						// 							) {
-						// 								switch (
-						// 									ctx.params.extractionOptions.extractionMode
-						// 								) {
-						// 									case "bulk":
-						// 										map(
-						// 											ctx.params.walkedFolders,
-						// 											async (folder, idx) => {
-						// 												let foo = await extractArchive(
-						// 													ctx.params
-						// 														.extractionOptions,
-						// 													folder
-						// 												);
-						// 												// console.log("levar", foo);
-						// 												let jsonStream =
-						// 													new JsonStreamStringify({
-						// 														foo,
-						// 													});
-						// 												return jsonStream;
-						// 											}
-						// 										);
-						//
-						// 									case "single":
-						// 										return await extractArchive(
-						// 											ctx.params.extractionOptions,
-						// 											ctx.params.walkedFolders[0]
-						// 										);
-						// 									default:
-						// 										console.log(
-						// 											"Unknown extraction mode selected."
-						// 										);
-						// 										return {
-						// 											message:
-						// 												"Unknown extraction mode selected.",
-						// 											errorCode: "90",
-						// 											data: `${ctx.params.extractionOptions}`,
-						// 										};
-						// 								}
-						// 							},
-						// 						},
 					},
 					methods: {},
 				},

utils/uncompression.utils.ts

@@ -50,10 +50,9 @@ const unrarer = require("node-unrar-js");
 const Walk = require("@root/walk");
 const fse = require("fs-extra");
 
-
 export const unrar = async (
 	extractionOptions: IExtractionOptions,
-  walkedFolder: IFolderData,
+	walkedFolder: IFolderData
 ): Promise<
 	| IExtractedComicBookCoverFile
 	| IExtractedComicBookCoverFile[]
@@ -63,8 +62,8 @@ export const unrar = async (
 	const directoryOptions = {
 		mode: 0o2775,
 	};
-  const fileBuffer = await readFile(paths.inputFilePath).catch(err =>
+	const fileBuffer = await readFile(paths.inputFilePath).catch((err) =>
-    console.error("Failed to read file", err),
+		console.error("Failed to read file", err)
 	);
 	try {
 		await fse.ensureDir(paths.targetPath, directoryOptions);
@@ -73,7 +72,9 @@ export const unrar = async (
 		logger.error(`${error}: Couldn't create directory.`);
 	}
 
-  const extractor = await unrarer.createExtractorFromData({ data: fileBuffer });
+	const extractor = await unrarer.createExtractorFromData({
+		data: fileBuffer,
+	});
 
 	switch (extractionOptions.extractTarget) {
 		case "cover":
@@ -82,21 +83,25 @@ export const unrar = async (
 					let fileNameToExtract = "";
 					const list = extractor.getFileList();
 					const fileHeaders = [...list.fileHeaders];
-          each(fileHeaders, async fileHeader => {
+					each(fileHeaders, async (fileHeader) => {
 						const fileName = explodePath(fileHeader.name).fileName;
 						if (
 							fileName !== "" &&
 							fileHeader.flags.directory === false &&
 							isEmpty(fileNameToExtract)
 						) {
-              logger.info(`Attempting to write ${fileHeader.name}`);
+							logger.info(
+								`Attempting to write ${fileHeader.name}`
+							);
 							fileNameToExtract = fileHeader.name;
-              const file = extractor.extract({ files: [fileHeader.name] });
+							const file = extractor.extract({
+								files: [fileHeader.name],
+							});
 							const extractedFile = [...file.files][0];
 							const fileArrayBuffer = extractedFile.extraction;
 							await writeFile(
 								paths.targetPath + "/" + fileName,
-                fileArrayBuffer,
+								fileArrayBuffer
 							);
 							resolve({
 								name: `${fileName}`,
@@ -116,14 +121,25 @@ export const unrar = async (
 				try {
 					const files = extractor.extract({});
 					const extractedFiles = [...files.files];
-          const comicBookCoverFiles: IExtractedComicBookCoverFile[] = [];
+					const comicBookCoverFiles: IExtractedComicBookCoverFile[] =
+						[];
 					for (const file of extractedFiles) {
-            logger.info(`Attempting to write ${file.fileHeader.name}`);
+						logger.info(
+							`Attempting to write ${file.fileHeader.name}`
+						);
 						const fileBuffer = file.extraction;
-            const fileName = explodePath(file.fileHeader.name).fileName;
+						const fileName = explodePath(
+							file.fileHeader.name
+						).fileName;
 
-            if (fileName !== "" && file.fileHeader.flags.directory === false) {
+						if (
-              await writeFile(paths.targetPath + "/" + fileName, fileBuffer);
+							fileName !== "" &&
+							file.fileHeader.flags.directory === false
+						) {
+							await writeFile(
+								paths.targetPath + "/" + fileName,
+								fileBuffer
+							);
 						}
 						comicBookCoverFiles.push({
 							name: `${file.fileHeader.name}`,
@@ -152,7 +168,7 @@ export const unrar = async (
 
 export const unzip = async (
 	extractionOptions: IExtractionOptions,
-  walkedFolder: IFolderData,
+	walkedFolder: IFolderData
 ): Promise<
 	| IExtractedComicBookCoverFile[]
 	| IExtractedComicBookCoverFile
@@ -172,7 +188,7 @@ export const unzip = async (
 
 	const extractedFiles: IExtractedComicBookCoverFile[] = [];
 	const zip = createReadStream(paths.inputFilePath).pipe(
-    unzipper.Parse({ forceStream: true }),
+		unzipper.Parse({ forceStream: true })
 	);
 	for await (const entry of zip) {
 		const fileName = explodePath(entry.path).fileName;
@@ -203,7 +219,7 @@ export const unzip = async (
 
 export const extractArchive = async (
 	extractionOptions: IExtractionOptions,
-  walkedFolder: IFolderData,
+	walkedFolder: IFolderData
 ): Promise<
 	| IExtractedComicBookCoverFile
 	| IExtractedComicBookCoverFile[]
@@ -225,7 +241,7 @@ export const extractArchive = async (
 
 export const getCovers = async (
 	options: IExtractionOptions,
-  walkedFolders: IFolderData[],
+	walkedFolders: IFolderData[]
 ): Promise<
 	| IExtractedComicBookCoverFile
 	| IExtractComicBookCoverErrorResponse
@@ -239,8 +255,13 @@ export const getCovers = async (
 > => {
 	switch (options.extractionMode) {
 		case "bulk":
-      const extractedDataPromises = map(walkedFolders, async folder => await extractArchive(options, folder));
+			const extractedDataPromises = map(
-      return Promise.all(extractedDataPromises).then(data => _.flatten(data));
+				walkedFolders,
+				async (folder) => await extractArchive(options, folder)
+			);
+			return Promise.all(extractedDataPromises).then((data) =>
+				_.flatten(data)
+			);
 		case "single":
 			return await extractArchive(options, walkedFolders[0]);
 		default:
@@ -278,7 +299,7 @@ export const walkFolder = async (folder: string): Promise<IFolderData[]> => {
 				isLink: dirent.isSymbolicLink(),
 			};
 			logger.info(
-        `Scanned ${dirent.name} contained in ${path.dirname(pathname)}`,
+				`Scanned ${dirent.name} contained in ${path.dirname(pathname)}`
 			);
 			result.push(walkResult);
 		}
@@ -288,7 +309,10 @@ export const walkFolder = async (folder: string): Promise<IFolderData[]> => {
 
 export const explodePath = (filePath: string): IExplodedPathResponse => {
 	const exploded = filePath.split("/");
-  const fileName = remove(exploded, item => indexOf(exploded, item) === exploded.length - 1).join("");
+	const fileName = remove(
+		exploded,
+		(item) => indexOf(exploded, item) === exploded.length - 1
+	).join("");
 
 	return {
 		exploded,
@@ -298,7 +322,7 @@ export const explodePath = (filePath: string): IExplodedPathResponse => {
 
 const constructPaths = (
 	extractionOptions: IExtractionOptions,
-  walkedFolder: IFolderData,
+	walkedFolder: IFolderData
 ) => ({
 	targetPath:
 		extractionOptions.targetExtractionFolder + "/" + walkedFolder.name,
@@ -310,7 +334,7 @@ const constructPaths = (
 });
 
 export const extractMetadataFromImage = async (
-  imageFilePath: string,
+	imageFilePath: string
 ): Promise<unknown> => {
 	const image = await sharp(imageFilePath)
 		.metadata()
@@ -320,4 +344,5 @@ export const extractMetadataFromImage = async (
 	return image;
 };
 
-const filterOutDotFiles = entities => entities.filter(ent => !ent.name.startsWith("."));
+const filterOutDotFiles = (entities) =>
+	entities.filter((ent) => !ent.name.startsWith("."));