From cd3d4e272d88fa78b08ccfa4e41d8524fc615565 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Sun, 9 May 2021 23:40:19 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20Some=20refactoring=20of=20import?= =?UTF-8?q?=20endpoints=20in=20facade?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 6 +- src/client/workers/extractCovers.worker.ts | 11 ---- .../route/routes/importComics.routes.ts | 60 ++++++++----------- src/server/utils/fs.utils.ts | 7 ++- yarn.lock | 37 +++++++++++- 5 files changed, 67 insertions(+), 54 deletions(-) diff --git a/package.json b/package.json index 181bacc..35623d8 100644 --- a/package.json +++ b/package.json @@ -27,13 +27,15 @@ "express": "^4.17.1", "express-paginate": "^1.0.2", "fs-extra": "^9.1.0", + "highland": "^2.13.5", "imghash": "^0.0.8", + "into-stream": "^7.0.0", "mongoose": "^5.10.11", + "ndjson": "^2.0.0", "react": "^17.0.1", "react-dom": "^17.0.1", "react-hooks-worker": "^1.0.0", - "sharp": "^0.28.1", - "through2": "^4.0.2" + "sharp": "^0.28.1" }, "devDependencies": { "@babel/cli": "^7.13.10", diff --git a/src/client/workers/extractCovers.worker.ts b/src/client/workers/extractCovers.worker.ts index 53eaa8a..0ed5cf4 100644 --- a/src/client/workers/extractCovers.worker.ts +++ b/src/client/workers/extractCovers.worker.ts @@ -28,16 +28,5 @@ export const greet = async ( }, fileObjects, ); - const reader = await ndjsonStream(fo).getReader(); - reader.read().then(function process({ done, value }) { - if (done) { - console.log("done"); - return; - } - - console.log(value); - - return reader.read().then(process); - }); // return JSON.stringify(fo); }; diff --git a/src/server/route/routes/importComics.routes.ts b/src/server/route/routes/importComics.routes.ts index c7b44ad..39d493c 100644 --- a/src/server/route/routes/importComics.routes.ts +++ b/src/server/route/routes/importComics.routes.ts @@ -1,52 +1,42 @@ import router from "../router"; import { default as paginate } from "express-paginate"; -import { walkFolder, extractArchive, getCovers } from "../../utils/fs.utils"; import { IExtractionOptions } from "../../interfaces/folder.interface"; import { Request, Response } from "express"; import _ from "lodash"; -import through2 from "through2"; -import { Readable } from "stream"; +import H from "highland"; +import intoStream from "into-stream"; +const ndjson = require("ndjson"); +import axios from "axios"; router.route("/getComicCovers").post(async (req: Request, res: Response) => { typeof req.body.extractionOptions === "object" ? req.body.extractionOptions : {}; - const foo = await getCovers( - req.body.extractionOptions, - req.body.walkedFolders, - ); - const stream = new Readable({ - objectMode: true, - highWaterMark: 1, - read() {}, - }); - - const ndjsonStream = through2( - { objectMode: true, highWaterMark: 1 }, - (data, enc, cb) => { - cb(null, JSON.stringify(data) + "\n"); - }, - ); - - // Through pipe we do a double addressing, our reading stream goes through the transformation - // to finally go through the stream response.. - stream.pipe(ndjsonStream).pipe(res); - - stream.push(foo); - stream.push(null); - - // return res.json({ - // foo, - // }); + axios + .request({ + url: "http://localhost:3853/api/import/getComicCovers", + method: "POST", + data: { + extractionOptions: req.body.extractionOptions, + walkedFolders: req.body.walkedFolders, + }, + }) + .then((data) => data) + .catch((error) => error); }); router.route("/walkFolder").post(async (req: Request, res: Response) => { const basePathToWalk = - typeof req.query.basePathToWalk === "string" - ? req.query.basePathToWalk - : ""; - const results = await walkFolder(basePathToWalk); - res.json(results); + typeof req.body.basePathToWalk === "string" ? req.body.basePathToWalk : ""; + axios + .request({ + method: "POST", + data: { + basePathToWalk, + }, + }) + .then((data) => data) + .catch((error) => error); }); export default router; diff --git a/src/server/utils/fs.utils.ts b/src/server/utils/fs.utils.ts index 91c6f95..509da4c 100644 --- a/src/server/utils/fs.utils.ts +++ b/src/server/utils/fs.utils.ts @@ -37,6 +37,7 @@ const Walk = require("@root/walk"); const fse = require("fs-extra"); import { default as unzipper } from "unzipper"; +import _ from "lodash"; import { createReadStream, createWriteStream } from "fs"; const { writeFile, readFile } = require("fs").promises; import path from "path"; @@ -130,7 +131,7 @@ export const unrar = async ( fileSize: file.fileHeader.packSize, }); } - resolve(comicBookCoverFiles); + resolve(_.flatten(comicBookCoverFiles)); } catch (error) { resolve({ message: `${error}`, @@ -196,7 +197,7 @@ export const unzip = async ( return new Promise(async (resolve, reject) => { logger.info(""); - resolve(extractedFiles); + resolve(_.flatten(extractedFiles)); }); }; @@ -241,7 +242,7 @@ export const getCovers = async ( const extractedDataPromises = map(walkedFolders, async (folder) => { return await extractArchive(options, folder); }); - return Promise.all(extractedDataPromises).then((data) => data); + return Promise.all(extractedDataPromises).then((data) => _.flatten(data)); case "single": return await extractArchive(options, walkedFolders[0]); default: diff --git a/yarn.lock b/yarn.lock index 086b24a..d17f359 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5485,7 +5485,7 @@ fresh@0.5.2: resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= -from2@^2.1.1: +from2@^2.1.1, from2@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/from2/-/from2-2.3.0.tgz#8bfb5502bde4a4d36cfdeea007fcca21d7e382af" integrity sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8= @@ -5993,6 +5993,13 @@ he@^1.2.0: resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== +highland@^2.13.5: + version "2.13.5" + resolved "https://registry.yarnpkg.com/highland/-/highland-2.13.5.tgz#d55cd8ac3f67a00fad79918668d51493222cfcc2" + integrity sha512-dn2flPapIIAa4BtkB2ahjshg8iSJtrJtdhEb9/oiOrS5HMQTR/GuhFpqJ+11YBdtnl3AwWKvbZd1Uxr8uAmA7A== + dependencies: + util-deprecate "^1.0.2" + history@^4.9.0: version "4.10.1" resolved "https://registry.yarnpkg.com/history/-/history-4.10.1.tgz#33371a65e3a83b267434e2b3f3b1b4c58aad4cf3" @@ -6492,6 +6499,14 @@ into-stream@^3.1.0: from2 "^2.1.1" p-is-promise "^1.1.0" +into-stream@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/into-stream/-/into-stream-7.0.0.tgz#d1a211e146be8acfdb84dabcbf00fe8205e72936" + integrity sha512-2dYz766i9HprMBasCMvHMuazJ7u4WzhJwo5kb3iPSiW/iRYV6uPari3zHoqZlnuaR7V1bEiNMxikhp37rdBXbw== + dependencies: + from2 "^2.3.0" + p-is-promise "^3.0.0" + invert-kv@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02" @@ -8490,6 +8505,17 @@ ndjson@^1.4.0: split2 "^2.1.0" through2 "^2.0.3" +ndjson@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-2.0.0.tgz#320ac86f6fe53f5681897349b86ac6f43bfa3a19" + integrity sha512-nGl7LRGrzugTtaFcJMhLbpzJM6XdivmbkdlaGcrk/LXg2KL/YBC6z1g70xh0/al+oFuVFP8N8kiWRucmeEH/qQ== + dependencies: + json-stringify-safe "^5.0.1" + minimist "^1.2.5" + readable-stream "^3.6.0" + split2 "^3.0.0" + through2 "^4.0.0" + negotiator@0.6.2: version "0.6.2" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" @@ -9153,6 +9179,11 @@ p-is-promise@^2.0.0: resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-2.1.0.tgz#918cebaea248a62cf7ffab8e3bca8c5f882fc42e" integrity sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg== +p-is-promise@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-3.0.0.tgz#58e78c7dfe2e163cf2a04ff869e7c1dba64a5971" + integrity sha512-Wo8VsW4IRQSKVXsJCn7TomUaVtyfjVDn3nUP7kE967BQk0CwFpdbZs0X0uk5sW9mkBa9eNM7hCMaG93WUAwxYQ== + p-limit@^1.1.0: version "1.3.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8" @@ -11132,7 +11163,7 @@ split2@^2.1.0: dependencies: through2 "^2.0.2" -split2@^3.1.1: +split2@^3.0.0, split2@^3.1.1: version "3.2.2" resolved "https://registry.yarnpkg.com/split2/-/split2-3.2.2.tgz#bf2cf2a37d838312c249c89206fd7a17dd12365f" integrity sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg== @@ -11660,7 +11691,7 @@ through2@^2.0.2, through2@^2.0.3: readable-stream "~2.3.6" xtend "~4.0.1" -through2@^4.0.2: +through2@^4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764" integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==