🔧 Some refactoring of import endpoints in facade

2021-05-09 23:40:19 -07:00
parent d35f4b50b4
commit cd3d4e272d
5 changed files with 67 additions and 54 deletions
--- a/package.json
+++ b/package.json
@@ -27,13 +27,15 @@
    "express": "^4.17.1",
    "express-paginate": "^1.0.2",
    "fs-extra": "^9.1.0",
+    "highland": "^2.13.5",
    "imghash": "^0.0.8",
+    "into-stream": "^7.0.0",
    "mongoose": "^5.10.11",
+    "ndjson": "^2.0.0",
    "react": "^17.0.1",
    "react-dom": "^17.0.1",
    "react-hooks-worker": "^1.0.0",
-    "sharp": "^0.28.1",
-    "through2": "^4.0.2"
+    "sharp": "^0.28.1"
  },
  "devDependencies": {
    "@babel/cli": "^7.13.10",
--- a/src/client/workers/extractCovers.worker.ts
+++ b/src/client/workers/extractCovers.worker.ts
@@ -28,16 +28,5 @@ export const greet = async (
    },
    fileObjects,
  );
-  const reader = await ndjsonStream(fo).getReader();
-  reader.read().then(function process({ done, value }) {
-    if (done) {
-      console.log("done");
-      return;
-    }
-
-    console.log(value);
-
-    return reader.read().then(process);
-  });
  // return JSON.stringify(fo);
 };
--- a/src/server/route/routes/importComics.routes.ts
+++ b/src/server/route/routes/importComics.routes.ts
@@ -1,52 +1,42 @@
 import router from "../router";
 import { default as paginate } from "express-paginate";
-import { walkFolder, extractArchive, getCovers } from "../../utils/fs.utils";
 import { IExtractionOptions } from "../../interfaces/folder.interface";
 import { Request, Response } from "express";
 import _ from "lodash";
-import through2 from "through2";
-import { Readable } from "stream";
+import H from "highland";
+import intoStream from "into-stream";
+const ndjson = require("ndjson");
+import axios from "axios";

 router.route("/getComicCovers").post(async (req: Request, res: Response) => {
  typeof req.body.extractionOptions === "object"
    ? req.body.extractionOptions
    : {};
-  const foo = await getCovers(
-    req.body.extractionOptions,
-    req.body.walkedFolders,
-  );
-  const stream = new Readable({
-    objectMode: true,
-    highWaterMark: 1,
-    read() {},
-  });
-
-  const ndjsonStream = through2(
-    { objectMode: true, highWaterMark: 1 },
-    (data, enc, cb) => {
-      cb(null, JSON.stringify(data) + "\n");
-    },
-  );
-
-  // Through pipe we do a double addressing, our reading stream goes through the transformation
-  // to finally go through the stream response..
-  stream.pipe(ndjsonStream).pipe(res);
-
-  stream.push(foo);
-  stream.push(null);
-
-  // return res.json({
-  //   foo,
-  // });
+  axios
+    .request({
+      url: "http://localhost:3853/api/import/getComicCovers",
+      method: "POST",
+      data: {
+        extractionOptions: req.body.extractionOptions,
+        walkedFolders: req.body.walkedFolders,
+      },
+    })
+    .then((data) => data)
+    .catch((error) => error);
 });

 router.route("/walkFolder").post(async (req: Request, res: Response) => {
  const basePathToWalk =
-    typeof req.query.basePathToWalk === "string"
-      ? req.query.basePathToWalk
-      : "";
-  const results = await walkFolder(basePathToWalk);
-  res.json(results);
+    typeof req.body.basePathToWalk === "string" ? req.body.basePathToWalk : "";
+  axios
+    .request({
+      method: "POST",
+      data: {
+        basePathToWalk,
+      },
+    })
+    .then((data) => data)
+    .catch((error) => error);
 });

 export default router;
--- a/src/server/utils/fs.utils.ts
+++ b/src/server/utils/fs.utils.ts
@@ -37,6 +37,7 @@ const Walk = require("@root/walk");
 const fse = require("fs-extra");

 import { default as unzipper } from "unzipper";
+import _ from "lodash";
 import { createReadStream, createWriteStream } from "fs";
 const { writeFile, readFile } = require("fs").promises;
 import path from "path";
@@ -130,7 +131,7 @@ export const unrar = async (
              fileSize: file.fileHeader.packSize,
            });
          }
-          resolve(comicBookCoverFiles);
+          resolve(_.flatten(comicBookCoverFiles));
        } catch (error) {
          resolve({
            message: `${error}`,
@@ -196,7 +197,7 @@ export const unzip = async (

  return new Promise(async (resolve, reject) => {
    logger.info("");
-    resolve(extractedFiles);
+    resolve(_.flatten(extractedFiles));
  });
 };

@@ -241,7 +242,7 @@ export const getCovers = async (
      const extractedDataPromises = map(walkedFolders, async (folder) => {
        return await extractArchive(options, folder);
      });
-      return Promise.all(extractedDataPromises).then((data) => data);
+      return Promise.all(extractedDataPromises).then((data) => _.flatten(data));
    case "single":
      return await extractArchive(options, walkedFolders[0]);
    default:
--- a/yarn.lock
+++ b/yarn.lock
@@ -5485,7 +5485,7 @@ fresh@0.5.2:
  resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7"
  integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=

-from2@^2.1.1:
+from2@^2.1.1, from2@^2.3.0:
  version "2.3.0"
  resolved "https://registry.yarnpkg.com/from2/-/from2-2.3.0.tgz#8bfb5502bde4a4d36cfdeea007fcca21d7e382af"
  integrity sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=
@@ -5993,6 +5993,13 @@ he@^1.2.0:
  resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
  integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==

+highland@^2.13.5:
+  version "2.13.5"
+  resolved "https://registry.yarnpkg.com/highland/-/highland-2.13.5.tgz#d55cd8ac3f67a00fad79918668d51493222cfcc2"
+  integrity sha512-dn2flPapIIAa4BtkB2ahjshg8iSJtrJtdhEb9/oiOrS5HMQTR/GuhFpqJ+11YBdtnl3AwWKvbZd1Uxr8uAmA7A==
+  dependencies:
+    util-deprecate "^1.0.2"
+
 history@^4.9.0:
  version "4.10.1"
  resolved "https://registry.yarnpkg.com/history/-/history-4.10.1.tgz#33371a65e3a83b267434e2b3f3b1b4c58aad4cf3"
@@ -6492,6 +6499,14 @@ into-stream@^3.1.0:
    from2 "^2.1.1"
    p-is-promise "^1.1.0"

+into-stream@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/into-stream/-/into-stream-7.0.0.tgz#d1a211e146be8acfdb84dabcbf00fe8205e72936"
+  integrity sha512-2dYz766i9HprMBasCMvHMuazJ7u4WzhJwo5kb3iPSiW/iRYV6uPari3zHoqZlnuaR7V1bEiNMxikhp37rdBXbw==
+  dependencies:
+    from2 "^2.3.0"
+    p-is-promise "^3.0.0"
+
 invert-kv@^2.0.0:
  version "2.0.0"
  resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
@@ -8490,6 +8505,17 @@ ndjson@^1.4.0:
    split2 "^2.1.0"
    through2 "^2.0.3"

+ndjson@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-2.0.0.tgz#320ac86f6fe53f5681897349b86ac6f43bfa3a19"
+  integrity sha512-nGl7LRGrzugTtaFcJMhLbpzJM6XdivmbkdlaGcrk/LXg2KL/YBC6z1g70xh0/al+oFuVFP8N8kiWRucmeEH/qQ==
+  dependencies:
+    json-stringify-safe "^5.0.1"
+    minimist "^1.2.5"
+    readable-stream "^3.6.0"
+    split2 "^3.0.0"
+    through2 "^4.0.0"
+
 negotiator@0.6.2:
  version "0.6.2"
  resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb"
@@ -9153,6 +9179,11 @@ p-is-promise@^2.0.0:
  resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-2.1.0.tgz#918cebaea248a62cf7ffab8e3bca8c5f882fc42e"
  integrity sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==

+p-is-promise@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-3.0.0.tgz#58e78c7dfe2e163cf2a04ff869e7c1dba64a5971"
+  integrity sha512-Wo8VsW4IRQSKVXsJCn7TomUaVtyfjVDn3nUP7kE967BQk0CwFpdbZs0X0uk5sW9mkBa9eNM7hCMaG93WUAwxYQ==
+
 p-limit@^1.1.0:
  version "1.3.0"
  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8"
@@ -11132,7 +11163,7 @@ split2@^2.1.0:
  dependencies:
    through2 "^2.0.2"

-split2@^3.1.1:
+split2@^3.0.0, split2@^3.1.1:
  version "3.2.2"
  resolved "https://registry.yarnpkg.com/split2/-/split2-3.2.2.tgz#bf2cf2a37d838312c249c89206fd7a17dd12365f"
  integrity sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==
@@ -11660,7 +11691,7 @@ through2@^2.0.2, through2@^2.0.3:
    readable-stream "~2.3.6"
    xtend "~4.0.1"

-through2@^4.0.2:
+through2@^4.0.0:
  version "4.0.2"
  resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764"
  integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==