🏞 Streams, B

package.json

@@ -22,19 +22,17 @@
     "@types/react-router-dom": "^5.1.7",
     "@types/sharp": "^0.28.0",
     "babel-polyfill": "^6.26.0",
-    "clooney": "^0.6.6",
     "comlink-loader": "^2.0.0",
     "express": "^4.17.1",
     "express-paginate": "^1.0.2",
     "fs-extra": "^9.1.0",
-    "highland": "^2.13.5",
     "imghash": "^0.0.8",
     "mongoose": "^5.10.11",
-    "oboe": "^2.1.5",
     "react": "^17.0.1",
     "react-dom": "^17.0.1",
     "react-hooks-worker": "^1.0.0",
-    "sharp": "^0.28.1"
+    "sharp": "^0.28.1",
+    "through2": "^4.0.2"
   },
   "devDependencies": {
     "@babel/cli": "^7.13.10",

src/server/route/routes/importComics.routes.ts

@@ -3,10 +3,9 @@ import { default as paginate } from "express-paginate";
 import { walkFolder, extractArchive, getCovers } from "../../utils/fs.utils";
 import { IExtractionOptions } from "../../interfaces/folder.interface";
 import { Request, Response } from "express";
-const H = require("highland");
 import _ from "lodash";
-const toStream = require("streammagic").toStream;
+import through2 from "through2";
-require("streammagic")();
+import { Readable } from "stream";
 
 router.route("/getComicCovers").post(async (req: Request, res: Response) => {
   typeof req.body.extractionOptions === "object"
@@ -16,32 +15,29 @@ router.route("/getComicCovers").post(async (req: Request, res: Response) => {
     req.body.extractionOptions,
     req.body.walkedFolders,
   );
-    let jsonStr;
+  const stream = new Readable({
-  // For each page of data you get, loop over the items like you say
+    objectMode: true,
-  _.each(foo, (item) => {
+    highWaterMark: 1,
-    _.each(item, (subItem) => {
+    read() {},
-      jsonStr = JSON.stringify(subItem) + "\n";
-      toStream(jsonStr).pipe(res); // Assuming 'res' is the Express response object
-    });
   });
-  // if (
-  //   _.isArray(foo) &&
-  //   !_.isUndefined(req.body.extractionOptions.paginationOptions.pageLimit)
-  // ) {
-  //   const pageCount = Math.ceil(
-  //     foo.length / req.body.extractionOptions.paginationOptions.pageLimit,
-  //   );
 
-  //   return res.json({
+  const ndjsonStream = through2(
-  //     has_more: paginate.hasNextPages(req)(pageCount),
+    { objectMode: true, highWaterMark: 1 },
-  //     pageCount,
+    (data, enc, cb) => {
-  //     itemCount: foo.length,
+      cb(null, JSON.stringify(data) + "\n");
-  //     extractedData,
+    },
-  //   });
+  );
-  // }
+
-  return res.json({
+  // Through pipe we do a double addressing, our reading stream goes through the transformation
-    foo,
+  // to finally go through the stream response..
-  });
+  stream.pipe(ndjsonStream).pipe(res);
+
+  stream.push(foo);
+  stream.push(null);
+
+  // return res.json({
+  //   foo,
+  // });
 });
 
 router.route("/walkFolder").post(async (req: Request, res: Response) => {

yarn.lock

@@ -3490,13 +3490,6 @@ clone@^1.0.2:
   resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e"
   integrity sha1-2jCcwmPfFZlMaIypAheco8fNfH4=
 
-clooney@^0.6.6:
-  version "0.6.6"
-  resolved "https://registry.yarnpkg.com/clooney/-/clooney-0.6.6.tgz#0b480843c30dd289ee558ec8c2336dbbb3fd38cd"
-  integrity sha512-kux3thg0N+uWZXTpYbthellECgsDw8PJ3IZmiVVyuavAEb/gBbC/u0ksZ6kHpZe5YT4H1Xs3TVSIQkzUfRqi8A==
-  dependencies:
-    comlinkjs "2.3.6"
-
 cmd-shim@^4.0.1:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd"
@@ -3621,11 +3614,6 @@ comlink@^4.2.0, comlink@^4.3.0:
   resolved "https://registry.yarnpkg.com/comlink/-/comlink-4.3.0.tgz#80b3366baccd87897dab3638ebfcfae28b2f87c7"
   integrity sha512-mu4KKKNuW8TvkfpW/H88HBPeILubBS6T94BdD1VWBXNXfiyqVtwUCVNO1GeNOBTsIswzsMjWlycYr+77F5b84g==
 
-comlinkjs@2.3.6:
-  version "2.3.6"
-  resolved "https://registry.yarnpkg.com/comlinkjs/-/comlinkjs-2.3.6.tgz#a756c2530062af292a705372484d4e2eff47729f"
-  integrity sha512-4WDd2iMM5AhnIRgUNMLWEj3gtIfEfu+JjSPUCYAa/K5IzJRvx/JAMC5jF53Exim7d67vnkRGd0i6vx27rGF2yA==
-
 commander@^2.12.1, commander@^2.20.0, commander@^2.8.1:
   version "2.20.3"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
@@ -5993,13 +5981,6 @@ he@^1.2.0:
   resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
   integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
 
-highland@^2.13.5:
-  version "2.13.5"
-  resolved "https://registry.yarnpkg.com/highland/-/highland-2.13.5.tgz#d55cd8ac3f67a00fad79918668d51493222cfcc2"
-  integrity sha512-dn2flPapIIAa4BtkB2ahjshg8iSJtrJtdhEb9/oiOrS5HMQTR/GuhFpqJ+11YBdtnl3AwWKvbZd1Uxr8uAmA7A==
-  dependencies:
-    util-deprecate "^1.0.2"
-
 history@^4.9.0:
   version "4.10.1"
   resolved "https://registry.yarnpkg.com/history/-/history-4.10.1.tgz#33371a65e3a83b267434e2b3f3b1b4c58aad4cf3"
@@ -10079,6 +10060,15 @@ read@1, read@~1.0.1, read@~1.0.7:
   dependencies:
     mute-stream "~0.0.4"
 
+readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
+  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
+  dependencies:
+    inherits "^2.0.3"
+    string_decoder "^1.1.1"
+    util-deprecate "^1.0.1"
+
 readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.3.0, readable-stream@^2.3.5, readable-stream@~2.3.6:
   version "2.3.7"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
@@ -10092,15 +10082,6 @@ readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
-readable-stream@^3.0.0, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
-  version "3.6.0"
-  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
-  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
-  dependencies:
-    inherits "^2.0.3"
-    string_decoder "^1.1.1"
-    util-deprecate "^1.0.1"
-
 readdir-scoped-modules@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309"
@@ -11219,11 +11200,6 @@ stealthy-require@^1.1.1:
   resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b"
   integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=
 
-streammagic@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/streammagic/-/streammagic-1.0.0.tgz#bcee7ef777e0b55ef937b87903f1cef65d14510f"
-  integrity sha1-vO5+93fgtV75N7h5A/HO9l0UUQ8=
-
 streamz@~1.8.10:
   version "1.8.12"
   resolved "https://registry.yarnpkg.com/streamz/-/streamz-1.8.12.tgz#e022bf773fb0093de81f3cd2105390bbc9231613"
@@ -11672,6 +11648,13 @@ through2@^2.0.2, through2@^2.0.3:
     readable-stream "~2.3.6"
     xtend "~4.0.1"
 
+through2@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764"
+  integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==
+  dependencies:
+    readable-stream "3"
+
 through@^2.3.8:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"