🏞 Streams, B

src/server/route/routes/importComics.routes.ts

@@ -3,10 +3,9 @@ import { default as paginate } from "express-paginate";
 import { walkFolder, extractArchive, getCovers } from "../../utils/fs.utils";
 import { IExtractionOptions } from "../../interfaces/folder.interface";
 import { Request, Response } from "express";
-const H = require("highland");
 import _ from "lodash";
-const toStream = require("streammagic").toStream;
-require("streammagic")();
+import through2 from "through2";
+import { Readable } from "stream";
 
 router.route("/getComicCovers").post(async (req: Request, res: Response) => {
   typeof req.body.extractionOptions === "object"
@@ -16,32 +15,29 @@ router.route("/getComicCovers").post(async (req: Request, res: Response) => {
     req.body.extractionOptions,
     req.body.walkedFolders,
   );
-    let jsonStr;
-  // For each page of data you get, loop over the items like you say
-  _.each(foo, (item) => {
-    _.each(item, (subItem) => {
-      jsonStr = JSON.stringify(subItem) + "\n";
-      toStream(jsonStr).pipe(res); // Assuming 'res' is the Express response object
-    });
+  const stream = new Readable({
+    objectMode: true,
+    highWaterMark: 1,
+    read() {},
   });
-  // if (
-  //   _.isArray(foo) &&
-  //   !_.isUndefined(req.body.extractionOptions.paginationOptions.pageLimit)
-  // ) {
-  //   const pageCount = Math.ceil(
-  //     foo.length / req.body.extractionOptions.paginationOptions.pageLimit,
-  //   );
 
-  //   return res.json({
-  //     has_more: paginate.hasNextPages(req)(pageCount),
-  //     pageCount,
-  //     itemCount: foo.length,
-  //     extractedData,
-  //   });
-  // }
-  return res.json({
-    foo,
-  });
+  const ndjsonStream = through2(
+    { objectMode: true, highWaterMark: 1 },
+    (data, enc, cb) => {
+      cb(null, JSON.stringify(data) + "\n");
+    },
+  );
+
+  // Through pipe we do a double addressing, our reading stream goes through the transformation
+  // to finally go through the stream response..
+  stream.pipe(ndjsonStream).pipe(res);
+
+  stream.push(foo);
+  stream.push(null);
+
+  // return res.json({
+  //   foo,
+  // });
 });
 
 router.route("/walkFolder").post(async (req: Request, res: Response) => {