🧑🏻‍🔧 Worker Streams

src/client/actions/fileops.actions.tsx

@@ -1,18 +1,19 @@
 import axios from "axios";
+import fetch, { Response } from "node-fetch";
 import {
   IExtractComicBookCoverErrorResponse,
   IExtractedComicBookCoverFile,
   IExtractionOptions,
   IFolderData,
 } from "../../server/interfaces/folder.interface";
-import { FS_API_BASE_URI } from "../constants/endpoints";
+import { API_BASE_URI } from "../constants/endpoints";
 
 export async function walkFolder(path: string): Promise<Array<IFolderData>> {
   return axios
     .request<Array<IFolderData>>({
-      url: FS_API_BASE_URI + "walkFolder",
+      url: API_BASE_URI + "walkFolder",
       method: "POST",
-      params: {
+      data: {
         basePathToWalk: path,
       },
       transformResponse: (r: string) => JSON.parse(r),
@@ -26,17 +27,16 @@ export async function walkFolder(path: string): Promise<Array<IFolderData>> {
 export async function extractCoverFromComicBookArchive(
   extractionOptions: IExtractionOptions,
   walkedFolders: Array<IFolderData>,
-): Promise<
-  | IExtractedComicBookCoverFile
-  | IExtractedComicBookCoverFile[]
-  | IExtractComicBookCoverErrorResponse
-> {
-  return await axios.request({
-    url: FS_API_BASE_URI + "getComicCovers",
+): Promise<Response> {
+  return await fetch(API_BASE_URI + "getComicCovers", {
     method: "POST",
-    data: {
+    headers: {
+      Accept: "application/json",
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
       extractionOptions,
       walkedFolders,
-    },
+    }),
   });
 }

src/client/constants/endpoints.ts

@@ -1,3 +1,3 @@
 export const COMICBOOKINFO_SERVICE_URI =
   "http://localhost:6050/api/comicbookinfo/";
-export const FS_API_BASE_URI = "http://localhost:8050/api/";
+export const API_BASE_URI = "http://localhost:8050/api/";

src/client/workers/extractCovers.worker.ts

@@ -4,6 +4,8 @@ import {
 } from "../actions/fileops.actions";
 import { IExtractedComicBookCoverFile } from "../../server/interfaces/folder.interface";
 const ndjsonStream = require("can-ndjson-stream");
+import fetch from "node-fetch";
+import { API_BASE_URI } from "../constants/endpoints";
 
 export const greet = async (
   path: string,
@@ -19,14 +21,23 @@ export const greet = async (
     pageLimit: 25,
     page: 1,
   };
-
+  const extractionOptions = {
+    ...targetOptions,
+    paginationOptions: pagingConfig,
+  };
   const fileObjects = await walkFolder("./comics");
-  const fo = await extractCoverFromComicBookArchive(
-    {
-      ...targetOptions,
-      paginationOptions: pagingConfig,
-    },
+  const fetchedResource = await extractCoverFromComicBookArchive(
+    extractionOptions,
     fileObjects,
   );
-  // return JSON.stringify(fo);
+
+  const reader = await ndjsonStream(fetchedResource.body).getReader();
+  reader.read().then(function process({ done, value }) {
+    if (done) {
+      console.log("done");
+      return;
+    }
+
+    return reader.read().then(process);
+  });
 };

src/server/route/routes/importComics.routes.ts

@@ -1,46 +1,74 @@
 import router from "../router";
 import { default as paginate } from "express-paginate";
-import { IExtractionOptions } from "../../interfaces/folder.interface";
+import { IExtractedComicBookCoverFile, IExtractionOptions } from "../../interfaces/folder.interface";
 import { Request, Response } from "express";
 import _ from "lodash";
 import H from "highland";
 import axios from "axios";
 import oboe from "oboe";
+import fs from "fs";
+import { Readable } from "stream";
+import through2 from "through2";
 
+const getData = (source) => {
+  const response: { value: string }[] = [];
+
+  for (let index = 0; index < 100; index++) {
+    response.push({ value: "rishi " + index });
+  }
+
+  return response;
+};
 router.route("/getComicCovers").post(async (req: Request, res: Response) => {
   typeof req.body.extractionOptions === "object"
     ? req.body.extractionOptions
     : {};
-  console.log(oboe);
-  oboe({
-    url: "http://localhost:3000/api/import/getComicCovers",
+  const foo = await axios({
+    url: "http://localhost:3853/api/import/getComicCovers",
     method: "POST",
-    body: {
+    data: {
       extractionOptions: req.body.extractionOptions,
       walkedFolders: req.body.walkedFolders,
     },
-    headers: {
-      "Content-Type": "application/json",
-      "Content-Length": req.body.length,
-    },
-  }).node("{name path fileSize}", (data) => {
-    console.log(data);
-    return res.sendStatus(200);
   });
+  const stream = new Readable({
+    objectMode: true,
+    highWaterMark: 1,
+    read() {},
+  });
+
+  // We create the stream transform using through2 library..
+  // We instruct it to handle objects, buffer size and transform function,
+  // that is, we convert our object to text to be able to send it through the stream response, which does not handle objects..
+  const ndjsonStream = through2(
+    { objectMode: true, highWaterMark: 1 },
+    (data, enc, cb) => {
+      cb(null, JSON.stringify(data) + "\n");
+    },
+  );
+
+  // console.log(ndjsonStream);
+  // Through pipe we do a double addressing, our reading stream goes through the transformation
+  // to finally go through the stream response..
+  stream.pipe(ndjsonStream).pipe(res);
+  stream.push({ source1: foo.data });
+  stream.push(null);
 });
 
 router.route("/walkFolder").post(async (req: Request, res: Response) => {
   const basePathToWalk =
     typeof req.body.basePathToWalk === "string" ? req.body.basePathToWalk : "";
-  axios
+  const walkedFolders = await axios
     .request({
+      url: "http://localhost:3853/api/import/walkFolders",
       method: "POST",
       data: {
         basePathToWalk,
       },
     })
-    .then((data) => data)
+    .then((data) => data.data)
     .catch((error) => error);
+  return res.json(walkedFolders);
 });
 
 export default router;