diff --git a/package.json b/package.json index cede947..5a0e4fc 100644 --- a/package.json +++ b/package.json @@ -22,19 +22,17 @@ "@types/react-router-dom": "^5.1.7", "@types/sharp": "^0.28.0", "babel-polyfill": "^6.26.0", - "clooney": "^0.6.6", "comlink-loader": "^2.0.0", "express": "^4.17.1", "express-paginate": "^1.0.2", "fs-extra": "^9.1.0", - "highland": "^2.13.5", "imghash": "^0.0.8", "mongoose": "^5.10.11", - "oboe": "^2.1.5", "react": "^17.0.1", "react-dom": "^17.0.1", "react-hooks-worker": "^1.0.0", - "sharp": "^0.28.1" + "sharp": "^0.28.1", + "through2": "^4.0.2" }, "devDependencies": { "@babel/cli": "^7.13.10", diff --git a/src/server/route/routes/importComics.routes.ts b/src/server/route/routes/importComics.routes.ts index dbd0f67..c7b44ad 100644 --- a/src/server/route/routes/importComics.routes.ts +++ b/src/server/route/routes/importComics.routes.ts @@ -3,10 +3,9 @@ import { default as paginate } from "express-paginate"; import { walkFolder, extractArchive, getCovers } from "../../utils/fs.utils"; import { IExtractionOptions } from "../../interfaces/folder.interface"; import { Request, Response } from "express"; -const H = require("highland"); import _ from "lodash"; -const toStream = require("streammagic").toStream; -require("streammagic")(); +import through2 from "through2"; +import { Readable } from "stream"; router.route("/getComicCovers").post(async (req: Request, res: Response) => { typeof req.body.extractionOptions === "object" @@ -16,32 +15,29 @@ router.route("/getComicCovers").post(async (req: Request, res: Response) => { req.body.extractionOptions, req.body.walkedFolders, ); - let jsonStr; - // For each page of data you get, loop over the items like you say - _.each(foo, (item) => { - _.each(item, (subItem) => { - jsonStr = JSON.stringify(subItem) + "\n"; - toStream(jsonStr).pipe(res); // Assuming 'res' is the Express response object - }); + const stream = new Readable({ + objectMode: true, + highWaterMark: 1, + read() {}, }); - // if ( - // _.isArray(foo) && - // !_.isUndefined(req.body.extractionOptions.paginationOptions.pageLimit) - // ) { - // const pageCount = Math.ceil( - // foo.length / req.body.extractionOptions.paginationOptions.pageLimit, - // ); - // return res.json({ - // has_more: paginate.hasNextPages(req)(pageCount), - // pageCount, - // itemCount: foo.length, - // extractedData, - // }); - // } - return res.json({ - foo, - }); + const ndjsonStream = through2( + { objectMode: true, highWaterMark: 1 }, + (data, enc, cb) => { + cb(null, JSON.stringify(data) + "\n"); + }, + ); + + // Through pipe we do a double addressing, our reading stream goes through the transformation + // to finally go through the stream response.. + stream.pipe(ndjsonStream).pipe(res); + + stream.push(foo); + stream.push(null); + + // return res.json({ + // foo, + // }); }); router.route("/walkFolder").post(async (req: Request, res: Response) => { diff --git a/yarn.lock b/yarn.lock index 20a67fb..7c2a17d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3490,13 +3490,6 @@ clone@^1.0.2: resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" integrity sha1-2jCcwmPfFZlMaIypAheco8fNfH4= -clooney@^0.6.6: - version "0.6.6" - resolved "https://registry.yarnpkg.com/clooney/-/clooney-0.6.6.tgz#0b480843c30dd289ee558ec8c2336dbbb3fd38cd" - integrity sha512-kux3thg0N+uWZXTpYbthellECgsDw8PJ3IZmiVVyuavAEb/gBbC/u0ksZ6kHpZe5YT4H1Xs3TVSIQkzUfRqi8A== - dependencies: - comlinkjs "2.3.6" - cmd-shim@^4.0.1: version "4.1.0" resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd" @@ -3621,11 +3614,6 @@ comlink@^4.2.0, comlink@^4.3.0: resolved "https://registry.yarnpkg.com/comlink/-/comlink-4.3.0.tgz#80b3366baccd87897dab3638ebfcfae28b2f87c7" integrity sha512-mu4KKKNuW8TvkfpW/H88HBPeILubBS6T94BdD1VWBXNXfiyqVtwUCVNO1GeNOBTsIswzsMjWlycYr+77F5b84g== -comlinkjs@2.3.6: - version "2.3.6" - resolved "https://registry.yarnpkg.com/comlinkjs/-/comlinkjs-2.3.6.tgz#a756c2530062af292a705372484d4e2eff47729f" - integrity sha512-4WDd2iMM5AhnIRgUNMLWEj3gtIfEfu+JjSPUCYAa/K5IzJRvx/JAMC5jF53Exim7d67vnkRGd0i6vx27rGF2yA== - commander@^2.12.1, commander@^2.20.0, commander@^2.8.1: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" @@ -5993,13 +5981,6 @@ he@^1.2.0: resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== -highland@^2.13.5: - version "2.13.5" - resolved "https://registry.yarnpkg.com/highland/-/highland-2.13.5.tgz#d55cd8ac3f67a00fad79918668d51493222cfcc2" - integrity sha512-dn2flPapIIAa4BtkB2ahjshg8iSJtrJtdhEb9/oiOrS5HMQTR/GuhFpqJ+11YBdtnl3AwWKvbZd1Uxr8uAmA7A== - dependencies: - util-deprecate "^1.0.2" - history@^4.9.0: version "4.10.1" resolved "https://registry.yarnpkg.com/history/-/history-4.10.1.tgz#33371a65e3a83b267434e2b3f3b1b4c58aad4cf3" @@ -10079,6 +10060,15 @@ read@1, read@~1.0.1, read@~1.0.7: dependencies: mute-stream "~0.0.4" +readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" + integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.3.0, readable-stream@^2.3.5, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" @@ -10092,15 +10082,6 @@ readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^3.0.0, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" - integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== - dependencies: - inherits "^2.0.3" - string_decoder "^1.1.1" - util-deprecate "^1.0.1" - readdir-scoped-modules@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309" @@ -11219,11 +11200,6 @@ stealthy-require@^1.1.1: resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks= -streammagic@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/streammagic/-/streammagic-1.0.0.tgz#bcee7ef777e0b55ef937b87903f1cef65d14510f" - integrity sha1-vO5+93fgtV75N7h5A/HO9l0UUQ8= - streamz@~1.8.10: version "1.8.12" resolved "https://registry.yarnpkg.com/streamz/-/streamz-1.8.12.tgz#e022bf773fb0093de81f3cd2105390bbc9231613" @@ -11672,6 +11648,13 @@ through2@^2.0.2, through2@^2.0.3: readable-stream "~2.3.6" xtend "~4.0.1" +through2@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764" + integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw== + dependencies: + readable-stream "3" + through@^2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"