frishi/threetwo
1
0
Fork 0
Code Issues 9 Pull Requests 6 Actions Packages Projects Releases 2 Wiki Activity

🏞 Streams, B

Browse Source
This commit is contained in:
Rishi Ghan
2021-05-07 11:32:04 -07:00
parent c7ac6e5979
commit 62c63de1fd
3 changed files with 41 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
package.json
View File

@@ -22,19 +22,17 @@
"@types/react-router-dom": "^5.1.7",
"@types/sharp": "^0.28.0",
"babel-polyfill": "^6.26.0",
"clooney": "^0.6.6",
"comlink-loader": "^2.0.0",
"express": "^4.17.1",
"express-paginate": "^1.0.2",
"fs-extra": "^9.1.0",
"highland": "^2.13.5",
"imghash": "^0.0.8",
"mongoose": "^5.10.11",
"oboe": "^2.1.5",
"react": "^17.0.1",
"react-dom": "^17.0.1",
"react-hooks-worker": "^1.0.0",
"sharp": "^0.28.1"
"sharp": "^0.28.1",
"through2": "^4.0.2"
},
"devDependencies": {
"@babel/cli": "^7.13.10",

50
src/server/route/routes/importComics.routes.ts
View File

@@ -3,10 +3,9 @@ import { default as paginate } from "express-paginate";
import { walkFolder, extractArchive, getCovers } from "../../utils/fs.utils";
import { IExtractionOptions } from "../../interfaces/folder.interface";
import { Request, Response } from "express";
const H = require("highland");
import _ from "lodash";
const toStream = require("streammagic").toStream;
require("streammagic")();
import through2 from "through2";
import { Readable } from "stream";
router.route("/getComicCovers").post(async (req: Request, res: Response) => {
typeof req.body.extractionOptions === "object"
@@ -16,32 +15,29 @@ router.route("/getComicCovers").post(async (req: Request, res: Response) => {
req.body.extractionOptions,
req.body.walkedFolders,
);
let jsonStr;
// For each page of data you get, loop over the items like you say
_.each(foo, (item) => {
_.each(item, (subItem) => {
jsonStr = JSON.stringify(subItem) + "\n";
toStream(jsonStr).pipe(res); // Assuming 'res' is the Express response object
});
const stream = new Readable({
objectMode: true,
highWaterMark: 1,
read() {},
});
// if (
// _.isArray(foo) &&
// !_.isUndefined(req.body.extractionOptions.paginationOptions.pageLimit)
// ) {
// const pageCount = Math.ceil(
// foo.length / req.body.extractionOptions.paginationOptions.pageLimit,
// );
// return res.json({
// has_more: paginate.hasNextPages(req)(pageCount),
// pageCount,
// itemCount: foo.length,
// extractedData,
// });
// }
return res.json({
foo,
});
const ndjsonStream = through2(
{ objectMode: true, highWaterMark: 1 },
(data, enc, cb) => {
cb(null, JSON.stringify(data) + "\n");
},
);
// Through pipe we do a double addressing, our reading stream goes through the transformation
// to finally go through the stream response..
stream.pipe(ndjsonStream).pipe(res);
stream.push(foo);
stream.push(null);
// return res.json({
// foo,
// });
});
router.route("/walkFolder").post(async (req: Request, res: Response) => {

49
yarn.lock
View File

@@ -3490,13 +3490,6 @@ clone@^1.0.2:
resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e"
integrity sha1-2jCcwmPfFZlMaIypAheco8fNfH4=
clooney@^0.6.6:
version "0.6.6"
resolved "https://registry.yarnpkg.com/clooney/-/clooney-0.6.6.tgz#0b480843c30dd289ee558ec8c2336dbbb3fd38cd"
integrity sha512-kux3thg0N+uWZXTpYbthellECgsDw8PJ3IZmiVVyuavAEb/gBbC/u0ksZ6kHpZe5YT4H1Xs3TVSIQkzUfRqi8A==
dependencies:
comlinkjs "2.3.6"
cmd-shim@^4.0.1:
version "4.1.0"
resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd"
@@ -3621,11 +3614,6 @@ comlink@^4.2.0, comlink@^4.3.0:
resolved "https://registry.yarnpkg.com/comlink/-/comlink-4.3.0.tgz#80b3366baccd87897dab3638ebfcfae28b2f87c7"
integrity sha512-mu4KKKNuW8TvkfpW/H88HBPeILubBS6T94BdD1VWBXNXfiyqVtwUCVNO1GeNOBTsIswzsMjWlycYr+77F5b84g==
comlinkjs@2.3.6:
version "2.3.6"
resolved "https://registry.yarnpkg.com/comlinkjs/-/comlinkjs-2.3.6.tgz#a756c2530062af292a705372484d4e2eff47729f"
integrity sha512-4WDd2iMM5AhnIRgUNMLWEj3gtIfEfu+JjSPUCYAa/K5IzJRvx/JAMC5jF53Exim7d67vnkRGd0i6vx27rGF2yA==
commander@^2.12.1, commander@^2.20.0, commander@^2.8.1:
version "2.20.3"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
@@ -5993,13 +5981,6 @@ he@^1.2.0:
resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
highland@^2.13.5:
version "2.13.5"
resolved "https://registry.yarnpkg.com/highland/-/highland-2.13.5.tgz#d55cd8ac3f67a00fad79918668d51493222cfcc2"
integrity sha512-dn2flPapIIAa4BtkB2ahjshg8iSJtrJtdhEb9/oiOrS5HMQTR/GuhFpqJ+11YBdtnl3AwWKvbZd1Uxr8uAmA7A==
dependencies:
util-deprecate "^1.0.2"
history@^4.9.0:
version "4.10.1"
resolved "https://registry.yarnpkg.com/history/-/history-4.10.1.tgz#33371a65e3a83b267434e2b3f3b1b4c58aad4cf3"
@@ -10079,6 +10060,15 @@ read@1, read@~1.0.1, read@~1.0.7:
dependencies:
mute-stream "~0.0.4"
readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
version "3.6.0"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
dependencies:
inherits "^2.0.3"
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.3.0, readable-stream@^2.3.5, readable-stream@~2.3.6:
version "2.3.7"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
@@ -10092,15 +10082,6 @@ readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable
string_decoder "~1.1.1"
util-deprecate "~1.0.1"
readable-stream@^3.0.0, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
version "3.6.0"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
dependencies:
inherits "^2.0.3"
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
readdir-scoped-modules@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309"
@@ -11219,11 +11200,6 @@ stealthy-require@^1.1.1:
resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b"
integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=
streammagic@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/streammagic/-/streammagic-1.0.0.tgz#bcee7ef777e0b55ef937b87903f1cef65d14510f"
integrity sha1-vO5+93fgtV75N7h5A/HO9l0UUQ8=
streamz@~1.8.10:
version "1.8.12"
resolved "https://registry.yarnpkg.com/streamz/-/streamz-1.8.12.tgz#e022bf773fb0093de81f3cd2105390bbc9231613"
@@ -11672,6 +11648,13 @@ through2@^2.0.2, through2@^2.0.3:
readable-stream "~2.3.6"
xtend "~4.0.1"
through2@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764"
integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==
dependencies:
readable-stream "3"
through@^2.3.8:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"