frishi/threetwo
1
0
Fork 0
Code Issues 9 Pull Requests 6 Actions Packages Projects Releases 2 Wiki Activity

👷🏽 Streaming JSON WIP

Browse Source
This commit is contained in:
Rishi Ghan
2021-05-13 07:58:59 -07:00
parent 267386ec94
commit c15b2cf0e4
3 changed files with 129 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
package.json
View File

@@ -25,14 +25,17 @@
"babel-polyfill": "^6.26.0",
"can-ndjson-stream": "^1.0.2",
"comlink-loader": "^2.0.0",
"event-stream": "^4.0.1",
"express": "^4.17.1",
"fs-extra": "^9.1.0",
"hyperquest": "^2.1.3",
"imghash": "^0.0.8",
"node-fetch": "^2.6.1",
"react": "^17.0.1",
"react-dom": "^17.0.1",
"react-hooks-worker": "^1.0.0",
"sharp": "^0.28.1",
"stream-json": "^1.7.1",
"through2": "^4.0.2"
},
"devDependencies": {

27
src/server/route/routes/importComics.routes.ts
View File

@@ -3,35 +3,22 @@ import { Request, Response } from "express";
import axios from "axios";
import { Readable } from "stream";
import through2 from "through2";
import hyperquest from "hyperquest";
import es from "event-stream";
router.route("/getComicCovers").post(async (req: Request, res: Response) => {
typeof req.body.extractionOptions === "object"
? req.body.extractionOptions
: {};
const comicBookCoversData = await axios({
url: "http://localhost:3000/api/import/getComicCovers",
method: "POST",
data: {
const comicBookCoversData = hyperquest.post("http://localhost:3853/api/import/getComicCovers",
{
data: {
extractionOptions: req.body.extractionOptions,
walkedFolders: req.body.walkedFolders,
},
});
const stream = new Readable({
objectMode: true,
highWaterMark: 1,
read() {},
});
const ndjsonStream = through2(
{ objectMode: true, highWaterMark: 1 },
(data, enc, cb) => {
cb(null, JSON.stringify(data) + "\n");
},
);
stream.pipe(ndjsonStream).pipe(res);
stream.push({ data: comicBookCoversData.data });
stream.push(null);
console.log(comicBookCoversData.data);
});
router.route("/walkFolder").post(async (req: Request, res: Response) => {

122
yarn.lock
View File

@@ -3085,6 +3085,11 @@ buffer-fill@^1.0.0:
resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c"
integrity sha1-+PeLdniYiO858gXNY39o5wISKyw=
buffer-from@^0.1.1:
version "0.1.2"
resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-0.1.2.tgz#15f4b9bcef012044df31142c14333caf6e0260d0"
integrity sha512-RiWIenusJsmI2KcvqQABB83tLxCByE3upSP8QU3rJDMVFGPWLvPQJt/O1Su9moRWeH7d+Q2HYb68f6+v+tw2vg==
buffer-from@^1.0.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
@@ -4458,6 +4463,13 @@ download@^7.1.0:
p-event "^2.1.0"
pify "^3.0.0"
duplexer2@~0.0.2:
version "0.0.2"
resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.0.2.tgz#c614dcf67e2fb14995a91711e5a617e8a60a31db"
integrity sha1-xhTc9n4vsUmVqRcR5aYX6KYKMds=
dependencies:
readable-stream "~1.1.9"
duplexer2@~0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1"
@@ -4470,6 +4482,11 @@ duplexer3@^0.1.4:
resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
integrity sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=
duplexer@^0.1.1, duplexer@~0.1.1:
version "0.1.2"
resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6"
integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==
ecc-jsbn@~0.1.1:
version "0.1.2"
resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9"
@@ -4946,6 +4963,19 @@ etl@^0.6.12:
moment "~2.21.0"
streamz "~1.8.10"
event-stream@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/event-stream/-/event-stream-4.0.1.tgz#4092808ec995d0dd75ea4580c1df6a74db2cde65"
integrity sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==
dependencies:
duplexer "^0.1.1"
from "^0.1.7"
map-stream "0.0.7"
pause-stream "^0.0.11"
split "^1.0.1"
stream-combiner "^0.2.2"
through "^2.3.8"
eventemitter3@^4.0.0:
version "4.0.7"
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
@@ -5508,6 +5538,11 @@ from2@^2.1.1:
inherits "^2.0.1"
readable-stream "^2.0.0"
from@^0.1.7:
version "0.1.7"
resolved "https://registry.yarnpkg.com/from/-/from-0.1.7.tgz#83c60afc58b9c56997007ed1a768b3ab303a44fe"
integrity sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=
fs-constants@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
@@ -6216,6 +6251,15 @@ humanize-ms@^1.2.1:
dependencies:
ms "^2.0.0"
hyperquest@^2.1.3:
version "2.1.3"
resolved "https://registry.yarnpkg.com/hyperquest/-/hyperquest-2.1.3.tgz#523127d7a343181b40bf324e231d2576edf52633"
integrity sha512-fUuDOrB47PqNK/BAMOS13v41UoaqIxqSLHX6CAbOD7OfT+/GCWO1/vPLfTNutOeXrv1ikuaZ3yux+33Z9vh+rw==
dependencies:
buffer-from "^0.1.1"
duplexer2 "~0.0.2"
through2 "~0.6.3"
iconv-lite@0.4.24:
version "0.4.24"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
@@ -6438,7 +6482,7 @@ inflight@^1.0.4:
once "^1.3.0"
wrappy "1"
inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.3:
inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
version "2.0.4"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -8028,6 +8072,11 @@ map-obj@^1.0.0, map-obj@^1.0.1:
resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d"
integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=
map-stream@0.0.7:
version "0.0.7"
resolved "https://registry.yarnpkg.com/map-stream/-/map-stream-0.0.7.tgz#8a1f07896d82b10926bd3744a2420009f88974a8"
integrity sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=
map-visit@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f"
@@ -9452,6 +9501,13 @@ path-type@^4.0.0:
resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b"
integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==
pause-stream@^0.0.11:
version "0.0.11"
resolved "https://registry.yarnpkg.com/pause-stream/-/pause-stream-0.0.11.tgz#fe5a34b0cbce12b5aa6a2b403ee2e73b602f1445"
integrity sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=
dependencies:
through "~2.3"
pend@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
@@ -10091,6 +10147,16 @@ readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.6, readable-stre
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
"readable-stream@>=1.0.33-1 <1.1.0-0":
version "1.0.34"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c"
integrity sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=
dependencies:
core-util-is "~1.0.0"
inherits "~2.0.1"
isarray "0.0.1"
string_decoder "~0.10.x"
readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.3.0, readable-stream@^2.3.5, readable-stream@~2.3.6:
version "2.3.7"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57"
@@ -10104,6 +10170,16 @@ readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable
string_decoder "~1.1.1"
util-deprecate "~1.0.1"
readable-stream@~1.1.9:
version "1.1.14"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9"
integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk=
dependencies:
core-util-is "~1.0.0"
inherits "~2.0.1"
isarray "0.0.1"
string_decoder "~0.10.x"
readdir-scoped-modules@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309"
@@ -11136,6 +11212,13 @@ split2@^3.1.1:
dependencies:
readable-stream "^3.0.0"
split@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9"
integrity sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==
dependencies:
through "2"
sprintf-js@~1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
@@ -11209,6 +11292,26 @@ stealthy-require@^1.1.1:
resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b"
integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=
stream-chain@^2.2.3:
version "2.2.4"
resolved "https://registry.yarnpkg.com/stream-chain/-/stream-chain-2.2.4.tgz#2e5e9858e00d25ebaf7e4339e8328ddcab5facad"
integrity sha512-9lsl3YM53V5N/I1C2uJtc3Kavyi3kNYN83VkKb/bMWRk7D9imiFyUPYa0PoZbLohSVOX1mYE9YsmwObZUsth6Q==
stream-combiner@^0.2.2:
version "0.2.2"
resolved "https://registry.yarnpkg.com/stream-combiner/-/stream-combiner-0.2.2.tgz#aec8cbac177b56b6f4fa479ced8c1912cee52858"
integrity sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=
dependencies:
duplexer "~0.1.1"
through "~2.3.4"
stream-json@^1.7.1:
version "1.7.1"
resolved "https://registry.yarnpkg.com/stream-json/-/stream-json-1.7.1.tgz#ec7e414c2eba456c89a4b4e5223794eabc3860c4"
integrity sha512-I7g0IDqvdJXbJ279/D3ZoTx0VMhmKnEF7u38CffeWdF8bfpMPsLo+5fWnkNjO2GU/JjWaRjdH+zmH03q+XGXFw==
dependencies:
stream-chain "^2.2.3"
streamz@~1.8.10:
version "1.8.12"
resolved "https://registry.yarnpkg.com/streamz/-/streamz-1.8.12.tgz#e022bf773fb0093de81f3cd2105390bbc9231613"
@@ -11330,6 +11433,11 @@ string_decoder@^1.1.1:
dependencies:
safe-buffer "~5.2.0"
string_decoder@~0.10.x:
version "0.10.31"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=
string_decoder@~1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
@@ -11664,7 +11772,15 @@ through2@^4.0.2:
dependencies:
readable-stream "3"
through@^2.3.8:
through2@~0.6.3:
version "0.6.5"
resolved "https://registry.yarnpkg.com/through2/-/through2-0.6.5.tgz#41ab9c67b29d57209071410e1d7a7a968cd3ad48"
integrity sha1-QaucZ7KdVyCQcUEOHXp6lozTrUg=
dependencies:
readable-stream ">=1.0.33-1 <1.1.0-0"
xtend ">=4.0.0 <4.1.0-0"
through@2, through@^2.3.8, through@~2.3, through@~2.3.4:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=
@@ -12626,7 +12742,7 @@ xmlchars@^2.2.0:
resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==
xtend@^4.0.0, xtend@~4.0.1:
"xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.0, xtend@~4.0.1:
version "4.0.2"
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==