From c15b2cf0e4ef485a9ebdeb25f2649f57bf3f1d36 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Thu, 13 May 2021 07:58:59 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=B7=F0=9F=8F=BD=20Streaming=20JSON=20W?= =?UTF-8?q?IP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 3 + .../route/routes/importComics.routes.ts | 27 +--- yarn.lock | 122 +++++++++++++++++- 3 files changed, 129 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index 65a0986..718409b 100644 --- a/package.json +++ b/package.json @@ -25,14 +25,17 @@ "babel-polyfill": "^6.26.0", "can-ndjson-stream": "^1.0.2", "comlink-loader": "^2.0.0", + "event-stream": "^4.0.1", "express": "^4.17.1", "fs-extra": "^9.1.0", + "hyperquest": "^2.1.3", "imghash": "^0.0.8", "node-fetch": "^2.6.1", "react": "^17.0.1", "react-dom": "^17.0.1", "react-hooks-worker": "^1.0.0", "sharp": "^0.28.1", + "stream-json": "^1.7.1", "through2": "^4.0.2" }, "devDependencies": { diff --git a/src/server/route/routes/importComics.routes.ts b/src/server/route/routes/importComics.routes.ts index 0c79292..c9b894e 100644 --- a/src/server/route/routes/importComics.routes.ts +++ b/src/server/route/routes/importComics.routes.ts @@ -3,35 +3,22 @@ import { Request, Response } from "express"; import axios from "axios"; import { Readable } from "stream"; import through2 from "through2"; +import hyperquest from "hyperquest"; +import es from "event-stream"; router.route("/getComicCovers").post(async (req: Request, res: Response) => { typeof req.body.extractionOptions === "object" ? req.body.extractionOptions : {}; - const comicBookCoversData = await axios({ - url: "http://localhost:3000/api/import/getComicCovers", - method: "POST", - data: { + const comicBookCoversData = hyperquest.post("http://localhost:3853/api/import/getComicCovers", + { + data: { extractionOptions: req.body.extractionOptions, walkedFolders: req.body.walkedFolders, }, }); - const stream = new Readable({ - objectMode: true, - highWaterMark: 1, - read() {}, - }); - - const ndjsonStream = through2( - { objectMode: true, highWaterMark: 1 }, - (data, enc, cb) => { - cb(null, JSON.stringify(data) + "\n"); - }, - ); - - stream.pipe(ndjsonStream).pipe(res); - stream.push({ data: comicBookCoversData.data }); - stream.push(null); + console.log(comicBookCoversData.data); + }); router.route("/walkFolder").post(async (req: Request, res: Response) => { diff --git a/yarn.lock b/yarn.lock index dbb7a2f..04a40c8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3085,6 +3085,11 @@ buffer-fill@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c" integrity sha1-+PeLdniYiO858gXNY39o5wISKyw= +buffer-from@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-0.1.2.tgz#15f4b9bcef012044df31142c14333caf6e0260d0" + integrity sha512-RiWIenusJsmI2KcvqQABB83tLxCByE3upSP8QU3rJDMVFGPWLvPQJt/O1Su9moRWeH7d+Q2HYb68f6+v+tw2vg== + buffer-from@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" @@ -4458,6 +4463,13 @@ download@^7.1.0: p-event "^2.1.0" pify "^3.0.0" +duplexer2@~0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.0.2.tgz#c614dcf67e2fb14995a91711e5a617e8a60a31db" + integrity sha1-xhTc9n4vsUmVqRcR5aYX6KYKMds= + dependencies: + readable-stream "~1.1.9" + duplexer2@~0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" @@ -4470,6 +4482,11 @@ duplexer3@^0.1.4: resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" integrity sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI= +duplexer@^0.1.1, duplexer@~0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6" + integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -4946,6 +4963,19 @@ etl@^0.6.12: moment "~2.21.0" streamz "~1.8.10" +event-stream@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/event-stream/-/event-stream-4.0.1.tgz#4092808ec995d0dd75ea4580c1df6a74db2cde65" + integrity sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA== + dependencies: + duplexer "^0.1.1" + from "^0.1.7" + map-stream "0.0.7" + pause-stream "^0.0.11" + split "^1.0.1" + stream-combiner "^0.2.2" + through "^2.3.8" + eventemitter3@^4.0.0: version "4.0.7" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" @@ -5508,6 +5538,11 @@ from2@^2.1.1: inherits "^2.0.1" readable-stream "^2.0.0" +from@^0.1.7: + version "0.1.7" + resolved "https://registry.yarnpkg.com/from/-/from-0.1.7.tgz#83c60afc58b9c56997007ed1a768b3ab303a44fe" + integrity sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4= + fs-constants@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" @@ -6216,6 +6251,15 @@ humanize-ms@^1.2.1: dependencies: ms "^2.0.0" +hyperquest@^2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/hyperquest/-/hyperquest-2.1.3.tgz#523127d7a343181b40bf324e231d2576edf52633" + integrity sha512-fUuDOrB47PqNK/BAMOS13v41UoaqIxqSLHX6CAbOD7OfT+/GCWO1/vPLfTNutOeXrv1ikuaZ3yux+33Z9vh+rw== + dependencies: + buffer-from "^0.1.1" + duplexer2 "~0.0.2" + through2 "~0.6.3" + iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" @@ -6438,7 +6482,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.3: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -8028,6 +8072,11 @@ map-obj@^1.0.0, map-obj@^1.0.1: resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d" integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0= +map-stream@0.0.7: + version "0.0.7" + resolved "https://registry.yarnpkg.com/map-stream/-/map-stream-0.0.7.tgz#8a1f07896d82b10926bd3744a2420009f88974a8" + integrity sha1-ih8HiW2CsQkmvTdEokIACfiJdKg= + map-visit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f" @@ -9452,6 +9501,13 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== +pause-stream@^0.0.11: + version "0.0.11" + resolved "https://registry.yarnpkg.com/pause-stream/-/pause-stream-0.0.11.tgz#fe5a34b0cbce12b5aa6a2b403ee2e73b602f1445" + integrity sha1-/lo0sMvOErWqaitAPuLnO2AvFEU= + dependencies: + through "~2.3" + pend@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" @@ -10091,6 +10147,16 @@ readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.6, readable-stre string_decoder "^1.1.1" util-deprecate "^1.0.1" +"readable-stream@>=1.0.33-1 <1.1.0-0": + version "1.0.34" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" + integrity sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw= + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.3.0, readable-stream@^2.3.5, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" @@ -10104,6 +10170,16 @@ readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable string_decoder "~1.1.1" util-deprecate "~1.0.1" +readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk= + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + readdir-scoped-modules@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309" @@ -11136,6 +11212,13 @@ split2@^3.1.1: dependencies: readable-stream "^3.0.0" +split@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" + integrity sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg== + dependencies: + through "2" + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" @@ -11209,6 +11292,26 @@ stealthy-require@^1.1.1: resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks= +stream-chain@^2.2.3: + version "2.2.4" + resolved "https://registry.yarnpkg.com/stream-chain/-/stream-chain-2.2.4.tgz#2e5e9858e00d25ebaf7e4339e8328ddcab5facad" + integrity sha512-9lsl3YM53V5N/I1C2uJtc3Kavyi3kNYN83VkKb/bMWRk7D9imiFyUPYa0PoZbLohSVOX1mYE9YsmwObZUsth6Q== + +stream-combiner@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/stream-combiner/-/stream-combiner-0.2.2.tgz#aec8cbac177b56b6f4fa479ced8c1912cee52858" + integrity sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg= + dependencies: + duplexer "~0.1.1" + through "~2.3.4" + +stream-json@^1.7.1: + version "1.7.1" + resolved "https://registry.yarnpkg.com/stream-json/-/stream-json-1.7.1.tgz#ec7e414c2eba456c89a4b4e5223794eabc3860c4" + integrity sha512-I7g0IDqvdJXbJ279/D3ZoTx0VMhmKnEF7u38CffeWdF8bfpMPsLo+5fWnkNjO2GU/JjWaRjdH+zmH03q+XGXFw== + dependencies: + stream-chain "^2.2.3" + streamz@~1.8.10: version "1.8.12" resolved "https://registry.yarnpkg.com/streamz/-/streamz-1.8.12.tgz#e022bf773fb0093de81f3cd2105390bbc9231613" @@ -11330,6 +11433,11 @@ string_decoder@^1.1.1: dependencies: safe-buffer "~5.2.0" +string_decoder@~0.10.x: + version "0.10.31" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" + integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ= + string_decoder@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" @@ -11664,7 +11772,15 @@ through2@^4.0.2: dependencies: readable-stream "3" -through@^2.3.8: +through2@~0.6.3: + version "0.6.5" + resolved "https://registry.yarnpkg.com/through2/-/through2-0.6.5.tgz#41ab9c67b29d57209071410e1d7a7a968cd3ad48" + integrity sha1-QaucZ7KdVyCQcUEOHXp6lozTrUg= + dependencies: + readable-stream ">=1.0.33-1 <1.1.0-0" + xtend ">=4.0.0 <4.1.0-0" + +through@2, through@^2.3.8, through@~2.3, through@~2.3.4: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= @@ -12626,7 +12742,7 @@ xmlchars@^2.2.0: resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== -xtend@^4.0.0, xtend@~4.0.1: +"xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.0, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==