From 833c4570e1f224f7cbcaf6970d9ed79eb52af11e Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Thu, 13 May 2021 10:32:03 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B6=20Still=20WIP=20on=20Streaming=20J?= =?UTF-8?q?SON?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 5 +++ .../route/routes/importComics.routes.ts | 16 ++++--- yarn.lock | 45 ++++++++++++++++++- 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 718409b..dab2b5a 100644 --- a/package.json +++ b/package.json @@ -16,12 +16,16 @@ "license": "MIT", "dependencies": { "@babel/runtime": "^7.13.17", + "@types/event-stream": "^3.3.34", "@types/node-fetch": "^2.5.10", + "@types/oboe": "^2.1.0", "@types/react": "^17.0.3", "@types/react-dom": "^17.0.2", "@types/react-redux": "^7.1.16", "@types/react-router-dom": "^5.1.7", "@types/sharp": "^0.28.0", + "@types/through2": "^2.0.36", + "JSONStream": "^1.3.5", "babel-polyfill": "^6.26.0", "can-ndjson-stream": "^1.0.2", "comlink-loader": "^2.0.0", @@ -31,6 +35,7 @@ "hyperquest": "^2.1.3", "imghash": "^0.0.8", "node-fetch": "^2.6.1", + "oboe": "^2.1.5", "react": "^17.0.1", "react-dom": "^17.0.1", "react-hooks-worker": "^1.0.0", diff --git a/src/server/route/routes/importComics.routes.ts b/src/server/route/routes/importComics.routes.ts index c9b894e..d21a6a9 100644 --- a/src/server/route/routes/importComics.routes.ts +++ b/src/server/route/routes/importComics.routes.ts @@ -1,24 +1,28 @@ import router from "../router"; import { Request, Response } from "express"; import axios from "axios"; -import { Readable } from "stream"; +import stream from "stream"; import through2 from "through2"; import hyperquest from "hyperquest"; import es from "event-stream"; +import JSONStream from "JSONStream"; +import oboe from "oboe"; router.route("/getComicCovers").post(async (req: Request, res: Response) => { typeof req.body.extractionOptions === "object" ? req.body.extractionOptions : {}; - const comicBookCoversData = hyperquest.post("http://localhost:3853/api/import/getComicCovers", - { - data: { + oboe({ + url: "http://localhost:3000/api/import/getComicCovers", + method: "POST", + body: { extractionOptions: req.body.extractionOptions, walkedFolders: req.body.walkedFolders, }, + }).on("node", ".*", (data) => { + console.log(data); + res.write(JSON.stringify(data)); }); - console.log(comicBookCoversData.data); - }); router.route("/walkFolder").post(async (req: Request, res: Response) => { diff --git a/yarn.lock b/yarn.lock index 04a40c8..23bd36a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1635,6 +1635,13 @@ resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.46.tgz#0fb6bfbbeabd7a30880504993369c4bf1deab1fe" integrity sha512-laIjwTQaD+5DukBZaygQ79K1Z0jb1bPEMRrkXSLjtCcZm+abyp5YbrqpSLzD42FwWW6gK/aS4NYpJ804nG2brg== +"@types/event-stream@^3.3.34": + version "3.3.34" + resolved "https://registry.yarnpkg.com/@types/event-stream/-/event-stream-3.3.34.tgz#104bcedd5c61f90917b734bde04e61c6e64f03e1" + integrity sha512-LLiivgWKii4JeMzFy3trrxqkRrVSdue8WmbXyHuSJLwNrhIQU5MTrc65jhxEPwMyh5HR1xevSdD+k2nnSRKw9g== + dependencies: + "@types/node" "*" + "@types/express-serve-static-core@^4.17.18": version "4.17.18" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.18.tgz#8371e260f40e0e1ca0c116a9afcd9426fa094c40" @@ -1786,6 +1793,13 @@ resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e" integrity sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA== +"@types/oboe@^2.1.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@types/oboe/-/oboe-2.1.0.tgz#a4965cd50010c6ce9ed30626f51c4a8c8264a5fe" + integrity sha512-2F95dk3QRgauL9gTUrydXkmnHA1tSCMzqwlSwXlfuBW8Y8F4IfgHecdTf7x4UX9sn3dpNKpfk297r+f/YHJXtA== + dependencies: + "@types/node" "*" + "@types/pino-pretty@*": version "4.7.0" resolved "https://registry.yarnpkg.com/@types/pino-pretty/-/pino-pretty-4.7.0.tgz#e4a18541f8464d1cc48216f5593cc6a0e62dc2c3" @@ -1918,6 +1932,13 @@ resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.0.tgz#7036640b4e21cc2f259ae826ce843d277dad8cff" integrity sha512-RJJrrySY7A8havqpGObOB4W92QXKJo63/jFLLgpvOtsGUqbQZ9Sbgl35KMm1DjC6j7AvmmU2bIno+3IyEaemaw== +"@types/through2@^2.0.36": + version "2.0.36" + resolved "https://registry.yarnpkg.com/@types/through2/-/through2-2.0.36.tgz#35fda0db635827d44c0e08e2c94653e647574a00" + integrity sha512-vuifQksQHJXhV9McpVsXKuhnf3lsoX70PnhcqIAbs9dqLH2NgrGz0DzZPDY3+Yh6eaRqcE1gnCQ6QhBn1/PT5A== + dependencies: + "@types/node" "*" + "@types/unzipper@^0.10.3": version "0.10.3" resolved "https://registry.yarnpkg.com/@types/unzipper/-/unzipper-0.10.3.tgz#9eea872fb1fa460da76f253878b6275af588f464" @@ -2155,6 +2176,14 @@ resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d" integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== +JSONStream@^1.3.5: + version "1.3.5" + resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0" + integrity sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ== + dependencies: + jsonparse "^1.2.0" + through ">=2.2.7 <3" + abab@^2.0.3, abab@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a" @@ -6184,6 +6213,11 @@ http-errors@~1.7.2: statuses ">= 1.5.0 < 2" toidentifier "1.0.0" +http-https@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/http-https/-/http-https-1.0.0.tgz#2f908dd5f1db4068c058cd6e6d4ce392c913389b" + integrity sha1-L5CN1fHbQGjAWM1ubUzjkskTOJs= + http-parser-js@>=0.5.1: version "0.5.3" resolved "https://registry.yarnpkg.com/http-parser-js/-/http-parser-js-0.5.3.tgz#01d2709c79d41698bb01d4decc5e9da4e4a033d9" @@ -7584,7 +7618,7 @@ jsonfile@^6.0.1: optionalDependencies: graceful-fs "^4.1.6" -jsonparse@^1.3.1: +jsonparse@^1.2.0, jsonparse@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA= @@ -9052,6 +9086,13 @@ object.values@^1.1.1: es-abstract "^1.18.0-next.1" has "^1.0.3" +oboe@^2.1.5: + version "2.1.5" + resolved "https://registry.yarnpkg.com/oboe/-/oboe-2.1.5.tgz#5554284c543a2266d7a38f17e073821fbde393cd" + integrity sha1-VVQoTFQ6ImbXo48X4HOCH73jk80= + dependencies: + http-https "^1.0.0" + obuf@^1.0.0, obuf@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/obuf/-/obuf-1.1.2.tgz#09bea3343d41859ebd446292d11c9d4db619084e" @@ -11780,7 +11821,7 @@ through2@~0.6.3: readable-stream ">=1.0.33-1 <1.1.0-0" xtend ">=4.0.0 <4.1.0-0" -through@2, through@^2.3.8, through@~2.3, through@~2.3.4: +through@2, "through@>=2.2.7 <3", through@^2.3.8, through@~2.3, through@~2.3.4: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=