From 269bdcb5ae4c044144295d92b8bd76a41d00ce2e Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Sun, 19 Dec 2021 09:34:12 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=82=20Recursive=20paginated=20call=20t?= =?UTF-8?q?o=20fetch=20all=20volumes=20WIP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 367 +++++++++++++++++++++++++++++++--- package.json | 3 + services/comicvine.service.ts | 82 +++++++- 3 files changed, 423 insertions(+), 29 deletions(-) diff --git a/package-lock.json b/package-lock.json index 80ab54f..032a06b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "requires": true, "packages": { "": { + "name": "comicvine-service", "version": "0.0.2", "dependencies": { "@types/axios": "^0.14.0", @@ -13,8 +14,11 @@ "@types/node": "^13.9.8", "axios": "^0.21.1", "dotenv": "^10.0.0", + "event-stream": "^4.0.1", + "hyperquest": "^2.1.3", "imghash": "^0.0.9", "ioredis": "^4.28.1", + "JSONStream": "^1.3.5", "leven": "^3.1.0", "lodash": "^4.17.21", "moleculer": "^0.14.0", @@ -2912,8 +2916,7 @@ "node_modules/core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "node_modules/cross-spawn": { "version": "6.0.5", @@ -3186,6 +3189,40 @@ "node": ">=10" } }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==" + }, + "node_modules/duplexer2": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.0.2.tgz", + "integrity": "sha1-xhTc9n4vsUmVqRcR5aYX6KYKMds=", + "dependencies": { + "readable-stream": "~1.1.9" + } + }, + "node_modules/duplexer2/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, "node_modules/ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -3781,6 +3818,20 @@ "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==", "dev": true }, + "node_modules/event-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz", + "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==", + "dependencies": { + "duplexer": "^0.1.1", + "from": "^0.1.7", + "map-stream": "0.0.7", + "pause-stream": "^0.0.11", + "split": "^1.0.1", + "stream-combiner": "^0.2.2", + "through": "^2.3.8" + } + }, "node_modules/eventemitter2": { "version": "6.4.4", "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.4.tgz", @@ -4376,6 +4427,11 @@ "node": ">= 0.6" } }, + "node_modules/from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=" + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -4743,6 +4799,21 @@ "node": ">=8.12.0" } }, + "node_modules/hyperquest": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/hyperquest/-/hyperquest-2.1.3.tgz", + "integrity": "sha512-fUuDOrB47PqNK/BAMOS13v41UoaqIxqSLHX6CAbOD7OfT+/GCWO1/vPLfTNutOeXrv1ikuaZ3yux+33Z9vh+rw==", + "dependencies": { + "buffer-from": "^0.1.1", + "duplexer2": "~0.0.2", + "through2": "~0.6.3" + } + }, + "node_modules/hyperquest/node_modules/buffer-from": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-0.1.2.tgz", + "integrity": "sha512-RiWIenusJsmI2KcvqQABB83tLxCByE3upSP8QU3rJDMVFGPWLvPQJt/O1Su9moRWeH7d+Q2HYb68f6+v+tw2vg==" + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -7352,9 +7423,9 @@ "dev": true }, "node_modules/json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", "dev": true }, "node_modules/json-schema-traverse": { @@ -7390,19 +7461,42 @@ "node": ">=6" } }, - "node_modules/jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "dev": true, + "node_modules/jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", "engines": [ - "node >=0.6.0" - ], + "node >= 0.2.0" + ] + }, + "node_modules/JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "dependencies": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + }, + "bin": { + "JSONStream": "bin.js" + }, + "engines": { + "node": "*" + } + }, + "node_modules/jsprim": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", + "dev": true, "dependencies": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", - "json-schema": "0.2.3", + "json-schema": "0.4.0", "verror": "1.10.0" + }, + "engines": { + "node": ">=0.6.0" } }, "node_modules/kind-of": { @@ -7773,6 +7867,11 @@ "node": ">=0.10.0" } }, + "node_modules/map-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", + "integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=" + }, "node_modules/map-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", @@ -8948,6 +9047,14 @@ "node": ">=4" } }, + "node_modules/pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", + "dependencies": { + "through": "~2.3" + } + }, "node_modules/performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -10338,6 +10445,17 @@ "integrity": "sha512-oie3/+gKf7QtpitB0LYLETe+k8SifzsX4KixvpOsbI6S0kRiRQ5MKOio8eMSAKQ17N06+wdEOXRiId+zOxo0hA==", "dev": true }, + "node_modules/split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, "node_modules/split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", @@ -10520,6 +10638,15 @@ "node": ">=0.10.0" } }, + "node_modules/stream-combiner": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=", + "dependencies": { + "duplexer": "~0.1.1", + "through": "~2.3.4" + } + }, "node_modules/streamsearch": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", @@ -10808,8 +10935,37 @@ "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", - "dev": true + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "node_modules/through2": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-0.6.5.tgz", + "integrity": "sha1-QaucZ7KdVyCQcUEOHXp6lozTrUg=", + "dependencies": { + "readable-stream": ">=1.0.33-1 <1.1.0-0", + "xtend": ">=4.0.0 <4.1.0-0" + } + }, + "node_modules/through2/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "node_modules/through2/node_modules/readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "node_modules/through2/node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" }, "node_modules/timers-ext": { "version": "0.1.7", @@ -11612,6 +11768,14 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", @@ -14037,8 +14201,7 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "cross-spawn": { "version": "6.0.5", @@ -14250,6 +14413,42 @@ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz", "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==" }, + "duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==" + }, + "duplexer2": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.0.2.tgz", + "integrity": "sha1-xhTc9n4vsUmVqRcR5aYX6KYKMds=", + "requires": { + "readable-stream": "~1.1.9" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -14760,6 +14959,20 @@ } } }, + "event-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz", + "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==", + "requires": { + "duplexer": "^0.1.1", + "from": "^0.1.7", + "map-stream": "0.0.7", + "pause-stream": "^0.0.11", + "split": "^1.0.1", + "stream-combiner": "^0.2.2", + "through": "^2.3.8" + } + }, "eventemitter2": { "version": "6.4.4", "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.4.tgz", @@ -15227,6 +15440,11 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, + "from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=" + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -15504,6 +15722,23 @@ "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==", "dev": true }, + "hyperquest": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/hyperquest/-/hyperquest-2.1.3.tgz", + "integrity": "sha512-fUuDOrB47PqNK/BAMOS13v41UoaqIxqSLHX6CAbOD7OfT+/GCWO1/vPLfTNutOeXrv1ikuaZ3yux+33Z9vh+rw==", + "requires": { + "buffer-from": "^0.1.1", + "duplexer2": "~0.0.2", + "through2": "~0.6.3" + }, + "dependencies": { + "buffer-from": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-0.1.2.tgz", + "integrity": "sha512-RiWIenusJsmI2KcvqQABB83tLxCByE3upSP8QU3rJDMVFGPWLvPQJt/O1Su9moRWeH7d+Q2HYb68f6+v+tw2vg==" + } + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -17462,9 +17697,9 @@ "dev": true }, "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", "dev": true }, "json-schema-traverse": { @@ -17494,15 +17729,29 @@ "minimist": "^1.2.5" } }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" + }, + "JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } + }, "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.2.tgz", + "integrity": "sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==", "dev": true, "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", - "json-schema": "0.2.3", + "json-schema": "0.4.0", "verror": "1.10.0" } }, @@ -17795,6 +18044,11 @@ "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=", "dev": true }, + "map-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", + "integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=" + }, "map-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", @@ -18624,6 +18878,14 @@ "pify": "^3.0.0" } }, + "pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", + "requires": { + "through": "~2.3" + } + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -19727,6 +19989,14 @@ "integrity": "sha512-oie3/+gKf7QtpitB0LYLETe+k8SifzsX4KixvpOsbI6S0kRiRQ5MKOio8eMSAKQ17N06+wdEOXRiId+zOxo0hA==", "dev": true }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "requires": { + "through": "2" + } + }, "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", @@ -19870,6 +20140,15 @@ "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", "dev": true }, + "stream-combiner": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=", + "requires": { + "duplexer": "~0.1.1", + "through": "~2.3.4" + } + }, "streamsearch": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", @@ -20085,8 +20364,39 @@ "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", - "dev": true + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, + "through2": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-0.6.5.tgz", + "integrity": "sha1-QaucZ7KdVyCQcUEOHXp6lozTrUg=", + "requires": { + "readable-stream": ">=1.0.33-1 <1.1.0-0", + "xtend": ">=4.0.0 <4.1.0-0" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } }, "timers-ext": { "version": "0.1.7", @@ -20733,6 +21043,11 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, "y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", diff --git a/package.json b/package.json index 1f89194..4a6c9ae 100644 --- a/package.json +++ b/package.json @@ -41,8 +41,11 @@ "@types/node": "^13.9.8", "axios": "^0.21.1", "dotenv": "^10.0.0", + "event-stream": "^4.0.1", + "hyperquest": "^2.1.3", "imghash": "^0.0.9", "ioredis": "^4.28.1", + "JSONStream": "^1.3.5", "leven": "^3.1.0", "lodash": "^4.17.21", "moleculer": "^0.14.0", diff --git a/services/comicvine.service.ts b/services/comicvine.service.ts index c88466b..6570626 100644 --- a/services/comicvine.service.ts +++ b/services/comicvine.service.ts @@ -1,7 +1,7 @@ "use strict"; import { Service, ServiceBroker, Context } from "moleculer"; -import axios from "axios"; +import axios, { AxiosResponse } from "axios"; import { matchScorer } from "../utils/searchmatchscorer.utils"; const CV_BASE_URL = "https://comicvine.gamespot.com/api/"; console.log("KEYYYYYYYY", process.env.COMICVINE_API_KEY); @@ -10,6 +10,7 @@ export default class ComicVineService extends Service { super(broker); this.parseServiceSchema({ name: "comicvine", + requestTimeout: 90000, actions: { fetchResource: { rest: "/fetchresource", @@ -40,7 +41,15 @@ export default class ComicVineService extends Service { }; }> ): Promise => { - const { format, sort, query, fieldList, limit, offset, resources } = ctx.params; + const { + format, + sort, + query, + fieldList, + limit, + offset, + resources, + } = ctx.params; const response = await axios.request({ url: CV_BASE_URL + @@ -109,7 +118,9 @@ export default class ComicVineService extends Service { ) => { const response = await axios.request({ url: - ctx.params.volumeURI + "?api_key=" + process.env.COMICVINE_API_KEY, + ctx.params.volumeURI + + "?api_key=" + + process.env.COMICVINE_API_KEY, params: ctx.params.data, headers: { Accept: "application/json" }, }); @@ -117,6 +128,71 @@ export default class ComicVineService extends Service { return data; }, }, + volumeBasedSearch: { + rest: "POST /volumeBasedSearch", + params: {}, + headers: { Accept: "application/json" }, + bulkhead: { + enabled: true, + concurrency: 10, + maxQueueSize: 10, + }, + handler: async ( + ctx: Context<{ + format: string; + sort: string; + query: string; + fieldList: string; + limit: number; + offset: number; + resources: string; + scorerConfiguration?: { + searchQuery: { + issue: object; + series: object; + }; + rawFileDetails: object; + }; + }> + ) => { + const fo: any = []; + const foo = await this.fetchVolumesFromCV( + ctx.params, + fo + ); + return foo; + }, + }, + }, + methods: { + fetchVolumesFromCV: async (params, output: any[] = []) => { + let currentPage = parseInt(params.page, 10); + const response = await axios.request({ + url: + CV_BASE_URL + + "search" + + "?api_key=" + + process.env.COMICVINE_API_KEY, + params, + headers: { Accept: "application/json" }, + }); + + const { data } = response; + // 1. calculate total pages + const totalPages = Math.floor( + parseInt(data.number_of_total_results, 10) / + parseInt(params.limit, 10) + ); + if (currentPage < totalPages) { + output.push(data.results); + currentPage += 1; + params.page = currentPage; + console.log(currentPage); + return await this.fetchVolumesFromCV(params, output); + } else { + return output; + } + }, }, }); }