From 2909d6686b2fa290aae7f1d8ab23f3f334e8b43d Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Tue, 15 Feb 2022 15:34:50 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20Refactored=20endpoints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 114 ------------------------------------ package.json | 1 - services/api.service.ts | 1 - services/library.service.ts | 76 ++++++++++++------------ services/search.service.ts | 4 +- yarn.lock | 50 +--------------- 6 files changed, 41 insertions(+), 205 deletions(-) diff --git a/package-lock.json b/package-lock.json index 37c3a13..ec3bf2f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,6 @@ "@types/string-similarity": "^4.0.0", "7zip-bin": "^5.1.1", "7zip-min": "^1.4.0", - "array-to-ndjson": "^1.0.1", "axios": "^0.25.0", "axios-retry": "^3.2.4", "bree": "^7.1.5", @@ -2457,18 +2456,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/array-to-ndjson": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/array-to-ndjson/-/array-to-ndjson-1.0.1.tgz", - "integrity": "sha1-LkSejKTVdH3jfjMN87h1hD2KlTU=", - "dependencies": { - "is-object": "^1.0.1", - "stream-array": "^1.1.2" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -2964,11 +2951,6 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, - "node_modules/buffer-shims": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz", - "integrity": "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E=" - }, "node_modules/bull": { "version": "3.29.3", "resolved": "https://registry.npmjs.org/bull/-/bull-3.29.3.tgz", @@ -5692,14 +5674,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-object": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz", - "integrity": "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-potential-custom-element-name": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", @@ -13795,41 +13769,6 @@ "node": ">=0.10.0" } }, - "node_modules/stream-array": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/stream-array/-/stream-array-1.1.2.tgz", - "integrity": "sha1-nl9zRfITfDDuO0mLkRToC1K7frU=", - "dependencies": { - "readable-stream": "~2.1.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/stream-array/node_modules/process-nextick-args": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", - "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" - }, - "node_modules/stream-array/node_modules/readable-stream": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.1.5.tgz", - "integrity": "sha1-ZvqLcg4UOLNkaB8q0aY8YYRIydA=", - "dependencies": { - "buffer-shims": "^1.0.0", - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "~1.0.0", - "process-nextick-args": "~1.0.6", - "string_decoder": "~0.10.x", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/stream-array/node_modules/string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - }, "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -16912,15 +16851,6 @@ "is-string": "^1.0.7" } }, - "array-to-ndjson": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/array-to-ndjson/-/array-to-ndjson-1.0.1.tgz", - "integrity": "sha1-LkSejKTVdH3jfjMN87h1hD2KlTU=", - "requires": { - "is-object": "^1.0.1", - "stream-array": "^1.1.2" - } - }, "array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -17295,11 +17225,6 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, - "buffer-shims": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz", - "integrity": "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E=" - }, "bull": { "version": "3.29.3", "resolved": "https://registry.npmjs.org/bull/-/bull-3.29.3.tgz", @@ -19387,11 +19312,6 @@ "has-tostringtag": "^1.0.0" } }, - "is-object": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz", - "integrity": "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==" - }, "is-potential-custom-element-name": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", @@ -25304,40 +25224,6 @@ "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" }, - "stream-array": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/stream-array/-/stream-array-1.1.2.tgz", - "integrity": "sha1-nl9zRfITfDDuO0mLkRToC1K7frU=", - "requires": { - "readable-stream": "~2.1.0" - }, - "dependencies": { - "process-nextick-args": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", - "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" - }, - "readable-stream": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.1.5.tgz", - "integrity": "sha1-ZvqLcg4UOLNkaB8q0aY8YYRIydA=", - "requires": { - "buffer-shims": "^1.0.0", - "core-util-is": "~1.0.0", - "inherits": "~2.0.1", - "isarray": "~1.0.0", - "process-nextick-args": "~1.0.6", - "string_decoder": "~0.10.x", - "util-deprecate": "~1.0.1" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - } - } - }, "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", diff --git a/package.json b/package.json index cd3741f..a048f17 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "@types/string-similarity": "^4.0.0", "7zip-bin": "^5.1.1", "7zip-min": "^1.4.0", - "array-to-ndjson": "^1.0.1", "axios": "^0.25.0", "axios-retry": "^3.2.4", "bree": "^7.1.5", diff --git a/services/api.service.ts b/services/api.service.ts index 1d1d856..ded0205 100644 --- a/services/api.service.ts +++ b/services/api.service.ts @@ -9,7 +9,6 @@ export const io = SocketIOMixin(); export default class ApiService extends Service { public constructor(broker: ServiceBroker) { super(broker); - // @ts-ignore this.parseServiceSchema({ name: "api", mixins: [ApiGateway, SocketIOMixin], diff --git a/services/library.service.ts b/services/library.service.ts index 58830ff..c72ceb2 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -42,7 +42,11 @@ import { } from "moleculer"; import { DbMixin } from "../mixins/db.mixin"; import Comic from "../models/comic.model"; -import { explodePath, walkFolder, getSizeOfDirectory } from "../utils/file.utils"; +import { + explodePath, + walkFolder, + getSizeOfDirectory, +} from "../utils/file.utils"; import { convertXMLToJSON } from "../utils/xml.utils"; import { IExtractComicBookCoverErrorResponse, @@ -50,7 +54,6 @@ import { IExtractionOptions, } from "threetwo-ui-typings"; import { unrarArchive } from "../utils/uncompression.utils"; -import { extractCoverFromFile2 } from "../utils/uncompression.utils"; import { scrapeIssuesFromDOM } from "../utils/scraping.utils"; const ObjectId = require("mongoose").Types.ObjectId; import fsExtra from "fs-extra"; @@ -340,12 +343,12 @@ export default class ImportService extends Service { const volumes = await Comic.aggregate([ { $group: { - _id: "$sourcedMetadata.comicvine.volume.id", - comicObjectId: { $first: "$_id" }, - volumeURI: { - $last: "$sourcedMetadata.comicvine.volume.api_detail_url", + _id: "$sourcedMetadata.comicvine.volume", + comicBookObjectId: { + $last: "$_id", }, count: { $sum: 1 }, + data: { $push: "$$ROOT.sourcedMetadata.comicvine.volumeInformation" }, }, }, { @@ -357,31 +360,7 @@ export default class ImportService extends Service { { $skip: 0 }, { $limit: 5 }, ]); - // 2. Map over the aggregation result and get volume metadata from CV - // 2a. Make a call to comicvine-service - volumesMetadata = map(volumes, async (volume) => { - console.log(volume); - if (!isNil(volume.volumeURI)) { - const volumeMetadata = await ctx.call( - "comicvine.getVolumes", - { - volumeURI: volume.volumeURI, - data: { - format: "json", - fieldList: - "id,name,deck,api_detail_url", - limit: "1", - offset: "0", - }, - } - ); - volumeMetadata["comicObjectId"] = - volume.comicObjectId; - return volumeMetadata; - } - }); - - return Promise.all(volumesMetadata); + return volumes; }, }, @@ -461,7 +440,10 @@ export default class ImportService extends Service { rest: "GET /libraryStatistics", params: {}, handler: async (ctx: Context<{}>) => { - const comicDirectorySize = await getSizeOfDirectory(COMICS_DIRECTORY, [".cbz", ".cbr", ".cb7"]) + const comicDirectorySize = await getSizeOfDirectory( + COMICS_DIRECTORY, + [".cbz", ".cbr", ".cb7"] + ); const totalCount = await Comic.countDocuments({}); const statistics = await Comic.aggregate([ { @@ -488,9 +470,10 @@ export default class ImportService extends Service { issues: [ { $match: { - "sourcedMetadata.comicvine": { "$gt": {} } - } - + "sourcedMetadata.comicvine": { + $gt: {}, + }, + }, }, { $group: { @@ -502,9 +485,28 @@ export default class ImportService extends Service { fileLessComics: [ { $match: { - "rawFileDetails": { "$exists": false } - } + rawFileDetails: { + $exists: false, + }, + }, }, + ], + publisherWithMostComicsInLibrary: [ + { + $unwind: + "$sourcedMetadata.comicvine.volumeInformation.publisher", + }, + { + $group: { + _id: "$sourcedMetadata.comicvine.volumeInformation.publisher.name", + count: { $sum: 1 }, + }, + }, + { $sort: { count: -1 } }, + { $limit: 1 }, + ], + mostPopulatCharacter: [ + ] }, }, diff --git a/services/search.service.ts b/services/search.service.ts index 10aa635..8ee333c 100644 --- a/services/search.service.ts +++ b/services/search.service.ts @@ -9,10 +9,8 @@ import { import { DbMixin } from "../mixins/db.mixin"; import Comic from "../models/comic.model"; -import { refineQuery } from "filename-parser"; -import { each, filter, flatten, isEmpty, isNull } from "lodash"; +import { flatten } from "lodash"; import { eSClient } from "../models/comic.model"; -import arrayToNDJSON from "array-to-ndjson"; const s = eSClient.helpers.msearch(); export default class SettingsService extends Service { diff --git a/yarn.lock b/yarn.lock index cc3395b..9e56a71 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1357,14 +1357,6 @@ "get-intrinsic" "^1.1.1" "is-string" "^1.0.7" -"array-to-ndjson@^1.0.1": - "integrity" "sha1-LkSejKTVdH3jfjMN87h1hD2KlTU=" - "resolved" "https://registry.npmjs.org/array-to-ndjson/-/array-to-ndjson-1.0.1.tgz" - "version" "1.0.1" - dependencies: - "is-object" "^1.0.1" - "stream-array" "^1.1.2" - "array-union@^2.1.0": "integrity" "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==" "resolved" "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz" @@ -1674,11 +1666,6 @@ "resolved" "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz" "version" "1.1.2" -"buffer-shims@^1.0.0": - "integrity" "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E=" - "resolved" "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz" - "version" "1.0.0" - "buffer@^5.5.0": "integrity" "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==" "resolved" "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz" @@ -3474,7 +3461,7 @@ "once" "^1.3.0" "wrappy" "1" -"inherits@^2.0.3", "inherits@^2.0.4", "inherits@~2.0.1", "inherits@~2.0.3", "inherits@2", "inherits@2.0.4": +"inherits@^2.0.3", "inherits@^2.0.4", "inherits@~2.0.3", "inherits@2", "inherits@2.0.4": "integrity" "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" "resolved" "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" "version" "2.0.4" @@ -3685,11 +3672,6 @@ "resolved" "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz" "version" "7.0.0" -"is-object@^1.0.1": - "integrity" "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==" - "resolved" "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz" - "version" "1.0.2" - "is-potential-custom-element-name@^1.0.1": "integrity" "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==" "resolved" "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz" @@ -5778,11 +5760,6 @@ "proc-log@*", "proc-log@^1.0.0": "version" "1.0.0" -"process-nextick-args@~1.0.6": - "integrity" "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" - "resolved" "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz" - "version" "1.0.7" - "process-nextick-args@~2.0.0": "integrity" "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" "resolved" "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz" @@ -5964,19 +5941,6 @@ "string_decoder" "^1.1.1" "util-deprecate" "^1.0.1" -"readable-stream@~2.1.0": - "integrity" "sha1-ZvqLcg4UOLNkaB8q0aY8YYRIydA=" - "resolved" "https://registry.npmjs.org/readable-stream/-/readable-stream-2.1.5.tgz" - "version" "2.1.5" - dependencies: - "buffer-shims" "^1.0.0" - "core-util-is" "~1.0.0" - "inherits" "~2.0.1" - "isarray" "~1.0.0" - "process-nextick-args" "~1.0.6" - "string_decoder" "~0.10.x" - "util-deprecate" "~1.0.1" - "readable-stream@3": "integrity" "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==" "resolved" "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz" @@ -6521,13 +6485,6 @@ "resolved" "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz" "version" "1.1.1" -"stream-array@^1.1.2": - "integrity" "sha1-nl9zRfITfDDuO0mLkRToC1K7frU=" - "resolved" "https://registry.npmjs.org/stream-array/-/stream-array-1.1.2.tgz" - "version" "1.1.2" - dependencies: - "readable-stream" "~2.1.0" - "string_decoder@^1.1.1", "string_decoder@~1.1.1": "integrity" "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==" "resolved" "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" @@ -6535,11 +6492,6 @@ dependencies: "safe-buffer" "~5.1.0" -"string_decoder@~0.10.x": - "integrity" "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - "resolved" "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" - "version" "0.10.31" - "string-argv@^0.3.1": "integrity" "sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg==" "resolved" "https://registry.npmjs.org/string-argv/-/string-argv-0.3.1.tgz"