From 2c9bd550808446123a8e19181fe8ccdcee6bd40f Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Tue, 1 Feb 2022 09:32:56 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=82=20Integrated=20elasticsearch=20sea?= =?UTF-8?q?rch=20for=20issues=20in=20volumes=20with=20BullMQ?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/comic.model.ts | 2 +- package-lock.json | 361 ++++++++++++++++++++++++++++++- package.json | 4 + services/import.service.ts | 58 ++++- services/libraryqueue.service.ts | 73 ++++++- services/search.service.ts | 33 ++- 6 files changed, 517 insertions(+), 14 deletions(-) diff --git a/models/comic.model.ts b/models/comic.model.ts index cde7b1c..9570931 100644 --- a/models/comic.model.ts +++ b/models/comic.model.ts @@ -3,7 +3,7 @@ var mexp = require('mongoose-elasticsearch-xp').v7; const paginate = require("mongoose-paginate-v2"); const { Client } = require("@elastic/elasticsearch"); -console.log("YENADADADADA", Client) + const eSClient = new Client({ node: "http://tower.local:9200", auth: { diff --git a/package-lock.json b/package-lock.json index cc04806..3e783b1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,14 +10,18 @@ "dependencies": { "@elastic/elasticsearch": "^7.16.0", "@root/walk": "^1.1.0", + "@types/axios": "^0.14.0", "@types/jest": "^25.1.4", "@types/mkdirp": "^1.0.0", "@types/node": "^13.9.8", "@types/string-similarity": "^4.0.0", "7zip-bin": "^5.1.1", "7zip-min": "^1.4.0", + "axios": "^0.25.0", + "axios-extensions": "^3.1.3", "chokidar": "^3.5.3", "dotenv": "^10.0.0", + "filename-parser": "^1.0.0", "fs-extra": "^10.0.0", "imghash": "^0.0.9", "ioredis": "^4.28.1", @@ -575,6 +579,18 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/runtime-corejs3": { + "version": "7.16.8", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.16.8.tgz", + "integrity": "sha512-3fKhuICS1lMz0plI5ktOE/yEtBRMVxplzRkdn6mJQ197XiY0JnrzYV0+Mxozq3JZ8SBV9Ecurmw1XsGbwOf+Sg==", + "dependencies": { + "core-js-pure": "^3.20.2", + "regenerator-runtime": "^0.13.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.16.7.tgz", @@ -1760,6 +1776,15 @@ "optional": true, "peer": true }, + "node_modules/@types/axios": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz", + "integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=", + "deprecated": "This is a stub types definition for axios (https://github.com/mzabriskie/axios). axios provides its own type definitions, so you don't need @types/axios installed!", + "dependencies": { + "axios": "*" + } + }, "node_modules/@types/babel__core": { "version": "7.1.18", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.18.tgz", @@ -1882,6 +1907,11 @@ "integrity": "sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==", "dev": true }, + "node_modules/@types/lru-cache": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz", + "integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw==" + }, "node_modules/@types/mkdirp": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz", @@ -2483,6 +2513,41 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, + "node_modules/axios": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz", + "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==", + "dependencies": { + "follow-redirects": "^1.14.7" + } + }, + "node_modules/axios-extensions": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/axios-extensions/-/axios-extensions-3.1.3.tgz", + "integrity": "sha512-/OB9OcJLNOIx9pdW4m4/hFRvNo12wlX5BaprIzqpMaLR02I88Mr98/wW4QN9rhx0/yg9rM7i6Af/RpV4MyxXjA==", + "dependencies": { + "@types/lru-cache": "^4.1.1", + "lru-cache": "^5.1.1", + "tslib": "^1.9.0", + "util": "^0.11.1" + }, + "peerDependencies": { + "axios": "*" + } + }, + "node_modules/axios-extensions/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/axios-extensions/node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + }, "node_modules/babel-jest": { "version": "27.4.6", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.4.6.tgz", @@ -3199,6 +3264,46 @@ "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==" }, + "node_modules/compromise": { + "version": "13.11.4", + "resolved": "https://registry.npmjs.org/compromise/-/compromise-13.11.4.tgz", + "integrity": "sha512-nBITcNdqIHSVDDluaG6guyFFCSNXN+Hu87fU8VlhkE5Z0PwTZN1nro2O7a8JcUH88nB5EOzrxd9zKfXLSNFqcg==", + "dependencies": { + "efrt-unpack": "2.2.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/compromise-dates": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/compromise-dates/-/compromise-dates-2.2.1.tgz", + "integrity": "sha512-mtdK67DR4EvowLLh0YJ3aMxMtKXDViQBJTNNdHI5ck+DVhCCd7rYnNcwC2eSES+3tup1JB9+InOzmvyEuFyjLQ==", + "dependencies": { + "spacetime": "6.16.0", + "spacetime-holiday": "0.1.0" + }, + "peerDependencies": { + "compromise": ">=12.0.0", + "compromise-numbers": ">=1.0.0" + } + }, + "node_modules/compromise-numbers": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/compromise-numbers/-/compromise-numbers-1.4.0.tgz", + "integrity": "sha512-3ceRpwZIWduVSMYn54ET1ELdI7bvXQk42uDwxffxiJBxgKCwcCfVbiLuTG62cI+qTHchwLDh4vp9i3WARXROFQ==", + "peerDependencies": { + "compromise": ">=12.0.0" + } + }, + "node_modules/compromise-sentences": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/compromise-sentences/-/compromise-sentences-0.3.0.tgz", + "integrity": "sha512-L6d5PXmmyPyWRMoJURYOM0yILhtV37zCOy9ZW8Xd6KgY+2qQBTwuokekaGL9MDpSH3/grRDL+AHKytYpGrTEmA==", + "peerDependencies": { + "compromise": ">=12.0.0" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -3234,6 +3339,16 @@ "node": ">= 0.6" } }, + "node_modules/core-js-pure": { + "version": "3.20.3", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.20.3.tgz", + "integrity": "sha512-Q2H6tQ5MtPtcC7f3HxJ48i4Q7T9ybPKgvWyuH7JXIoNa2pm0KuBnycsET/qw1SLLZYfbsbrZQNMeIOClb+6WIA==", + "hasInstallScript": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", @@ -3564,6 +3679,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "node_modules/efrt-unpack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/efrt-unpack/-/efrt-unpack-2.2.0.tgz", + "integrity": "sha512-9xUSSj7qcUxz+0r4X3+bwUNttEfGfK5AH+LVa1aTpqdAfrN5VhROYCfcF+up4hp5OL7IUKcZJJrzAGipQRDoiQ==" + }, "node_modules/elasticsearch": { "version": "16.7.2", "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz", @@ -4562,6 +4682,20 @@ "node": ">=6" } }, + "node_modules/filename-parser": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/filename-parser/-/filename-parser-1.0.0.tgz", + "integrity": "sha512-3j7TgfElImSXhYcBzCP75+mC08IdRGmIBOm6XcJciI3xnQU6A4imzzUEJ2Ps717s5ycBkzQgUtjArGWGh/Gmkg==", + "dependencies": { + "compromise": "^13.11.4", + "compromise-dates": "^2.2.1", + "compromise-numbers": "^1.4.0", + "compromise-sentences": "^0.3.0", + "lodash": "^4.17.21", + "voca": "^1.4.0", + "xregexp": "^5.1.0" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -4612,6 +4746,25 @@ "integrity": "sha512-8/sOawo8tJ4QOBX8YlQBMxL8+RLZfxMQOif9o0KUKTNTjMYElWPE0r/m5VNFxTRd0NSw8qSy8dajrwX4RYI1Hw==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.14.7", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz", + "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -10027,6 +10180,11 @@ "node": ">=4" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.9", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" + }, "node_modules/regexp-clone": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/regexp-clone/-/regexp-clone-1.0.0.tgz", @@ -10642,6 +10800,19 @@ "source-map": "^0.6.0" } }, + "node_modules/spacetime": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/spacetime/-/spacetime-6.16.0.tgz", + "integrity": "sha512-mkuniNOp6ssfPyJidj81tb54zKxK4vEKPTmcUsC/NEGIF8S07ppoSotdg6numT1/26rthQYmdxMY/M5a9WeJVQ==" + }, + "node_modules/spacetime-holiday": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/spacetime-holiday/-/spacetime-holiday-0.1.0.tgz", + "integrity": "sha512-rYIpSDbHnznZRstUrmYYFAaruW8e96t+1JfS0b6qMiAAQ2DrkLKc8oMotAAkB9qMTUwXXf5bIkdTHfP434uitQ==", + "peerDependencies": { + "spacetime": "^6.3.0" + } + }, "node_modules/sparse-bitfield": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", @@ -11320,8 +11491,7 @@ "node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/tsutils": { "version": "3.21.0", @@ -11478,6 +11648,14 @@ "punycode": "^2.1.0" } }, + "node_modules/util": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", + "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "dependencies": { + "inherits": "2.0.3" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -11498,6 +11676,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/util/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -11561,6 +11744,11 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "node_modules/voca": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/voca/-/voca-1.4.0.tgz", + "integrity": "sha512-8Xz4H3vhYRGbFupLtl6dHwMx0ojUcjt0HYkqZ9oBCfipd/5mD7Md58m2/dq7uPuZU/0T3Gb1m66KS9jn+I+14Q==" + }, "node_modules/w3c-hr-time": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", @@ -11832,6 +12020,14 @@ "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" }, + "node_modules/xregexp": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-5.1.0.tgz", + "integrity": "sha512-PynwUWtXnSZr8tpQlDPMZfPTyv78EYuA4oI959ukxcQ0a9O/lvndLVKy5wpImzzA26eMxpZmnAXJYiQA13AtWA==", + "dependencies": { + "@babel/runtime-corejs3": "^7.14.9" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -12318,6 +12514,15 @@ "@babel/helper-plugin-utils": "^7.16.7" } }, + "@babel/runtime-corejs3": { + "version": "7.16.8", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.16.8.tgz", + "integrity": "sha512-3fKhuICS1lMz0plI5ktOE/yEtBRMVxplzRkdn6mJQ197XiY0JnrzYV0+Mxozq3JZ8SBV9Ecurmw1XsGbwOf+Sg==", + "requires": { + "core-js-pure": "^3.20.2", + "regenerator-runtime": "^0.13.4" + } + }, "@babel/template": { "version": "7.16.7", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.16.7.tgz", @@ -13241,6 +13446,14 @@ "optional": true, "peer": true }, + "@types/axios": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz", + "integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=", + "requires": { + "axios": "*" + } + }, "@types/babel__core": { "version": "7.1.18", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.18.tgz", @@ -13363,6 +13576,11 @@ "integrity": "sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==", "dev": true }, + "@types/lru-cache": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz", + "integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw==" + }, "@types/mkdirp": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz", @@ -13797,6 +14015,40 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz", "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==" }, + "axios": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz", + "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==", + "requires": { + "follow-redirects": "^1.14.7" + } + }, + "axios-extensions": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/axios-extensions/-/axios-extensions-3.1.3.tgz", + "integrity": "sha512-/OB9OcJLNOIx9pdW4m4/hFRvNo12wlX5BaprIzqpMaLR02I88Mr98/wW4QN9rhx0/yg9rM7i6Af/RpV4MyxXjA==", + "requires": { + "@types/lru-cache": "^4.1.1", + "lru-cache": "^5.1.1", + "tslib": "^1.9.0", + "util": "^0.11.1" + }, + "dependencies": { + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "requires": { + "yallist": "^3.0.2" + } + }, + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + } + } + }, "babel-jest": { "version": "27.4.6", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.4.6.tgz", @@ -14339,6 +14591,35 @@ "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==" }, + "compromise": { + "version": "13.11.4", + "resolved": "https://registry.npmjs.org/compromise/-/compromise-13.11.4.tgz", + "integrity": "sha512-nBITcNdqIHSVDDluaG6guyFFCSNXN+Hu87fU8VlhkE5Z0PwTZN1nro2O7a8JcUH88nB5EOzrxd9zKfXLSNFqcg==", + "requires": { + "efrt-unpack": "2.2.0" + } + }, + "compromise-dates": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/compromise-dates/-/compromise-dates-2.2.1.tgz", + "integrity": "sha512-mtdK67DR4EvowLLh0YJ3aMxMtKXDViQBJTNNdHI5ck+DVhCCd7rYnNcwC2eSES+3tup1JB9+InOzmvyEuFyjLQ==", + "requires": { + "spacetime": "6.16.0", + "spacetime-holiday": "0.1.0" + } + }, + "compromise-numbers": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/compromise-numbers/-/compromise-numbers-1.4.0.tgz", + "integrity": "sha512-3ceRpwZIWduVSMYn54ET1ELdI7bvXQk42uDwxffxiJBxgKCwcCfVbiLuTG62cI+qTHchwLDh4vp9i3WARXROFQ==", + "requires": {} + }, + "compromise-sentences": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/compromise-sentences/-/compromise-sentences-0.3.0.tgz", + "integrity": "sha512-L6d5PXmmyPyWRMoJURYOM0yILhtV37zCOy9ZW8Xd6KgY+2qQBTwuokekaGL9MDpSH3/grRDL+AHKytYpGrTEmA==", + "requires": {} + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -14368,6 +14649,11 @@ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==" }, + "core-js-pure": { + "version": "3.20.3", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.20.3.tgz", + "integrity": "sha512-Q2H6tQ5MtPtcC7f3HxJ48i4Q7T9ybPKgvWyuH7JXIoNa2pm0KuBnycsET/qw1SLLZYfbsbrZQNMeIOClb+6WIA==" + }, "core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", @@ -14628,6 +14914,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "efrt-unpack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/efrt-unpack/-/efrt-unpack-2.2.0.tgz", + "integrity": "sha512-9xUSSj7qcUxz+0r4X3+bwUNttEfGfK5AH+LVa1aTpqdAfrN5VhROYCfcF+up4hp5OL7IUKcZJJrzAGipQRDoiQ==" + }, "elasticsearch": { "version": "16.7.2", "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz", @@ -15438,6 +15729,20 @@ "resolved": "https://registry.npmjs.org/file-type/-/file-type-10.11.0.tgz", "integrity": "sha512-uzk64HRpUZyTGZtVuvrjP0FYxzQrBf4rojot6J65YMEbwBLB0CWm0CLojVpwpmFmxcE/lkvYICgfcGozbBq6rw==" }, + "filename-parser": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/filename-parser/-/filename-parser-1.0.0.tgz", + "integrity": "sha512-3j7TgfElImSXhYcBzCP75+mC08IdRGmIBOm6XcJciI3xnQU6A4imzzUEJ2Ps717s5ycBkzQgUtjArGWGh/Gmkg==", + "requires": { + "compromise": "^13.11.4", + "compromise-dates": "^2.2.1", + "compromise-numbers": "^1.4.0", + "compromise-sentences": "^0.3.0", + "lodash": "^4.17.21", + "voca": "^1.4.0", + "xregexp": "^5.1.0" + } + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -15476,6 +15781,11 @@ "integrity": "sha512-8/sOawo8tJ4QOBX8YlQBMxL8+RLZfxMQOif9o0KUKTNTjMYElWPE0r/m5VNFxTRd0NSw8qSy8dajrwX4RYI1Hw==", "dev": true }, + "follow-redirects": { + "version": "1.14.7", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz", + "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==" + }, "for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -19416,6 +19726,11 @@ "redis-errors": "^1.0.0" } }, + "regenerator-runtime": { + "version": "0.13.9", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" + }, "regexp-clone": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/regexp-clone/-/regexp-clone-1.0.0.tgz", @@ -19875,6 +20190,17 @@ "source-map": "^0.6.0" } }, + "spacetime": { + "version": "6.16.0", + "resolved": "https://registry.npmjs.org/spacetime/-/spacetime-6.16.0.tgz", + "integrity": "sha512-mkuniNOp6ssfPyJidj81tb54zKxK4vEKPTmcUsC/NEGIF8S07ppoSotdg6numT1/26rthQYmdxMY/M5a9WeJVQ==" + }, + "spacetime-holiday": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/spacetime-holiday/-/spacetime-holiday-0.1.0.tgz", + "integrity": "sha512-rYIpSDbHnznZRstUrmYYFAaruW8e96t+1JfS0b6qMiAAQ2DrkLKc8oMotAAkB9qMTUwXXf5bIkdTHfP434uitQ==", + "requires": {} + }, "sparse-bitfield": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", @@ -20409,8 +20735,7 @@ "tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "tsutils": { "version": "3.21.0", @@ -20524,6 +20849,21 @@ "punycode": "^2.1.0" } }, + "util": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", + "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "requires": { + "inherits": "2.0.3" + }, + "dependencies": { + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + } + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -20593,6 +20933,11 @@ } } }, + "voca": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/voca/-/voca-1.4.0.tgz", + "integrity": "sha512-8Xz4H3vhYRGbFupLtl6dHwMx0ojUcjt0HYkqZ9oBCfipd/5mD7Md58m2/dq7uPuZU/0T3Gb1m66KS9jn+I+14Q==" + }, "w3c-hr-time": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz", @@ -20807,6 +21152,14 @@ "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" }, + "xregexp": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-5.1.0.tgz", + "integrity": "sha512-PynwUWtXnSZr8tpQlDPMZfPTyv78EYuA4oI959ukxcQ0a9O/lvndLVKy5wpImzzA26eMxpZmnAXJYiQA13AtWA==", + "requires": { + "@babel/runtime-corejs3": "^7.14.9" + } + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index c8cb4e2..aed69a6 100644 --- a/package.json +++ b/package.json @@ -37,14 +37,18 @@ "dependencies": { "@elastic/elasticsearch": "^7.16.0", "@root/walk": "^1.1.0", + "@types/axios": "^0.14.0", "@types/jest": "^25.1.4", "@types/mkdirp": "^1.0.0", "@types/node": "^13.9.8", "@types/string-similarity": "^4.0.0", "7zip-bin": "^5.1.1", "7zip-min": "^1.4.0", + "axios": "^0.25.0", + "axios-extensions": "^3.1.3", "chokidar": "^3.5.3", "dotenv": "^10.0.0", + "filename-parser": "^1.0.0", "fs-extra": "^10.0.0", "imghash": "^0.0.9", "ioredis": "^4.28.1", diff --git a/services/import.service.ts b/services/import.service.ts index 4122262..8b03914 100644 --- a/services/import.service.ts +++ b/services/import.service.ts @@ -34,7 +34,7 @@ SOFTWARE. "use strict"; -import { isNil, map } from "lodash"; +import { isNil, isUndefined, map } from "lodash"; import { Context, Service, @@ -55,6 +55,7 @@ import { import { unrarArchive } from "../utils/uncompression.utils"; import { extractCoverFromFile2 } from "../utils/uncompression.utils"; import { scrapeIssuesFromDOM } from "../utils/scraping.utils"; +import axios from "axios"; const ObjectId = require("mongoose").Types.ObjectId; import fsExtra from "fs-extra"; const through2 = require("through2"); @@ -394,6 +395,7 @@ export default class ImportService extends Service { { $group: { _id: "$sourcedMetadata.comicvine.volume.id", + comicObjectId : { $first: "$_id" }, volumeURI: { $last: "$sourcedMetadata.comicvine.volume.api_detail_url", }, @@ -412,8 +414,9 @@ export default class ImportService extends Service { // 2. Map over the aggregation result and get volume metadata from CV // 2a. Make a call to comicvine-service volumesMetadata = map(volumes, async (volume) => { + console.log(volume); if (!isNil(volume.volumeURI)) { - return await ctx.call("comicvine.getVolumes", { + const volumeMetadata = await ctx.call("comicvine.getVolumes", { volumeURI: volume.volumeURI, data: { format: "json", @@ -423,12 +426,63 @@ export default class ImportService extends Service { offset: "0", }, }); + volumeMetadata["comicObjectId"] = volume.comicObjectId; + return volumeMetadata; } }); return Promise.all(volumesMetadata); }, }, + getIssuesForSeries: { + rest: "POST /getIssuesForSeries", + params: {}, + handler: async (ctx:Context<{ comicObjectID: string }>) => { + // 1. Query mongo to get issues for a given volume + const comicBookDetails: any = await this.broker.call( + "import.getComicBookById", + { id: ctx.params.comicObjectID } + ); + + // 2. Query CV and get metadata for them + comicBookDetails.sourcedMetadata.comicvine.volumeInformation.issues.map( + async (issue: any, idx: any) => { + const issueMetadata: any = + await axios.request({ + url: `${issue.api_detail_url}?api_key=${process.env.COMICVINE_API_KEY}`, + params: { + resources: "issues", + limit: "100", + format: "json", + }, + headers: { + "User-Agent": "ThreeTwo", + }, + }); + const metadata = + issueMetadata.data.results; + + // 2a. Query Mongo with Elastic to see if a match exists for a given issue's name, and issue number + if ( + !isUndefined(metadata.volume.name) && + !isUndefined(metadata.issue_number) + ) { + console.log("asdasd", metadata.volume.name); + await ctx.broker.call("libraryqueue.issuesForSeries", { queryObject: { + issueName: + metadata.volume + .name, + issueNumber: + metadata.issue_number, + }}); + + } + + return issueMetadata.data.results; + } + ); + } + }, flushDB: { rest: "POST /flushDB", params: {}, diff --git a/services/libraryqueue.service.ts b/services/libraryqueue.service.ts index eee46f2..62f6fd5 100644 --- a/services/libraryqueue.service.ts +++ b/services/libraryqueue.service.ts @@ -31,9 +31,9 @@ SOFTWARE. * Initial: 2022/01/28 Rishi Ghan */ - "use strict"; +import { isNil, isUndefined } from "lodash"; import { Context, Service, @@ -41,6 +41,7 @@ import { ServiceSchema, Errors, } from "moleculer"; + import BullMQMixin from "moleculer-bull"; import { SandboxedJob } from "moleculer-bull"; import { DbMixin } from "../mixins/db.mixin"; @@ -95,6 +96,30 @@ export default class LibraryQueueService extends Service { }); }, }, + "issue.findMatchesInLibrary": { + concurrency: 20, + async process(job: SandboxedJob) { + try { + console.log( + "reached the issuematchinlibrary queue" + ); + console.log(job.data); + const matchesInLibrary = await this.broker.call( + "search.searchComic", + { + queryObject: job.data.queryObject, + } + ); + console.log( + `Matches in Library: ${matchesInLibrary}` + ); + + return Promise.all(matchesInLibrary); + } catch (error) { + throw error; + } + }, + }, }, actions: { enqueue: { @@ -110,6 +135,23 @@ export default class LibraryQueueService extends Service { }); }, }, + issuesForSeries: { + rest: "POST /findIssuesForSeries", + params: {}, + handler: async ( + ctx: Context<{ queryObject: { + issueName: string, + issueNumber: string, + } }> + ) => { + return await this.createJob( + "issue.findMatchesInLibrary", + { + queryObject: ctx.params.queryObject, + } + ); + }, + }, }, methods: {}, async started(): Promise { @@ -142,6 +184,35 @@ export default class LibraryQueueService extends Service { ); } ); + + await this.getQueue("issue.findMatchesInLibrary").on( + "failed", + async (job, error) => { + console.error( + `An error occured in 'issue.findMatchesInLibrary' queue on job id '${job.id}': ${error.message}` + ); + } + ); + await this.getQueue("issue.findMatchesInLibrary").on( + "completed", + async (job, res) => { + client.emit("action", { + type: "LS_COVER_EXTRACTED", + result: res, + }); + console.info( + `Job with the id '${job.id}' completed.` + ); + } + ); + await this.getQueue("issue.findMatchesInLibrary").on( + "stalled", + async (job) => { + console.warn( + `The job with the id '${job} got stalled!` + ); + } + ); }); }, }); diff --git a/services/search.service.ts b/services/search.service.ts index d9688ff..c729749 100644 --- a/services/search.service.ts +++ b/services/search.service.ts @@ -18,6 +18,8 @@ const client = new Client({ import { DbMixin } from "../mixins/db.mixin"; import Comic from "../models/comic.model"; +import { refineQuery } from "filename-parser"; +import { filter } from "lodash"; console.log(client); @@ -38,15 +40,34 @@ export default class SettingsService extends Service { searchComic: { rest: "POST /searchComic", params: {}, - async handler(ctx: Context<{}>) { - Comic.esSearch({ - query_string: { - query: "batman", + timeout: 400000, + async handler( + ctx: Context<{ queryObject: { + issueName: string, + issueNumber: string, + } }> + ) { + console.log(ctx.params); + return Comic.esSearch({ + query: { + match: { + "rawFileDetails.name": { + query: ctx.params.queryObject.issueName, + operator: "or", + fuzziness: "AUTO", + }, + }, + }, }).then(function (results) { // results here - console.log(results.body.hits.hits); - results.body.hits.hits.forEach((item) => console.log(item._source)) + const foo = results.body.hits.hits.map((hit) => { + const parsedFilename = refineQuery(hit._source.rawFileDetails.name); + if(parsedFilename.searchParams.searchTerms.number === parseInt(ctx.params.queryObject.issueNumber, 10)) { + return hit; + } + }); + return filter(foo, null); }); }, },