🐂 Integrated elasticsearch search for issues in volumes with BullMQ

This commit is contained in:
2022-02-01 09:32:56 -08:00
parent 6bbfcf7603
commit 2c9bd55080
6 changed files with 517 additions and 14 deletions

View File

@@ -3,7 +3,7 @@ var mexp = require('mongoose-elasticsearch-xp').v7;
const paginate = require("mongoose-paginate-v2");
const { Client } = require("@elastic/elasticsearch");
console.log("YENADADADADA", Client)
const eSClient = new Client({
node: "http://tower.local:9200",
auth: {

361
package-lock.json generated
View File

@@ -10,14 +10,18 @@
"dependencies": {
"@elastic/elasticsearch": "^7.16.0",
"@root/walk": "^1.1.0",
"@types/axios": "^0.14.0",
"@types/jest": "^25.1.4",
"@types/mkdirp": "^1.0.0",
"@types/node": "^13.9.8",
"@types/string-similarity": "^4.0.0",
"7zip-bin": "^5.1.1",
"7zip-min": "^1.4.0",
"axios": "^0.25.0",
"axios-extensions": "^3.1.3",
"chokidar": "^3.5.3",
"dotenv": "^10.0.0",
"filename-parser": "^1.0.0",
"fs-extra": "^10.0.0",
"imghash": "^0.0.9",
"ioredis": "^4.28.1",
@@ -575,6 +579,18 @@
"@babel/core": "^7.0.0-0"
}
},
"node_modules/@babel/runtime-corejs3": {
"version": "7.16.8",
"resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.16.8.tgz",
"integrity": "sha512-3fKhuICS1lMz0plI5ktOE/yEtBRMVxplzRkdn6mJQ197XiY0JnrzYV0+Mxozq3JZ8SBV9Ecurmw1XsGbwOf+Sg==",
"dependencies": {
"core-js-pure": "^3.20.2",
"regenerator-runtime": "^0.13.4"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/template": {
"version": "7.16.7",
"resolved": "https://registry.npmjs.org/@babel/template/-/template-7.16.7.tgz",
@@ -1760,6 +1776,15 @@
"optional": true,
"peer": true
},
"node_modules/@types/axios": {
"version": "0.14.0",
"resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz",
"integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=",
"deprecated": "This is a stub types definition for axios (https://github.com/mzabriskie/axios). axios provides its own type definitions, so you don't need @types/axios installed!",
"dependencies": {
"axios": "*"
}
},
"node_modules/@types/babel__core": {
"version": "7.1.18",
"resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.18.tgz",
@@ -1882,6 +1907,11 @@
"integrity": "sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==",
"dev": true
},
"node_modules/@types/lru-cache": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz",
"integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw=="
},
"node_modules/@types/mkdirp": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz",
@@ -2483,6 +2513,41 @@
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz",
"integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA=="
},
"node_modules/axios": {
"version": "0.25.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz",
"integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
"dependencies": {
"follow-redirects": "^1.14.7"
}
},
"node_modules/axios-extensions": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/axios-extensions/-/axios-extensions-3.1.3.tgz",
"integrity": "sha512-/OB9OcJLNOIx9pdW4m4/hFRvNo12wlX5BaprIzqpMaLR02I88Mr98/wW4QN9rhx0/yg9rM7i6Af/RpV4MyxXjA==",
"dependencies": {
"@types/lru-cache": "^4.1.1",
"lru-cache": "^5.1.1",
"tslib": "^1.9.0",
"util": "^0.11.1"
},
"peerDependencies": {
"axios": "*"
}
},
"node_modules/axios-extensions/node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
"dependencies": {
"yallist": "^3.0.2"
}
},
"node_modules/axios-extensions/node_modules/yallist": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
},
"node_modules/babel-jest": {
"version": "27.4.6",
"resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.4.6.tgz",
@@ -3199,6 +3264,46 @@
"resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
"integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg=="
},
"node_modules/compromise": {
"version": "13.11.4",
"resolved": "https://registry.npmjs.org/compromise/-/compromise-13.11.4.tgz",
"integrity": "sha512-nBITcNdqIHSVDDluaG6guyFFCSNXN+Hu87fU8VlhkE5Z0PwTZN1nro2O7a8JcUH88nB5EOzrxd9zKfXLSNFqcg==",
"dependencies": {
"efrt-unpack": "2.2.0"
},
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/compromise-dates": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/compromise-dates/-/compromise-dates-2.2.1.tgz",
"integrity": "sha512-mtdK67DR4EvowLLh0YJ3aMxMtKXDViQBJTNNdHI5ck+DVhCCd7rYnNcwC2eSES+3tup1JB9+InOzmvyEuFyjLQ==",
"dependencies": {
"spacetime": "6.16.0",
"spacetime-holiday": "0.1.0"
},
"peerDependencies": {
"compromise": ">=12.0.0",
"compromise-numbers": ">=1.0.0"
}
},
"node_modules/compromise-numbers": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/compromise-numbers/-/compromise-numbers-1.4.0.tgz",
"integrity": "sha512-3ceRpwZIWduVSMYn54ET1ELdI7bvXQk42uDwxffxiJBxgKCwcCfVbiLuTG62cI+qTHchwLDh4vp9i3WARXROFQ==",
"peerDependencies": {
"compromise": ">=12.0.0"
}
},
"node_modules/compromise-sentences": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/compromise-sentences/-/compromise-sentences-0.3.0.tgz",
"integrity": "sha512-L6d5PXmmyPyWRMoJURYOM0yILhtV37zCOy9ZW8Xd6KgY+2qQBTwuokekaGL9MDpSH3/grRDL+AHKytYpGrTEmA==",
"peerDependencies": {
"compromise": ">=12.0.0"
}
},
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -3234,6 +3339,16 @@
"node": ">= 0.6"
}
},
"node_modules/core-js-pure": {
"version": "3.20.3",
"resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.20.3.tgz",
"integrity": "sha512-Q2H6tQ5MtPtcC7f3HxJ48i4Q7T9ybPKgvWyuH7JXIoNa2pm0KuBnycsET/qw1SLLZYfbsbrZQNMeIOClb+6WIA==",
"hasInstallScript": true,
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/core-js"
}
},
"node_modules/core-util-is": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
@@ -3564,6 +3679,11 @@
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
},
"node_modules/efrt-unpack": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/efrt-unpack/-/efrt-unpack-2.2.0.tgz",
"integrity": "sha512-9xUSSj7qcUxz+0r4X3+bwUNttEfGfK5AH+LVa1aTpqdAfrN5VhROYCfcF+up4hp5OL7IUKcZJJrzAGipQRDoiQ=="
},
"node_modules/elasticsearch": {
"version": "16.7.2",
"resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz",
@@ -4562,6 +4682,20 @@
"node": ">=6"
}
},
"node_modules/filename-parser": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/filename-parser/-/filename-parser-1.0.0.tgz",
"integrity": "sha512-3j7TgfElImSXhYcBzCP75+mC08IdRGmIBOm6XcJciI3xnQU6A4imzzUEJ2Ps717s5ycBkzQgUtjArGWGh/Gmkg==",
"dependencies": {
"compromise": "^13.11.4",
"compromise-dates": "^2.2.1",
"compromise-numbers": "^1.4.0",
"compromise-sentences": "^0.3.0",
"lodash": "^4.17.21",
"voca": "^1.4.0",
"xregexp": "^5.1.0"
}
},
"node_modules/fill-range": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -4612,6 +4746,25 @@
"integrity": "sha512-8/sOawo8tJ4QOBX8YlQBMxL8+RLZfxMQOif9o0KUKTNTjMYElWPE0r/m5VNFxTRd0NSw8qSy8dajrwX4RYI1Hw==",
"dev": true
},
"node_modules/follow-redirects": {
"version": "1.14.7",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz",
"integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==",
"funding": [
{
"type": "individual",
"url": "https://github.com/sponsors/RubenVerborgh"
}
],
"engines": {
"node": ">=4.0"
},
"peerDependenciesMeta": {
"debug": {
"optional": true
}
}
},
"node_modules/for-each": {
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
@@ -10027,6 +10180,11 @@
"node": ">=4"
}
},
"node_modules/regenerator-runtime": {
"version": "0.13.9",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz",
"integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA=="
},
"node_modules/regexp-clone": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/regexp-clone/-/regexp-clone-1.0.0.tgz",
@@ -10642,6 +10800,19 @@
"source-map": "^0.6.0"
}
},
"node_modules/spacetime": {
"version": "6.16.0",
"resolved": "https://registry.npmjs.org/spacetime/-/spacetime-6.16.0.tgz",
"integrity": "sha512-mkuniNOp6ssfPyJidj81tb54zKxK4vEKPTmcUsC/NEGIF8S07ppoSotdg6numT1/26rthQYmdxMY/M5a9WeJVQ=="
},
"node_modules/spacetime-holiday": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/spacetime-holiday/-/spacetime-holiday-0.1.0.tgz",
"integrity": "sha512-rYIpSDbHnznZRstUrmYYFAaruW8e96t+1JfS0b6qMiAAQ2DrkLKc8oMotAAkB9qMTUwXXf5bIkdTHfP434uitQ==",
"peerDependencies": {
"spacetime": "^6.3.0"
}
},
"node_modules/sparse-bitfield": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
@@ -11320,8 +11491,7 @@
"node_modules/tslib": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
"dev": true
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
},
"node_modules/tsutils": {
"version": "3.21.0",
@@ -11478,6 +11648,14 @@
"punycode": "^2.1.0"
}
},
"node_modules/util": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz",
"integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==",
"dependencies": {
"inherits": "2.0.3"
}
},
"node_modules/util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -11498,6 +11676,11 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/util/node_modules/inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
},
"node_modules/uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
@@ -11561,6 +11744,11 @@
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"node_modules/voca": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/voca/-/voca-1.4.0.tgz",
"integrity": "sha512-8Xz4H3vhYRGbFupLtl6dHwMx0ojUcjt0HYkqZ9oBCfipd/5mD7Md58m2/dq7uPuZU/0T3Gb1m66KS9jn+I+14Q=="
},
"node_modules/w3c-hr-time": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz",
@@ -11832,6 +12020,14 @@
"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="
},
"node_modules/xregexp": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/xregexp/-/xregexp-5.1.0.tgz",
"integrity": "sha512-PynwUWtXnSZr8tpQlDPMZfPTyv78EYuA4oI959ukxcQ0a9O/lvndLVKy5wpImzzA26eMxpZmnAXJYiQA13AtWA==",
"dependencies": {
"@babel/runtime-corejs3": "^7.14.9"
}
},
"node_modules/y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
@@ -12318,6 +12514,15 @@
"@babel/helper-plugin-utils": "^7.16.7"
}
},
"@babel/runtime-corejs3": {
"version": "7.16.8",
"resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.16.8.tgz",
"integrity": "sha512-3fKhuICS1lMz0plI5ktOE/yEtBRMVxplzRkdn6mJQ197XiY0JnrzYV0+Mxozq3JZ8SBV9Ecurmw1XsGbwOf+Sg==",
"requires": {
"core-js-pure": "^3.20.2",
"regenerator-runtime": "^0.13.4"
}
},
"@babel/template": {
"version": "7.16.7",
"resolved": "https://registry.npmjs.org/@babel/template/-/template-7.16.7.tgz",
@@ -13241,6 +13446,14 @@
"optional": true,
"peer": true
},
"@types/axios": {
"version": "0.14.0",
"resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz",
"integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=",
"requires": {
"axios": "*"
}
},
"@types/babel__core": {
"version": "7.1.18",
"resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.18.tgz",
@@ -13363,6 +13576,11 @@
"integrity": "sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==",
"dev": true
},
"@types/lru-cache": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz",
"integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw=="
},
"@types/mkdirp": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz",
@@ -13797,6 +14015,40 @@
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz",
"integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA=="
},
"axios": {
"version": "0.25.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.25.0.tgz",
"integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
"requires": {
"follow-redirects": "^1.14.7"
}
},
"axios-extensions": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/axios-extensions/-/axios-extensions-3.1.3.tgz",
"integrity": "sha512-/OB9OcJLNOIx9pdW4m4/hFRvNo12wlX5BaprIzqpMaLR02I88Mr98/wW4QN9rhx0/yg9rM7i6Af/RpV4MyxXjA==",
"requires": {
"@types/lru-cache": "^4.1.1",
"lru-cache": "^5.1.1",
"tslib": "^1.9.0",
"util": "^0.11.1"
},
"dependencies": {
"lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
"requires": {
"yallist": "^3.0.2"
}
},
"yallist": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
}
}
},
"babel-jest": {
"version": "27.4.6",
"resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.4.6.tgz",
@@ -14339,6 +14591,35 @@
"resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
"integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg=="
},
"compromise": {
"version": "13.11.4",
"resolved": "https://registry.npmjs.org/compromise/-/compromise-13.11.4.tgz",
"integrity": "sha512-nBITcNdqIHSVDDluaG6guyFFCSNXN+Hu87fU8VlhkE5Z0PwTZN1nro2O7a8JcUH88nB5EOzrxd9zKfXLSNFqcg==",
"requires": {
"efrt-unpack": "2.2.0"
}
},
"compromise-dates": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/compromise-dates/-/compromise-dates-2.2.1.tgz",
"integrity": "sha512-mtdK67DR4EvowLLh0YJ3aMxMtKXDViQBJTNNdHI5ck+DVhCCd7rYnNcwC2eSES+3tup1JB9+InOzmvyEuFyjLQ==",
"requires": {
"spacetime": "6.16.0",
"spacetime-holiday": "0.1.0"
}
},
"compromise-numbers": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/compromise-numbers/-/compromise-numbers-1.4.0.tgz",
"integrity": "sha512-3ceRpwZIWduVSMYn54ET1ELdI7bvXQk42uDwxffxiJBxgKCwcCfVbiLuTG62cI+qTHchwLDh4vp9i3WARXROFQ==",
"requires": {}
},
"compromise-sentences": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/compromise-sentences/-/compromise-sentences-0.3.0.tgz",
"integrity": "sha512-L6d5PXmmyPyWRMoJURYOM0yILhtV37zCOy9ZW8Xd6KgY+2qQBTwuokekaGL9MDpSH3/grRDL+AHKytYpGrTEmA==",
"requires": {}
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -14368,6 +14649,11 @@
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz",
"integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA=="
},
"core-js-pure": {
"version": "3.20.3",
"resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.20.3.tgz",
"integrity": "sha512-Q2H6tQ5MtPtcC7f3HxJ48i4Q7T9ybPKgvWyuH7JXIoNa2pm0KuBnycsET/qw1SLLZYfbsbrZQNMeIOClb+6WIA=="
},
"core-util-is": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
@@ -14628,6 +14914,11 @@
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
},
"efrt-unpack": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/efrt-unpack/-/efrt-unpack-2.2.0.tgz",
"integrity": "sha512-9xUSSj7qcUxz+0r4X3+bwUNttEfGfK5AH+LVa1aTpqdAfrN5VhROYCfcF+up4hp5OL7IUKcZJJrzAGipQRDoiQ=="
},
"elasticsearch": {
"version": "16.7.2",
"resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz",
@@ -15438,6 +15729,20 @@
"resolved": "https://registry.npmjs.org/file-type/-/file-type-10.11.0.tgz",
"integrity": "sha512-uzk64HRpUZyTGZtVuvrjP0FYxzQrBf4rojot6J65YMEbwBLB0CWm0CLojVpwpmFmxcE/lkvYICgfcGozbBq6rw=="
},
"filename-parser": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/filename-parser/-/filename-parser-1.0.0.tgz",
"integrity": "sha512-3j7TgfElImSXhYcBzCP75+mC08IdRGmIBOm6XcJciI3xnQU6A4imzzUEJ2Ps717s5ycBkzQgUtjArGWGh/Gmkg==",
"requires": {
"compromise": "^13.11.4",
"compromise-dates": "^2.2.1",
"compromise-numbers": "^1.4.0",
"compromise-sentences": "^0.3.0",
"lodash": "^4.17.21",
"voca": "^1.4.0",
"xregexp": "^5.1.0"
}
},
"fill-range": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -15476,6 +15781,11 @@
"integrity": "sha512-8/sOawo8tJ4QOBX8YlQBMxL8+RLZfxMQOif9o0KUKTNTjMYElWPE0r/m5VNFxTRd0NSw8qSy8dajrwX4RYI1Hw==",
"dev": true
},
"follow-redirects": {
"version": "1.14.7",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz",
"integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ=="
},
"for-each": {
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
@@ -19416,6 +19726,11 @@
"redis-errors": "^1.0.0"
}
},
"regenerator-runtime": {
"version": "0.13.9",
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz",
"integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA=="
},
"regexp-clone": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/regexp-clone/-/regexp-clone-1.0.0.tgz",
@@ -19875,6 +20190,17 @@
"source-map": "^0.6.0"
}
},
"spacetime": {
"version": "6.16.0",
"resolved": "https://registry.npmjs.org/spacetime/-/spacetime-6.16.0.tgz",
"integrity": "sha512-mkuniNOp6ssfPyJidj81tb54zKxK4vEKPTmcUsC/NEGIF8S07ppoSotdg6numT1/26rthQYmdxMY/M5a9WeJVQ=="
},
"spacetime-holiday": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/spacetime-holiday/-/spacetime-holiday-0.1.0.tgz",
"integrity": "sha512-rYIpSDbHnznZRstUrmYYFAaruW8e96t+1JfS0b6qMiAAQ2DrkLKc8oMotAAkB9qMTUwXXf5bIkdTHfP434uitQ==",
"requires": {}
},
"sparse-bitfield": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
@@ -20409,8 +20735,7 @@
"tslib": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
"dev": true
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
},
"tsutils": {
"version": "3.21.0",
@@ -20524,6 +20849,21 @@
"punycode": "^2.1.0"
}
},
"util": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz",
"integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==",
"requires": {
"inherits": "2.0.3"
},
"dependencies": {
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
}
}
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -20593,6 +20933,11 @@
}
}
},
"voca": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/voca/-/voca-1.4.0.tgz",
"integrity": "sha512-8Xz4H3vhYRGbFupLtl6dHwMx0ojUcjt0HYkqZ9oBCfipd/5mD7Md58m2/dq7uPuZU/0T3Gb1m66KS9jn+I+14Q=="
},
"w3c-hr-time": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz",
@@ -20807,6 +21152,14 @@
"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="
},
"xregexp": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/xregexp/-/xregexp-5.1.0.tgz",
"integrity": "sha512-PynwUWtXnSZr8tpQlDPMZfPTyv78EYuA4oI959ukxcQ0a9O/lvndLVKy5wpImzzA26eMxpZmnAXJYiQA13AtWA==",
"requires": {
"@babel/runtime-corejs3": "^7.14.9"
}
},
"y18n": {
"version": "5.0.8",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

View File

@@ -37,14 +37,18 @@
"dependencies": {
"@elastic/elasticsearch": "^7.16.0",
"@root/walk": "^1.1.0",
"@types/axios": "^0.14.0",
"@types/jest": "^25.1.4",
"@types/mkdirp": "^1.0.0",
"@types/node": "^13.9.8",
"@types/string-similarity": "^4.0.0",
"7zip-bin": "^5.1.1",
"7zip-min": "^1.4.0",
"axios": "^0.25.0",
"axios-extensions": "^3.1.3",
"chokidar": "^3.5.3",
"dotenv": "^10.0.0",
"filename-parser": "^1.0.0",
"fs-extra": "^10.0.0",
"imghash": "^0.0.9",
"ioredis": "^4.28.1",

View File

@@ -34,7 +34,7 @@ SOFTWARE.
"use strict";
import { isNil, map } from "lodash";
import { isNil, isUndefined, map } from "lodash";
import {
Context,
Service,
@@ -55,6 +55,7 @@ import {
import { unrarArchive } from "../utils/uncompression.utils";
import { extractCoverFromFile2 } from "../utils/uncompression.utils";
import { scrapeIssuesFromDOM } from "../utils/scraping.utils";
import axios from "axios";
const ObjectId = require("mongoose").Types.ObjectId;
import fsExtra from "fs-extra";
const through2 = require("through2");
@@ -394,6 +395,7 @@ export default class ImportService extends Service {
{
$group: {
_id: "$sourcedMetadata.comicvine.volume.id",
comicObjectId : { $first: "$_id" },
volumeURI: {
$last: "$sourcedMetadata.comicvine.volume.api_detail_url",
},
@@ -412,8 +414,9 @@ export default class ImportService extends Service {
// 2. Map over the aggregation result and get volume metadata from CV
// 2a. Make a call to comicvine-service
volumesMetadata = map(volumes, async (volume) => {
console.log(volume);
if (!isNil(volume.volumeURI)) {
return await ctx.call("comicvine.getVolumes", {
const volumeMetadata = await ctx.call("comicvine.getVolumes", {
volumeURI: volume.volumeURI,
data: {
format: "json",
@@ -423,12 +426,63 @@ export default class ImportService extends Service {
offset: "0",
},
});
volumeMetadata["comicObjectId"] = volume.comicObjectId;
return volumeMetadata;
}
});
return Promise.all(volumesMetadata);
},
},
getIssuesForSeries: {
rest: "POST /getIssuesForSeries",
params: {},
handler: async (ctx:Context<{ comicObjectID: string }>) => {
// 1. Query mongo to get issues for a given volume
const comicBookDetails: any = await this.broker.call(
"import.getComicBookById",
{ id: ctx.params.comicObjectID }
);
// 2. Query CV and get metadata for them
comicBookDetails.sourcedMetadata.comicvine.volumeInformation.issues.map(
async (issue: any, idx: any) => {
const issueMetadata: any =
await axios.request({
url: `${issue.api_detail_url}?api_key=${process.env.COMICVINE_API_KEY}`,
params: {
resources: "issues",
limit: "100",
format: "json",
},
headers: {
"User-Agent": "ThreeTwo",
},
});
const metadata =
issueMetadata.data.results;
// 2a. Query Mongo with Elastic to see if a match exists for a given issue's name, and issue number
if (
!isUndefined(metadata.volume.name) &&
!isUndefined(metadata.issue_number)
) {
console.log("asdasd", metadata.volume.name);
await ctx.broker.call("libraryqueue.issuesForSeries", { queryObject: {
issueName:
metadata.volume
.name,
issueNumber:
metadata.issue_number,
}});
}
return issueMetadata.data.results;
}
);
}
},
flushDB: {
rest: "POST /flushDB",
params: {},

View File

@@ -31,9 +31,9 @@ SOFTWARE.
* Initial: 2022/01/28 Rishi Ghan
*/
"use strict";
import { isNil, isUndefined } from "lodash";
import {
Context,
Service,
@@ -41,6 +41,7 @@ import {
ServiceSchema,
Errors,
} from "moleculer";
import BullMQMixin from "moleculer-bull";
import { SandboxedJob } from "moleculer-bull";
import { DbMixin } from "../mixins/db.mixin";
@@ -95,6 +96,30 @@ export default class LibraryQueueService extends Service {
});
},
},
"issue.findMatchesInLibrary": {
concurrency: 20,
async process(job: SandboxedJob) {
try {
console.log(
"reached the issuematchinlibrary queue"
);
console.log(job.data);
const matchesInLibrary = await this.broker.call(
"search.searchComic",
{
queryObject: job.data.queryObject,
}
);
console.log(
`Matches in Library: ${matchesInLibrary}`
);
return Promise.all(matchesInLibrary);
} catch (error) {
throw error;
}
},
},
},
actions: {
enqueue: {
@@ -110,6 +135,23 @@ export default class LibraryQueueService extends Service {
});
},
},
issuesForSeries: {
rest: "POST /findIssuesForSeries",
params: {},
handler: async (
ctx: Context<{ queryObject: {
issueName: string,
issueNumber: string,
} }>
) => {
return await this.createJob(
"issue.findMatchesInLibrary",
{
queryObject: ctx.params.queryObject,
}
);
},
},
},
methods: {},
async started(): Promise<any> {
@@ -142,6 +184,35 @@ export default class LibraryQueueService extends Service {
);
}
);
await this.getQueue("issue.findMatchesInLibrary").on(
"failed",
async (job, error) => {
console.error(
`An error occured in 'issue.findMatchesInLibrary' queue on job id '${job.id}': ${error.message}`
);
}
);
await this.getQueue("issue.findMatchesInLibrary").on(
"completed",
async (job, res) => {
client.emit("action", {
type: "LS_COVER_EXTRACTED",
result: res,
});
console.info(
`Job with the id '${job.id}' completed.`
);
}
);
await this.getQueue("issue.findMatchesInLibrary").on(
"stalled",
async (job) => {
console.warn(
`The job with the id '${job} got stalled!`
);
}
);
});
},
});

View File

@@ -18,6 +18,8 @@ const client = new Client({
import { DbMixin } from "../mixins/db.mixin";
import Comic from "../models/comic.model";
import { refineQuery } from "filename-parser";
import { filter } from "lodash";
console.log(client);
@@ -38,15 +40,34 @@ export default class SettingsService extends Service {
searchComic: {
rest: "POST /searchComic",
params: {},
async handler(ctx: Context<{}>) {
Comic.esSearch({
query_string: {
query: "batman",
timeout: 400000,
async handler(
ctx: Context<{ queryObject: {
issueName: string,
issueNumber: string,
} }>
) {
console.log(ctx.params);
return Comic.esSearch({
query: {
match: {
"rawFileDetails.name": {
query: ctx.params.queryObject.issueName,
operator: "or",
fuzziness: "AUTO",
},
},
},
}).then(function (results) {
// results here
console.log(results.body.hits.hits);
results.body.hits.hits.forEach((item) => console.log(item._source))
const foo = results.body.hits.hits.map((hit) => {
const parsedFilename = refineQuery(hit._source.rawFileDetails.name);
if(parsedFilename.searchParams.searchTerms.number === parseInt(ctx.params.queryObject.issueNumber, 10)) {
return hit;
}
});
return filter(foo, null);
});
},
},