👇🏼 Inferring issue metadata upon import
This commit is contained in:
@@ -4,8 +4,8 @@ const paginate = require("mongoose-paginate-v2");
|
||||
|
||||
const { Client } = require("@elastic/elasticsearch");
|
||||
|
||||
const eSClient = new Client({
|
||||
node: "http://tower.local:9200",
|
||||
export const eSClient = new Client({
|
||||
node: "http://localhost:9200",
|
||||
auth: {
|
||||
username: "elastic",
|
||||
password: "password",
|
||||
@@ -60,6 +60,14 @@ const ComicSchema = mongoose.Schema({
|
||||
coverWriteResult: String,
|
||||
}
|
||||
},
|
||||
inferredMetadata: {
|
||||
issue: {
|
||||
name: String,
|
||||
number: { type: Number, es_indexed: true },
|
||||
year: String,
|
||||
subtitle: String,
|
||||
}
|
||||
},
|
||||
acquisition: {
|
||||
wanted: Boolean,
|
||||
release: {},
|
||||
|
||||
299
package-lock.json
generated
299
package-lock.json
generated
@@ -17,11 +17,13 @@
|
||||
"@types/string-similarity": "^4.0.0",
|
||||
"7zip-bin": "^5.1.1",
|
||||
"7zip-min": "^1.4.0",
|
||||
"array-to-ndjson": "^1.0.1",
|
||||
"axios": "^0.25.0",
|
||||
"axios-extensions": "^3.1.3",
|
||||
"axios-retry": "^3.2.4",
|
||||
"chokidar": "^3.5.3",
|
||||
"delay": "^5.0.0",
|
||||
"dotenv": "^10.0.0",
|
||||
"filename-parser": "^1.0.0",
|
||||
"filename-parser": "^1.0.1",
|
||||
"fs-extra": "^10.0.0",
|
||||
"imghash": "^0.0.9",
|
||||
"ioredis": "^4.28.1",
|
||||
@@ -46,7 +48,7 @@
|
||||
"node-unrar-js": "^1.0.2",
|
||||
"sharp": "^0.28.3",
|
||||
"socket.io": "^4.4.0",
|
||||
"threetwo-ui-typings": "^1.0.12",
|
||||
"threetwo-ui-typings": "^1.0.13",
|
||||
"through2": "^4.0.2",
|
||||
"unrar": "^0.2.0",
|
||||
"xml2js": "^0.4.23"
|
||||
@@ -579,6 +581,17 @@
|
||||
"@babel/core": "^7.0.0-0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/runtime": {
|
||||
"version": "7.17.0",
|
||||
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.0.tgz",
|
||||
"integrity": "sha512-etcO/ohMNaNA2UBdaXBBSX/3aEzFMRrVfaPv8Ptc0k+cWpWW0QFiGZ2XnVqQZI1Cf734LbPGmqBKWESfW4x/dQ==",
|
||||
"dependencies": {
|
||||
"regenerator-runtime": "^0.13.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/runtime-corejs3": {
|
||||
"version": "7.16.8",
|
||||
"resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.16.8.tgz",
|
||||
@@ -1907,11 +1920,6 @@
|
||||
"integrity": "sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/@types/lru-cache": {
|
||||
"version": "4.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz",
|
||||
"integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw=="
|
||||
},
|
||||
"node_modules/@types/mkdirp": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz",
|
||||
@@ -2439,6 +2447,18 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/array-to-ndjson": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/array-to-ndjson/-/array-to-ndjson-1.0.1.tgz",
|
||||
"integrity": "sha1-LkSejKTVdH3jfjMN87h1hD2KlTU=",
|
||||
"dependencies": {
|
||||
"is-object": "^1.0.1",
|
||||
"stream-array": "^1.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/array-union": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
|
||||
@@ -2521,33 +2541,15 @@
|
||||
"follow-redirects": "^1.14.7"
|
||||
}
|
||||
},
|
||||
"node_modules/axios-extensions": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/axios-extensions/-/axios-extensions-3.1.3.tgz",
|
||||
"integrity": "sha512-/OB9OcJLNOIx9pdW4m4/hFRvNo12wlX5BaprIzqpMaLR02I88Mr98/wW4QN9rhx0/yg9rM7i6Af/RpV4MyxXjA==",
|
||||
"node_modules/axios-retry": {
|
||||
"version": "3.2.4",
|
||||
"resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.2.4.tgz",
|
||||
"integrity": "sha512-Co3UXiv4npi6lM963mfnuH90/YFLKWWDmoBYfxkHT5xtkSSWNqK9zdG3fw5/CP/dsoKB5aMMJCsgab+tp1OxLQ==",
|
||||
"dependencies": {
|
||||
"@types/lru-cache": "^4.1.1",
|
||||
"lru-cache": "^5.1.1",
|
||||
"tslib": "^1.9.0",
|
||||
"util": "^0.11.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"axios": "*"
|
||||
"@babel/runtime": "^7.15.4",
|
||||
"is-retry-allowed": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/axios-extensions/node_modules/lru-cache": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
|
||||
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
|
||||
"dependencies": {
|
||||
"yallist": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/axios-extensions/node_modules/yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
|
||||
},
|
||||
"node_modules/babel-jest": {
|
||||
"version": "27.4.6",
|
||||
"resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.4.6.tgz",
|
||||
@@ -2926,6 +2928,11 @@
|
||||
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/buffer-shims": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz",
|
||||
"integrity": "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E="
|
||||
},
|
||||
"node_modules/bull": {
|
||||
"version": "3.29.3",
|
||||
"resolved": "https://registry.npmjs.org/bull/-/bull-3.29.3.tgz",
|
||||
@@ -3554,6 +3561,17 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/delay": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
|
||||
"integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
@@ -4683,9 +4701,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/filename-parser": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/filename-parser/-/filename-parser-1.0.0.tgz",
|
||||
"integrity": "sha512-3j7TgfElImSXhYcBzCP75+mC08IdRGmIBOm6XcJciI3xnQU6A4imzzUEJ2Ps717s5ycBkzQgUtjArGWGh/Gmkg==",
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/filename-parser/-/filename-parser-1.0.1.tgz",
|
||||
"integrity": "sha512-MMzuklXc1r4N6uQXg8CQYxoiTX7w6QPeJE73L5lCTSoNR7CftCXHIA6tyINomkvWIUanrlqTB629DyTIqCucEA==",
|
||||
"dependencies": {
|
||||
"compromise": "^13.11.4",
|
||||
"compromise-dates": "^2.2.1",
|
||||
@@ -5569,6 +5587,14 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/is-object": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz",
|
||||
"integrity": "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA==",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/is-potential-custom-element-name": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
|
||||
@@ -5590,6 +5616,17 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/is-retry-allowed": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz",
|
||||
"integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/is-shared-array-buffer": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz",
|
||||
@@ -10894,6 +10931,41 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/stream-array": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/stream-array/-/stream-array-1.1.2.tgz",
|
||||
"integrity": "sha1-nl9zRfITfDDuO0mLkRToC1K7frU=",
|
||||
"dependencies": {
|
||||
"readable-stream": "~2.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/stream-array/node_modules/process-nextick-args": {
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
|
||||
"integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M="
|
||||
},
|
||||
"node_modules/stream-array/node_modules/readable-stream": {
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.1.5.tgz",
|
||||
"integrity": "sha1-ZvqLcg4UOLNkaB8q0aY8YYRIydA=",
|
||||
"dependencies": {
|
||||
"buffer-shims": "^1.0.0",
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.1",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~1.0.6",
|
||||
"string_decoder": "~0.10.x",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/stream-array/node_modules/string_decoder": {
|
||||
"version": "0.10.31",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
|
||||
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
|
||||
},
|
||||
"node_modules/string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
@@ -11247,9 +11319,9 @@
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/threetwo-ui-typings": {
|
||||
"version": "1.0.12",
|
||||
"resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.12.tgz",
|
||||
"integrity": "sha512-lKADNpD2Oa3Wmf6tdZhaPSNfIFa0KtstsZLE0yalTjqKmtQZM7Ct2OnlCkH7aonDuVn+jgcRMwkcP9krCqa2fw==",
|
||||
"version": "1.0.13",
|
||||
"resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.13.tgz",
|
||||
"integrity": "sha512-AQiY8/hbp+TobBoehNTEoNco97AoiKYQjAANSFDR3pSD5jFn5qjLlKntvqdNF9Fg5tcS0ReYe0AjsvKshKpixQ==",
|
||||
"dependencies": {
|
||||
"typescript": "^4.3.2"
|
||||
}
|
||||
@@ -11491,7 +11563,8 @@
|
||||
"node_modules/tslib": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
|
||||
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
|
||||
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/tsutils": {
|
||||
"version": "3.21.0",
|
||||
@@ -11648,14 +11721,6 @@
|
||||
"punycode": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/util": {
|
||||
"version": "0.11.1",
|
||||
"resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz",
|
||||
"integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==",
|
||||
"dependencies": {
|
||||
"inherits": "2.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
@@ -11676,11 +11741,6 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/util/node_modules/inherits": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
|
||||
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
|
||||
},
|
||||
"node_modules/uuid": {
|
||||
"version": "8.3.2",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
|
||||
@@ -12514,6 +12574,14 @@
|
||||
"@babel/helper-plugin-utils": "^7.16.7"
|
||||
}
|
||||
},
|
||||
"@babel/runtime": {
|
||||
"version": "7.17.0",
|
||||
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.17.0.tgz",
|
||||
"integrity": "sha512-etcO/ohMNaNA2UBdaXBBSX/3aEzFMRrVfaPv8Ptc0k+cWpWW0QFiGZ2XnVqQZI1Cf734LbPGmqBKWESfW4x/dQ==",
|
||||
"requires": {
|
||||
"regenerator-runtime": "^0.13.4"
|
||||
}
|
||||
},
|
||||
"@babel/runtime-corejs3": {
|
||||
"version": "7.16.8",
|
||||
"resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.16.8.tgz",
|
||||
@@ -13576,11 +13644,6 @@
|
||||
"integrity": "sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/lru-cache": {
|
||||
"version": "4.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz",
|
||||
"integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw=="
|
||||
},
|
||||
"@types/mkdirp": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz",
|
||||
@@ -13959,6 +14022,15 @@
|
||||
"is-string": "^1.0.7"
|
||||
}
|
||||
},
|
||||
"array-to-ndjson": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/array-to-ndjson/-/array-to-ndjson-1.0.1.tgz",
|
||||
"integrity": "sha1-LkSejKTVdH3jfjMN87h1hD2KlTU=",
|
||||
"requires": {
|
||||
"is-object": "^1.0.1",
|
||||
"stream-array": "^1.1.2"
|
||||
}
|
||||
},
|
||||
"array-union": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
|
||||
@@ -14023,30 +14095,13 @@
|
||||
"follow-redirects": "^1.14.7"
|
||||
}
|
||||
},
|
||||
"axios-extensions": {
|
||||
"version": "3.1.3",
|
||||
"resolved": "https://registry.npmjs.org/axios-extensions/-/axios-extensions-3.1.3.tgz",
|
||||
"integrity": "sha512-/OB9OcJLNOIx9pdW4m4/hFRvNo12wlX5BaprIzqpMaLR02I88Mr98/wW4QN9rhx0/yg9rM7i6Af/RpV4MyxXjA==",
|
||||
"axios-retry": {
|
||||
"version": "3.2.4",
|
||||
"resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.2.4.tgz",
|
||||
"integrity": "sha512-Co3UXiv4npi6lM963mfnuH90/YFLKWWDmoBYfxkHT5xtkSSWNqK9zdG3fw5/CP/dsoKB5aMMJCsgab+tp1OxLQ==",
|
||||
"requires": {
|
||||
"@types/lru-cache": "^4.1.1",
|
||||
"lru-cache": "^5.1.1",
|
||||
"tslib": "^1.9.0",
|
||||
"util": "^0.11.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"lru-cache": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
|
||||
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
|
||||
"requires": {
|
||||
"yallist": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
|
||||
}
|
||||
"@babel/runtime": "^7.15.4",
|
||||
"is-retry-allowed": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"babel-jest": {
|
||||
@@ -14327,6 +14382,11 @@
|
||||
"integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
|
||||
"dev": true
|
||||
},
|
||||
"buffer-shims": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz",
|
||||
"integrity": "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E="
|
||||
},
|
||||
"bull": {
|
||||
"version": "3.29.3",
|
||||
"resolved": "https://registry.npmjs.org/bull/-/bull-3.29.3.tgz",
|
||||
@@ -14822,6 +14882,11 @@
|
||||
"object-keys": "^1.0.12"
|
||||
}
|
||||
},
|
||||
"delay": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
|
||||
"integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw=="
|
||||
},
|
||||
"delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
@@ -15730,9 +15795,9 @@
|
||||
"integrity": "sha512-uzk64HRpUZyTGZtVuvrjP0FYxzQrBf4rojot6J65YMEbwBLB0CWm0CLojVpwpmFmxcE/lkvYICgfcGozbBq6rw=="
|
||||
},
|
||||
"filename-parser": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/filename-parser/-/filename-parser-1.0.0.tgz",
|
||||
"integrity": "sha512-3j7TgfElImSXhYcBzCP75+mC08IdRGmIBOm6XcJciI3xnQU6A4imzzUEJ2Ps717s5ycBkzQgUtjArGWGh/Gmkg==",
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/filename-parser/-/filename-parser-1.0.1.tgz",
|
||||
"integrity": "sha512-MMzuklXc1r4N6uQXg8CQYxoiTX7w6QPeJE73L5lCTSoNR7CftCXHIA6tyINomkvWIUanrlqTB629DyTIqCucEA==",
|
||||
"requires": {
|
||||
"compromise": "^13.11.4",
|
||||
"compromise-dates": "^2.2.1",
|
||||
@@ -16350,6 +16415,11 @@
|
||||
"has-tostringtag": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"is-object": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.2.tgz",
|
||||
"integrity": "sha512-2rRIahhZr2UWb45fIOuvZGpFtz0TyOZLf32KxBbSoUCeZR495zCKlWUKKUByk3geS2eAs7ZAABt0Y/Rx0GiQGA=="
|
||||
},
|
||||
"is-potential-custom-element-name": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
|
||||
@@ -16365,6 +16435,11 @@
|
||||
"has-tostringtag": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"is-retry-allowed": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz",
|
||||
"integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg=="
|
||||
},
|
||||
"is-shared-array-buffer": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz",
|
||||
@@ -20264,6 +20339,40 @@
|
||||
"resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
|
||||
"integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
|
||||
},
|
||||
"stream-array": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/stream-array/-/stream-array-1.1.2.tgz",
|
||||
"integrity": "sha1-nl9zRfITfDDuO0mLkRToC1K7frU=",
|
||||
"requires": {
|
||||
"readable-stream": "~2.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"process-nextick-args": {
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
|
||||
"integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M="
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "2.1.5",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.1.5.tgz",
|
||||
"integrity": "sha1-ZvqLcg4UOLNkaB8q0aY8YYRIydA=",
|
||||
"requires": {
|
||||
"buffer-shims": "^1.0.0",
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.1",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~1.0.6",
|
||||
"string_decoder": "~0.10.x",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"string_decoder": {
|
||||
"version": "0.10.31",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
|
||||
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
|
||||
}
|
||||
}
|
||||
},
|
||||
"string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
@@ -20540,9 +20649,9 @@
|
||||
"dev": true
|
||||
},
|
||||
"threetwo-ui-typings": {
|
||||
"version": "1.0.12",
|
||||
"resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.12.tgz",
|
||||
"integrity": "sha512-lKADNpD2Oa3Wmf6tdZhaPSNfIFa0KtstsZLE0yalTjqKmtQZM7Ct2OnlCkH7aonDuVn+jgcRMwkcP9krCqa2fw==",
|
||||
"version": "1.0.13",
|
||||
"resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.13.tgz",
|
||||
"integrity": "sha512-AQiY8/hbp+TobBoehNTEoNco97AoiKYQjAANSFDR3pSD5jFn5qjLlKntvqdNF9Fg5tcS0ReYe0AjsvKshKpixQ==",
|
||||
"requires": {
|
||||
"typescript": "^4.3.2"
|
||||
},
|
||||
@@ -20735,7 +20844,8 @@
|
||||
"tslib": {
|
||||
"version": "1.14.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
|
||||
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
|
||||
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
|
||||
"dev": true
|
||||
},
|
||||
"tsutils": {
|
||||
"version": "3.21.0",
|
||||
@@ -20849,21 +20959,6 @@
|
||||
"punycode": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"util": {
|
||||
"version": "0.11.1",
|
||||
"resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz",
|
||||
"integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==",
|
||||
"requires": {
|
||||
"inherits": "2.0.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"inherits": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
|
||||
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
|
||||
}
|
||||
}
|
||||
},
|
||||
"util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
|
||||
@@ -44,11 +44,13 @@
|
||||
"@types/string-similarity": "^4.0.0",
|
||||
"7zip-bin": "^5.1.1",
|
||||
"7zip-min": "^1.4.0",
|
||||
"array-to-ndjson": "^1.0.1",
|
||||
"axios": "^0.25.0",
|
||||
"axios-extensions": "^3.1.3",
|
||||
"axios-retry": "^3.2.4",
|
||||
"chokidar": "^3.5.3",
|
||||
"delay": "^5.0.0",
|
||||
"dotenv": "^10.0.0",
|
||||
"filename-parser": "^1.0.0",
|
||||
"filename-parser": "^1.0.1",
|
||||
"fs-extra": "^10.0.0",
|
||||
"imghash": "^0.0.9",
|
||||
"ioredis": "^4.28.1",
|
||||
@@ -73,7 +75,7 @@
|
||||
"node-unrar-js": "^1.0.2",
|
||||
"sharp": "^0.28.3",
|
||||
"socket.io": "^4.4.0",
|
||||
"threetwo-ui-typings": "^1.0.12",
|
||||
"threetwo-ui-typings": "^1.0.13",
|
||||
"through2": "^4.0.2",
|
||||
"unrar": "^0.2.0",
|
||||
"xml2js": "^0.4.23"
|
||||
|
||||
@@ -100,7 +100,7 @@ export default class ApiService extends Service {
|
||||
// 1. Send task to queue
|
||||
console.log(`Recieved ${action.type} event.`)
|
||||
await this.broker.call(
|
||||
"import.newImport",
|
||||
"library.newImport",
|
||||
action.data,
|
||||
{}
|
||||
);
|
||||
@@ -142,10 +142,10 @@ export default class ApiService extends Service {
|
||||
"File detected, starting import..."
|
||||
);
|
||||
const walkedFolder: IFolderData =
|
||||
await broker.call("import.walkFolders", {
|
||||
await broker.call("library.walkFolders", {
|
||||
basePathToWalk: path,
|
||||
});
|
||||
await this.broker.call("libraryqueue.enqueue", {
|
||||
await this.broker.call("library.processImport", {
|
||||
fileObject: {
|
||||
filePath: walkedFolder[0].filePath,
|
||||
fileSize: walkedFolder[0].fileSize,
|
||||
|
||||
@@ -32,7 +32,7 @@ SOFTWARE.
|
||||
*/
|
||||
|
||||
"use strict";
|
||||
import { isDate, isNil, isUndefined, map } from "lodash";
|
||||
import { isNil, isNull, isUndefined, map } from "lodash";
|
||||
import {
|
||||
Context,
|
||||
Service,
|
||||
@@ -44,7 +44,6 @@ import { DbMixin } from "../mixins/db.mixin";
|
||||
import Comic from "../models/comic.model";
|
||||
import { explodePath, walkFolder } from "../utils/file.utils";
|
||||
import { convertXMLToJSON } from "../utils/xml.utils";
|
||||
import https from "https";
|
||||
import {
|
||||
IExtractComicBookCoverErrorResponse,
|
||||
IExtractedComicBookCoverFile,
|
||||
@@ -66,7 +65,7 @@ export default class ImportService extends Service {
|
||||
public constructor(public broker: ServiceBroker) {
|
||||
super(broker);
|
||||
this.parseServiceSchema({
|
||||
name: "import",
|
||||
name: "library",
|
||||
mixins: [DbMixin("comics", Comic)],
|
||||
hooks: {},
|
||||
actions: {
|
||||
@@ -129,7 +128,7 @@ export default class ImportService extends Service {
|
||||
});
|
||||
if (!comicExists) {
|
||||
// 2. Send the extraction job to the queue
|
||||
await broker.call("libraryqueue.enqueue", {
|
||||
await broker.call("queue.processImport", {
|
||||
fileObject: {
|
||||
filePath: item.path,
|
||||
fileSize: item.stats.size,
|
||||
@@ -146,86 +145,7 @@ export default class ImportService extends Service {
|
||||
});
|
||||
},
|
||||
},
|
||||
nicefyPath: {
|
||||
rest: "POST /nicefyPath",
|
||||
params: {},
|
||||
async handler(
|
||||
ctx: Context<{
|
||||
filePath: string;
|
||||
}>
|
||||
) {
|
||||
return explodePath(ctx.params.filePath);
|
||||
},
|
||||
},
|
||||
processAndImportToDB: {
|
||||
rest: "POST /processAndImportToDB",
|
||||
|
||||
params: {},
|
||||
async handler(
|
||||
ctx: Context<{
|
||||
walkedFolder: {
|
||||
name: string;
|
||||
path: string;
|
||||
extension: string;
|
||||
containedIn: string;
|
||||
fileSize: number;
|
||||
isFile: boolean;
|
||||
isLink: boolean;
|
||||
};
|
||||
}>
|
||||
) {
|
||||
try {
|
||||
const { walkedFolder } = ctx.params;
|
||||
let comicExists = await Comic.exists({
|
||||
"rawFileDetails.name": `${walkedFolder.name}`,
|
||||
});
|
||||
// rough flow of import process
|
||||
// 1. Walk folder
|
||||
// 2. For each folder, call extract function
|
||||
// 3. For each successful extraction, run dbImport
|
||||
|
||||
if (!comicExists) {
|
||||
// 1. Extract cover and cover metadata
|
||||
let comicBookCoverMetadata:
|
||||
| IExtractedComicBookCoverFile
|
||||
| IExtractComicBookCoverErrorResponse
|
||||
| IExtractedComicBookCoverFile[] = await extractCoverFromFile2(
|
||||
walkedFolder[0]
|
||||
);
|
||||
|
||||
// 2. Add to mongo
|
||||
const dbImportResult = await this.broker.call(
|
||||
"import.rawImportToDB",
|
||||
{
|
||||
importStatus: {
|
||||
isImported: true,
|
||||
tagged: false,
|
||||
matchedResult: {
|
||||
score: "0",
|
||||
},
|
||||
},
|
||||
rawFileDetails: comicBookCoverMetadata,
|
||||
sourcedMetadata: {
|
||||
comicvine: {},
|
||||
},
|
||||
},
|
||||
{}
|
||||
);
|
||||
|
||||
return {
|
||||
comicBookCoverMetadata,
|
||||
dbImportResult,
|
||||
};
|
||||
} else {
|
||||
console.info(
|
||||
`Comic: \"${walkedFolder.name}\" already exists in the database`
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error importing comic books", error);
|
||||
}
|
||||
},
|
||||
},
|
||||
rawImportToDB: {
|
||||
rest: "POST /rawImportToDB",
|
||||
params: {},
|
||||
@@ -250,13 +170,16 @@ export default class ImportService extends Service {
|
||||
comicMetadata.sourcedMetadata.comicvine.volume
|
||||
)
|
||||
) {
|
||||
volumeDetails =
|
||||
await this.getComicVineVolumeMetadata(
|
||||
comicMetadata.sourcedMetadata.comicvine
|
||||
.volume.api_detail_url
|
||||
);
|
||||
volumeDetails = await this.broker.call(
|
||||
"comicvine.getVolumes",
|
||||
{
|
||||
volumeURI:
|
||||
comicMetadata.sourcedMetadata.comicvine
|
||||
.volume.api_detail_url,
|
||||
}
|
||||
);
|
||||
comicMetadata.sourcedMetadata.comicvine.volumeInformation =
|
||||
volumeDetails;
|
||||
volumeDetails.results;
|
||||
}
|
||||
return new Promise(async (resolve, reject) => {
|
||||
Comic.create(ctx.params, (error, data) => {
|
||||
@@ -291,35 +214,39 @@ export default class ImportService extends Service {
|
||||
const comicObjectId = new ObjectId(
|
||||
ctx.params.comicObjectId
|
||||
);
|
||||
const matchedResult = ctx.params.match;
|
||||
let volumeDetailsPromise;
|
||||
if (!isNil(matchedResult.volume)) {
|
||||
volumeDetailsPromise =
|
||||
this.getComicVineVolumeMetadata(
|
||||
matchedResult.volume.api_detail_url
|
||||
);
|
||||
}
|
||||
|
||||
return new Promise(async (resolve, reject) => {
|
||||
const volumeDetails = await volumeDetailsPromise;
|
||||
matchedResult.volumeInformation = volumeDetails;
|
||||
Comic.findByIdAndUpdate(
|
||||
comicObjectId,
|
||||
{
|
||||
sourcedMetadata: {
|
||||
comicvine: matchedResult,
|
||||
},
|
||||
},
|
||||
{ new: true },
|
||||
(err, result) => {
|
||||
if (err) {
|
||||
console.info(err);
|
||||
reject(err);
|
||||
} else {
|
||||
// 3. Fetch and append volume information
|
||||
resolve(result);
|
||||
let volumeDetails = {};
|
||||
const matchedResult = ctx.params.match;
|
||||
if (!isNil(matchedResult.volume)) {
|
||||
const volumeDetails = await this.broker.call(
|
||||
"comicvine.getVolumes",
|
||||
{
|
||||
volumeURI:
|
||||
matchedResult.volume.api_detail_url,
|
||||
}
|
||||
}
|
||||
);
|
||||
);
|
||||
matchedResult.volumeInformation =
|
||||
volumeDetails.results;
|
||||
Comic.findByIdAndUpdate(
|
||||
comicObjectId,
|
||||
{
|
||||
sourcedMetadata: {
|
||||
comicvine: matchedResult,
|
||||
},
|
||||
},
|
||||
{ new: true },
|
||||
(err, result) => {
|
||||
if (err) {
|
||||
console.info(err);
|
||||
reject(err);
|
||||
} else {
|
||||
// 3. Fetch and append volume information
|
||||
resolve(result);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
});
|
||||
},
|
||||
},
|
||||
@@ -386,15 +313,17 @@ export default class ImportService extends Service {
|
||||
getComicBooksByIds: {
|
||||
rest: "POST /getComicBooksByIds",
|
||||
params: { ids: "array" },
|
||||
handler: async (ctx: Context<{ ids: [string]}>) => {
|
||||
handler: async (ctx: Context<{ ids: [string] }>) => {
|
||||
console.log(ctx.params.ids);
|
||||
const queryIds = ctx.params.ids.map((id) => new ObjectId(id));
|
||||
const queryIds = ctx.params.ids.map(
|
||||
(id) => new ObjectId(id)
|
||||
);
|
||||
return await Comic.find({
|
||||
'_id': {
|
||||
_id: {
|
||||
$in: queryIds,
|
||||
}
|
||||
})
|
||||
}
|
||||
},
|
||||
});
|
||||
},
|
||||
},
|
||||
getComicBookGroups: {
|
||||
rest: "GET /getComicBookGroups",
|
||||
@@ -449,61 +378,104 @@ export default class ImportService extends Service {
|
||||
return Promise.all(volumesMetadata);
|
||||
},
|
||||
},
|
||||
findIssuesForSeriesInLibrary: {
|
||||
rest: "POST /findIssuesForSeriesInLibrary",
|
||||
|
||||
findIssuesForSeries: {
|
||||
rest: "POST /findIssueForSeries",
|
||||
params: {},
|
||||
handler: async (
|
||||
ctx: Context<{ comicObjectID: string }>
|
||||
ctx: Context<{
|
||||
queryObjects: [
|
||||
{
|
||||
issueId: string;
|
||||
issueName: string;
|
||||
volumeName: string;
|
||||
issueNumber: string;
|
||||
}
|
||||
];
|
||||
}>
|
||||
) => {
|
||||
// 1. Query mongo to get the comic document by its _id
|
||||
const comicBookDetails: any = await this.broker.call(
|
||||
"import.getComicBookById",
|
||||
{ id: ctx.params.comicObjectID }
|
||||
// 2a. Enqueue the Elasticsearch job
|
||||
const { queryObjects } = ctx.params;
|
||||
// construct the query for ElasticSearch
|
||||
let elasticSearchQuery = {};
|
||||
const elasticSearchQueries = queryObjects.map(
|
||||
(queryObject) => {
|
||||
console.log("Volume: ", queryObject.volumeName);
|
||||
console.log("Issue: ", queryObject.issueName);
|
||||
if (queryObject.issueName === null) {
|
||||
queryObject.issueName = "";
|
||||
}
|
||||
if (queryObject.volumeName === null) {
|
||||
queryObject.volumeName = "";
|
||||
}
|
||||
elasticSearchQuery = {
|
||||
bool: {
|
||||
must: [
|
||||
// {
|
||||
// match_phrase: {
|
||||
// "rawFileDetails.name":
|
||||
// queryObject.issueName,
|
||||
// },
|
||||
// },
|
||||
{
|
||||
match_phrase: {
|
||||
"rawFileDetails.name":
|
||||
queryObject.volumeName,
|
||||
},
|
||||
},
|
||||
{
|
||||
term: {
|
||||
"inferredMetadata.issue.number":
|
||||
parseInt(queryObject.issueNumber, 10),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
return [
|
||||
{
|
||||
index: "comics",
|
||||
search_type: "dfs_query_then_fetch",
|
||||
},
|
||||
// { issueId: queryObject.issueId },
|
||||
{
|
||||
query: elasticSearchQuery,
|
||||
// script_fields: {
|
||||
// issueId: {
|
||||
// script: {
|
||||
// lang: "painless",
|
||||
// params: {
|
||||
// match: {
|
||||
// issueId:
|
||||
// queryObject.issueId,
|
||||
// },
|
||||
// },
|
||||
// inline: "params.match",
|
||||
// },
|
||||
// },
|
||||
// fileName: {
|
||||
// script: {
|
||||
// lang: "painless",
|
||||
// inline: "params['_source']['rawFileDetails']",
|
||||
// },
|
||||
// },
|
||||
// },
|
||||
},
|
||||
];
|
||||
}
|
||||
);
|
||||
console.log(
|
||||
JSON.stringify(elasticSearchQueries, null, 2)
|
||||
);
|
||||
|
||||
// 2. Query CV and get metadata for them
|
||||
const foo =
|
||||
await comicBookDetails.sourcedMetadata.comicvine.volumeInformation.issues.map(
|
||||
async (issue: any, idx: any) => {
|
||||
const metadata: any = await axios.request({
|
||||
url: `${issue.api_detail_url}?api_key=${process.env.COMICVINE_API_KEY}`,
|
||||
params: {
|
||||
resources: "issues",
|
||||
limit: "100",
|
||||
format: "json",
|
||||
},
|
||||
headers: {
|
||||
"User-Agent": "ThreeTwo",
|
||||
},
|
||||
});
|
||||
const issueMetadata = metadata.data.results;
|
||||
|
||||
// 2a. Enqueue the Elasticsearch job
|
||||
if (
|
||||
!isUndefined(issueMetadata.volume.name) &&
|
||||
!isUndefined(issueMetadata.issue_number)
|
||||
) {
|
||||
await ctx.broker.call(
|
||||
"libraryqueue.issuesForSeries",
|
||||
{
|
||||
queryObject: {
|
||||
issueId: issue.id,
|
||||
issueName: issueMetadata.name,
|
||||
volumeName:
|
||||
issueMetadata.volume
|
||||
.name,
|
||||
issueNumber:
|
||||
issueMetadata.issue_number,
|
||||
issueMetadata,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
// 3. Just return the issues
|
||||
return issueMetadata;
|
||||
}
|
||||
);
|
||||
return Promise.all(foo);
|
||||
return await ctx.broker.call("search.searchComic", {
|
||||
elasticSearchQueries,
|
||||
queryObjects,
|
||||
});
|
||||
// await ctx.broker.call("queue.issuesForSeries", {
|
||||
// elasticSearchQueries,
|
||||
// });
|
||||
},
|
||||
},
|
||||
flushDB: {
|
||||
@@ -551,40 +523,7 @@ export default class ImportService extends Service {
|
||||
},
|
||||
},
|
||||
},
|
||||
methods: {
|
||||
getComicVineVolumeMetadata: (apiDetailURL) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const options = {
|
||||
headers: {
|
||||
"User-Agent": "ThreeTwo",
|
||||
},
|
||||
};
|
||||
return https
|
||||
.get(
|
||||
`${apiDetailURL}?api_key=${process.env.COMICVINE_API_KEY}&format=json&limit=1&offset=0`,
|
||||
options,
|
||||
(resp) => {
|
||||
let data = "";
|
||||
resp.on("data", (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
resp.on("end", () => {
|
||||
console.log(
|
||||
`${apiDetailURL} returned data.`
|
||||
);
|
||||
const volumeInformation =
|
||||
JSON.parse(data);
|
||||
resolve(volumeInformation.results);
|
||||
});
|
||||
}
|
||||
)
|
||||
.on("error", (err) => {
|
||||
console.info("Error: " + err.message);
|
||||
reject(err);
|
||||
});
|
||||
}),
|
||||
},
|
||||
methods: {},
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -47,15 +47,16 @@ import { SandboxedJob } from "moleculer-bull";
|
||||
import { DbMixin } from "../mixins/db.mixin";
|
||||
import Comic from "../models/comic.model";
|
||||
import { extractCoverFromFile2 } from "../utils/uncompression.utils";
|
||||
import { refineQuery } from "filename-parser";
|
||||
import { io } from "./api.service";
|
||||
const REDIS_URI = process.env.REDIS_URI || `redis://0.0.0.0:6379`;
|
||||
|
||||
console.log(`REDIS -> ${REDIS_URI}`);
|
||||
export default class LibraryQueueService extends Service {
|
||||
export default class QueueService extends Service {
|
||||
public constructor(public broker: ServiceBroker) {
|
||||
super(broker);
|
||||
this.parseServiceSchema({
|
||||
name: "libraryqueue",
|
||||
name: "queue",
|
||||
mixins: [BullMQMixin(REDIS_URI), DbMixin("comics", Comic)],
|
||||
settings: {},
|
||||
hooks: {},
|
||||
@@ -70,9 +71,13 @@ export default class LibraryQueueService extends Service {
|
||||
job.data.fileObject
|
||||
);
|
||||
|
||||
// infer any issue-related metadata from the filename
|
||||
const { inferredIssueDetails } = refineQuery(result.name);
|
||||
console.log("Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2));
|
||||
|
||||
// write to mongo
|
||||
const dbImportResult = await this.broker.call(
|
||||
"import.rawImportToDB",
|
||||
"library.rawImportToDB",
|
||||
{
|
||||
importStatus: {
|
||||
isImported: true,
|
||||
@@ -82,6 +87,9 @@ export default class LibraryQueueService extends Service {
|
||||
},
|
||||
},
|
||||
rawFileDetails: result,
|
||||
inferredMetadata: {
|
||||
issue: inferredIssueDetails,
|
||||
},
|
||||
sourcedMetadata: {
|
||||
comicvine: {},
|
||||
},
|
||||
@@ -96,45 +104,11 @@ export default class LibraryQueueService extends Service {
|
||||
});
|
||||
},
|
||||
},
|
||||
"issue.findMatchesInLibrary": {
|
||||
concurrency: 20,
|
||||
async process(job: SandboxedJob) {
|
||||
try {
|
||||
console.log(
|
||||
"Job recieved to find issue matches in library."
|
||||
);
|
||||
const matchesInLibrary = await this.broker.call(
|
||||
"search.searchComic",
|
||||
{
|
||||
queryObject: job.data.queryObject,
|
||||
}
|
||||
);
|
||||
if (
|
||||
!isNil(matchesInLibrary) &&
|
||||
!isUndefined(matchesInLibrary)
|
||||
) {
|
||||
console.log("Matches found in library:");
|
||||
console.log(matchesInLibrary);
|
||||
|
||||
const foo = extend(
|
||||
{ issue: job.data.queryObject.issueMetadata },
|
||||
{ matches: matchesInLibrary }
|
||||
);
|
||||
return foo;
|
||||
} else {
|
||||
console.log(
|
||||
"No match was found for this issue in the library."
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
},
|
||||
actions: {
|
||||
enqueue: {
|
||||
rest: "POST /enqueue",
|
||||
processImport: {
|
||||
rest: "POST /processImport",
|
||||
params: {},
|
||||
async handler(
|
||||
ctx: Context<{
|
||||
@@ -146,28 +120,6 @@ export default class LibraryQueueService extends Service {
|
||||
});
|
||||
},
|
||||
},
|
||||
issuesForSeries: {
|
||||
rest: "POST /findIssuesForSeries",
|
||||
params: {},
|
||||
handler: async (
|
||||
ctx: Context<{
|
||||
queryObject: {
|
||||
issueName: string;
|
||||
volumeName: string;
|
||||
issueNumber: string;
|
||||
issueId: string;
|
||||
issueMetadata: object;
|
||||
};
|
||||
}>
|
||||
) => {
|
||||
return await this.createJob(
|
||||
"issue.findMatchesInLibrary",
|
||||
{
|
||||
queryObject: ctx.params.queryObject,
|
||||
}
|
||||
);
|
||||
},
|
||||
},
|
||||
},
|
||||
methods: {},
|
||||
async started(): Promise<any> {
|
||||
@@ -213,7 +165,7 @@ export default class LibraryQueueService extends Service {
|
||||
"completed",
|
||||
async (job, res) => {
|
||||
client.emit("action", {
|
||||
type: "CV_ISSUES_FOR_VOLUME_IN_LIBRARY_SUCCESS",
|
||||
type: "CV_ISSUES_FOR_VOLUME_IN_LIBRARY_UPDATED",
|
||||
result: res,
|
||||
});
|
||||
console.info(
|
||||
|
||||
@@ -7,21 +7,13 @@ import {
|
||||
Errors,
|
||||
} from "moleculer";
|
||||
|
||||
const { Client } = require("@elastic/elasticsearch");
|
||||
const client = new Client({
|
||||
node: "http://tower.local:9200",
|
||||
auth: {
|
||||
username: "elastic",
|
||||
password: "password",
|
||||
},
|
||||
});
|
||||
|
||||
import { DbMixin } from "../mixins/db.mixin";
|
||||
import Comic from "../models/comic.model";
|
||||
import { refineQuery } from "filename-parser";
|
||||
import { filter, isEmpty, isNull } from "lodash";
|
||||
|
||||
console.log(client);
|
||||
import { each, filter, flatten, isEmpty, isNull } from "lodash";
|
||||
import { eSClient } from "../models/comic.model";
|
||||
import arrayToNDJSON from "array-to-ndjson";
|
||||
const s = eSClient.helpers.msearch();
|
||||
|
||||
export default class SettingsService extends Service {
|
||||
// @ts-ignore
|
||||
@@ -34,7 +26,7 @@ export default class SettingsService extends Service {
|
||||
Service.mergeSchemas(
|
||||
{
|
||||
name: "search",
|
||||
mixins: [client, DbMixin("comics", Comic)],
|
||||
mixins: [DbMixin("comics", Comic)],
|
||||
hooks: {},
|
||||
actions: {
|
||||
searchComic: {
|
||||
@@ -43,95 +35,55 @@ export default class SettingsService extends Service {
|
||||
timeout: 400000,
|
||||
async handler(
|
||||
ctx: Context<{
|
||||
queryObject: {
|
||||
issueName: string;
|
||||
volumeName: string;
|
||||
issueNumber: string;
|
||||
};
|
||||
queryObjects: [],
|
||||
elasticSearchQueries: [
|
||||
{
|
||||
elasticSearchQuery: object;
|
||||
}
|
||||
];
|
||||
}>
|
||||
) {
|
||||
let elasticSearchQuery = {};
|
||||
console.log(
|
||||
"Volume: ",
|
||||
ctx.params.queryObject.volumeName
|
||||
const flattenedQueryArray = flatten(
|
||||
ctx.params.elasticSearchQueries
|
||||
);
|
||||
console.log(
|
||||
"Issue: ",
|
||||
ctx.params.queryObject.issueName
|
||||
);
|
||||
if (isNull(ctx.params.queryObject.volumeName)) {
|
||||
elasticSearchQuery = {
|
||||
match: {
|
||||
"rawFileDetails.name": {
|
||||
query: ctx.params.queryObject
|
||||
.issueName,
|
||||
operator: "and",
|
||||
fuzziness: "AUTO",
|
||||
},
|
||||
},
|
||||
};
|
||||
} else if (
|
||||
isNull(ctx.params.queryObject.issueName)
|
||||
) {
|
||||
elasticSearchQuery = {
|
||||
match: {
|
||||
"rawFileDetails.name": {
|
||||
query: ctx.params.queryObject
|
||||
.volumeName,
|
||||
operator: "and",
|
||||
fuzziness: "AUTO",
|
||||
},
|
||||
},
|
||||
};
|
||||
} else {
|
||||
elasticSearchQuery = {
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
match_phrase: {
|
||||
"rawFileDetails.name":
|
||||
ctx.params
|
||||
.queryObject
|
||||
.issueName,
|
||||
},
|
||||
},
|
||||
{
|
||||
match_phrase: {
|
||||
"rawFileDetails.name":
|
||||
ctx.params
|
||||
.queryObject
|
||||
.volumeName,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
console.log(elasticSearchQuery);
|
||||
return Comic.esSearch({
|
||||
query: elasticSearchQuery,
|
||||
}).then(function (results) {
|
||||
// results here
|
||||
const foo = results.body.hits.hits.map(
|
||||
(hit) => {
|
||||
const parsedFilename = refineQuery(
|
||||
hit._source.rawFileDetails.name
|
||||
);
|
||||
if (
|
||||
parsedFilename.searchParams
|
||||
.searchTerms.number ===
|
||||
parseInt(
|
||||
ctx.params.queryObject
|
||||
.issueNumber,
|
||||
10
|
||||
)
|
||||
) {
|
||||
return hit;
|
||||
}
|
||||
}
|
||||
);
|
||||
return filter(foo, null);
|
||||
let queries = flattenedQueryArray
|
||||
.map((item) => JSON.stringify(item))
|
||||
.join("\n");
|
||||
queries += "\n";
|
||||
const { body } = await eSClient.msearch({
|
||||
body: queries,
|
||||
});
|
||||
|
||||
body.responses.forEach((match) => {
|
||||
console.log(match.hits.hits);
|
||||
})
|
||||
|
||||
return body.responses;
|
||||
|
||||
// return Comic.esSearch({
|
||||
// query: elasticSearchQuery,
|
||||
// }).then(function (results) {
|
||||
// // results here
|
||||
// const foo = results.body.hits.hits.map(
|
||||
// (hit) => {
|
||||
// const parsedFilename = refineQuery(
|
||||
// hit._source.rawFileDetails.name
|
||||
// );
|
||||
// if (
|
||||
// parsedFilename.searchParams
|
||||
// .searchTerms.number ===
|
||||
// parseInt(
|
||||
// ctx.params.queryObject
|
||||
// .issueNumber,
|
||||
// 10
|
||||
// )
|
||||
// ) {
|
||||
// return hit;
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
// return filter(foo, null);
|
||||
// });
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -53,7 +53,7 @@ import { USERDATA_DIRECTORY, COMICS_DIRECTORY } from "../constants/directories";
|
||||
|
||||
export const extractCoverFromFile2 = async (
|
||||
fileObject: any
|
||||
): Promise<any> => {
|
||||
): Promise<IExtractedComicBookCoverFile> => {
|
||||
try {
|
||||
const { filePath, fileSize } = fileObject;
|
||||
|
||||
@@ -116,7 +116,7 @@ export const extractCoverFromFile2 = async (
|
||||
cover: {
|
||||
filePath: path.relative(process.cwd(),renditionPath),
|
||||
},
|
||||
containedIn: path.dirname(fileNameWithExtension),
|
||||
containedIn: path.resolve(fileNameWithExtension),
|
||||
calibreMetadata: {
|
||||
coverWriteResult: result,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user