diff --git a/models/comic.model.ts b/models/comic.model.ts index 3a05cfd..da8f65d 100644 --- a/models/comic.model.ts +++ b/models/comic.model.ts @@ -2,6 +2,13 @@ const mongoose = require("mongoose"); var mexp = require("mongoose-elasticsearch-xp").v7; const paginate = require("mongoose-paginate-v2"); const { Client } = require("@elastic/elasticsearch"); +import ComicVineMetadataSchema from "./comicvine.metadata.model"; +import { mongoosastic } from "mongoosastic-ts"; +import { + MongoosasticDocument, + MongoosasticModel, + MongoosasticPluginOpts, +} from "mongoosastic-ts/dist/types"; export const eSClient = new Client({ node: "http://localhost:9200", @@ -11,6 +18,23 @@ export const eSClient = new Client({ }, }); +const RawFileDetailsSchema = mongoose.Schema({ + _id: false, + name: String, + filePath: String, + fileSize: Number, + extension: String, + containedIn: String, + pageCount: Number, + cover: { + filePath: String, + stats: Object, + }, + calibreMetadata: { + coverWriteResult: String, + }, +}); + const ComicSchema = mongoose.Schema( { importStatus: { @@ -25,36 +49,25 @@ const ComicSchema = mongoose.Schema( }, sourcedMetadata: { comicInfo: { type: mongoose.Schema.Types.Mixed, default: {} }, - comicvine: { type: mongoose.Schema.Types.Mixed, default: {} }, + comicvine: { + type: ComicVineMetadataSchema, + es_indexed: true, + default: {}, + }, shortboxed: {}, gcd: {}, }, - rawFileDetails: { - name: { type: String, es_indexed: true }, - filePath: String, - fileSize: Number, - extension: String, - containedIn: String, - pageCount: Number, - cover: { - filePath: String, - stats: Object, - }, - calibreMetadata: { - coverWriteResult: String, - }, - }, + rawFileDetails: { type: RawFileDetailsSchema, es_indexed: true }, inferredMetadata: { issue: { - name: String, + name: { type: String, es_indexed: true }, number: { type: Number, - es_indexed: true, required: false, default: 0, }, year: String, - subtitle: String, + subtitle: { type: String, es_indexed: true }, }, }, acquisition: { @@ -75,9 +88,22 @@ const ComicSchema = mongoose.Schema( { timestamps: true, minimize: false } ); -ComicSchema.plugin(mexp, { - client: eSClient, +ComicSchema.plugin(mongoosastic, { + esClient: eSClient, }); ComicSchema.plugin(paginate); -const Comic = mongoose.model("Comic", ComicSchema); + +const Comic = mongoose.model("Comic", ComicSchema), + stream = Comic.synchronize(); +let count = 0; + +stream.on("data", function (err, doc) { + count++; +}); +stream.on("close", function () { + console.log("indexed " + count + " documents!"); +}); +stream.on("error", function (err) { + console.log(err); +}); export default Comic; diff --git a/models/comicvine.metadata.model.ts b/models/comicvine.metadata.model.ts index 9973654..0955e52 100644 --- a/models/comicvine.metadata.model.ts +++ b/models/comicvine.metadata.model.ts @@ -1,5 +1,6 @@ const mongoose = require("mongoose"); const Things = mongoose.Schema({ + _id: false, api_detail_url: String, id: Number, name: String, @@ -7,13 +8,15 @@ const Things = mongoose.Schema({ count: String, }); const Issue = mongoose.Schema({ + _id: false, api_detail_url: String, id: Number, name: String, issue_number: String, }); const VolumeInformation = mongoose.Schema({ - aliases: Array, + _id: false, + aliases: [], api_detail_url: String, characters: [Things], concepts: [Things], @@ -60,7 +63,8 @@ const VolumeInformation = mongoose.Schema({ start_year: String, }); -const ComicVineMetadata = mongoose.Schema({ +const ComicVineMetadataSchema = mongoose.Schema({ + _id: false, aliases: [String], api_detail_url: String, cover_date: String, @@ -75,4 +79,4 @@ const ComicVineMetadata = mongoose.Schema({ volumeInformation: VolumeInformation, }); -export default ComicVineMetadata; +export default ComicVineMetadataSchema; diff --git a/package-lock.json b/package-lock.json index b2a2761..0b3b302 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,6 +38,7 @@ "moleculer-db-adapter-mongo": "^0.4.7", "moleculer-db-adapter-mongoose": "^0.8.9", "moleculer-web": "^0.10.4", + "mongoosastic-ts": "^5.0.7", "mongoose": "^5.12.7", "mongoose-elasticsearch-xp": "^5.8.0", "mongoose-paginate-v2": "^1.3.18", @@ -2250,7 +2251,6 @@ "version": "3.5.2", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", - "peer": true, "dependencies": { "humanize-ms": "^1.2.1" }, @@ -2321,7 +2321,6 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -3036,7 +3035,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "peer": true, "dependencies": { "ansi-styles": "^2.2.1", "escape-string-regexp": "^1.0.2", @@ -3754,7 +3752,6 @@ "version": "16.7.2", "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz", "integrity": "sha512-1ZLKZlG2ABfYVBX2d7/JgxOsKJrM5Yu62GvshWu7ZSvhxPomCN4Gas90DS51yYI56JolY0XGhyiRlUhLhIL05Q==", - "peer": true, "dependencies": { "agentkeepalive": "^3.4.1", "chalk": "^1.0.0", @@ -5127,7 +5124,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "peer": true, "dependencies": { "ansi-regex": "^2.0.0" }, @@ -5277,7 +5273,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=", - "peer": true, "dependencies": { "ms": "^2.0.0" } @@ -8514,6 +8509,11 @@ "lodash._basetostring": "~4.12.0" } }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" + }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -9157,6 +9157,18 @@ } } }, + "node_modules/mongoosastic-ts": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/mongoosastic-ts/-/mongoosastic-ts-5.0.7.tgz", + "integrity": "sha512-os/wLY7jYt/oLDuZCXaXKU7D2acC89imk+jVa9HCPbDlDMC6A6RMW5x0GlGyXtoMtNTeFBtc5V8vuy1GgEQk1Q==", + "dependencies": { + "elasticsearch": "16.7.2", + "lodash.clonedeep": "4.5.0" + }, + "engines": { + "node": ">= 8.0" + } + }, "node_modules/mongoose": { "version": "5.13.14", "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.13.14.tgz", @@ -10139,6 +10151,8 @@ }, "node_modules/npm/node_modules/cli-columns/node_modules/string-width": { "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "inBundle": true, "license": "MIT", @@ -10343,6 +10357,8 @@ }, "node_modules/npm/node_modules/debug/node_modules/ms": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true, "inBundle": true, "license": "MIT" @@ -10514,6 +10530,8 @@ }, "node_modules/npm/node_modules/gauge/node_modules/string-width": { "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, "inBundle": true, "license": "MIT", @@ -13910,7 +13928,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", - "peer": true, "engines": { "node": ">=0.8.0" } @@ -16706,7 +16723,6 @@ "version": "3.5.2", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", - "peer": true, "requires": { "humanize-ms": "^1.2.1" } @@ -16753,8 +16769,7 @@ "ansi-styles": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", - "peer": true + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" }, "anymatch": { "version": "3.1.2", @@ -17298,7 +17313,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "peer": true, "requires": { "ansi-styles": "^2.2.1", "escape-string-regexp": "^1.0.2", @@ -17863,7 +17877,6 @@ "version": "16.7.2", "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz", "integrity": "sha512-1ZLKZlG2ABfYVBX2d7/JgxOsKJrM5Yu62GvshWu7ZSvhxPomCN4Gas90DS51yYI56JolY0XGhyiRlUhLhIL05Q==", - "peer": true, "requires": { "agentkeepalive": "^3.4.1", "chalk": "^1.0.0", @@ -18942,7 +18955,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "peer": true, "requires": { "ansi-regex": "^2.0.0" } @@ -19055,7 +19067,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=", - "peer": true, "requires": { "ms": "^2.0.0" } @@ -21458,6 +21469,11 @@ "lodash._basetostring": "~4.12.0" } }, + "lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" + }, "lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -21868,6 +21884,15 @@ "saslprep": "^1.0.0" } }, + "mongoosastic-ts": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/mongoosastic-ts/-/mongoosastic-ts-5.0.7.tgz", + "integrity": "sha512-os/wLY7jYt/oLDuZCXaXKU7D2acC89imk+jVa9HCPbDlDMC6A6RMW5x0GlGyXtoMtNTeFBtc5V8vuy1GgEQk1Q==", + "requires": { + "elasticsearch": "16.7.2", + "lodash.clonedeep": "4.5.0" + } + }, "mongoose": { "version": "5.13.14", "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.13.14.tgz", @@ -22562,6 +22587,8 @@ }, "string-width": { "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "bundled": true, "dev": true, "requires": { @@ -22706,6 +22733,8 @@ "dependencies": { "ms": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "bundled": true, "dev": true } @@ -22837,6 +22866,8 @@ }, "string-width": { "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "bundled": true, "dev": true, "requires": { @@ -25337,8 +25368,7 @@ "supports-color": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", - "peer": true + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, "supports-hyperlinks": { "version": "2.2.0", diff --git a/package.json b/package.json index 0ae6a85..93019d7 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "moleculer-db-adapter-mongo": "^0.4.7", "moleculer-db-adapter-mongoose": "^0.8.9", "moleculer-web": "^0.10.4", + "mongoosastic-ts": "^5.0.7", "mongoose": "^5.12.7", "mongoose-elasticsearch-xp": "^5.8.0", "mongoose-paginate-v2": "^1.3.18", diff --git a/services/library.service.ts b/services/library.service.ts index f1e1ca3..842e120 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -190,10 +190,11 @@ export default class ImportService extends Service { comicMetadata.sourcedMetadata.comicvine.volumeInformation = volumeDetails.results; } - return new Promise(async (resolve, reject) => { - Comic.create(ctx.params, (error, data) => { + return await Comic.create( + ctx.params, + (error, data) => { if (data) { - resolve(data); + return data; } else if (error) { console.log("data", data); console.log("error", error); @@ -204,8 +205,9 @@ export default class ImportService extends Service { error ); } - }); - }); + } + ); + }, }, applyComicVineMetadata: { @@ -484,14 +486,15 @@ export default class ImportService extends Service { issues: [ { $match: { - "sourcedMetadata.comicvine": { - $gt: {}, - }, + "sourcedMetadata.comicvine.volumeInformation": + { + $gt: {}, + }, }, }, { $group: { - _id: "$sourcedMetadata.comicvine", + _id: "$sourcedMetadata.comicvine.volumeInformation", data: { $push: "$$ROOT._id" }, }, }, diff --git a/services/libraryqueue.service.ts b/services/libraryqueue.service.ts index cc33a30..5217efa 100644 --- a/services/libraryqueue.service.ts +++ b/services/libraryqueue.service.ts @@ -102,8 +102,7 @@ export default class QueueService extends Service { acquisition: { wanted: false, } - }, - {} + } ); return Promise.resolve({ dbImportResult, diff --git a/services/search.service.ts b/services/search.service.ts index d3c2616..e25f5ee 100644 --- a/services/search.service.ts +++ b/services/search.service.ts @@ -9,7 +9,7 @@ import { import { DbMixin } from "../mixins/db.mixin"; import Comic from "../models/comic.model"; -import { flatten, isEmpty, isUndefined } from "lodash"; +import { flatten, isEmpty, isUndefined, map } from "lodash"; import { eSClient } from "../models/comic.model"; const s = eSClient.helpers.msearch(); @@ -52,11 +52,11 @@ export default class SettingsService extends Service { body: queries, }); - body.responses.forEach((match) => { + body.results.forEach((match) => { console.log(match.hits.hits); }); - return body.responses; + return body.results; }, }, issue: { @@ -68,11 +68,16 @@ export default class SettingsService extends Service { volumeName: string; issueNumber: string; }; + pagination: { + size: number; + from: number, + }; }> ) => { - console.log(ctx.params.query); + console.log(ctx.params); + const { query, pagination } = ctx.params; let eSQuery = {}; - if (isEmpty(ctx.params.query)) { + if (isEmpty(query)) { Object.assign(eSQuery, { match_all: {}, }); @@ -80,24 +85,23 @@ export default class SettingsService extends Service { Object.assign(eSQuery, { match: { "rawFileDetails.name": - ctx.params.query.volumeName, + query.volumeName, }, }); } - console.log("eSQuery", eSQuery); + console.log(query); const result = await eSClient.search( { index: "comics", body: { query: eSQuery, }, - size: 50, + ...pagination, }, { hydrate: true } ); - console.log(result.body) - const { hits } = result.body; - return hits; + + return result; }, }, }, diff --git a/yarn.lock b/yarn.lock index 16c5454..99208e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2408,7 +2408,7 @@ "resolved" "https://registry.npmjs.org/efrt-unpack/-/efrt-unpack-2.2.0.tgz" "version" "2.2.0" -"elasticsearch@^12.1.3 || ^13.0.0 || ^14.0.0 || ^16.0.0": +"elasticsearch@^12.1.3 || ^13.0.0 || ^14.0.0 || ^16.0.0", "elasticsearch@16.7.2": "integrity" "sha512-1ZLKZlG2ABfYVBX2d7/JgxOsKJrM5Yu62GvshWu7ZSvhxPomCN4Gas90DS51yYI56JolY0XGhyiRlUhLhIL05Q==" "resolved" "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz" "version" "16.7.2" @@ -4602,6 +4602,11 @@ dependencies: "lodash._basetostring" "~4.12.0" +"lodash.clonedeep@4.5.0": + "integrity" "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" + "resolved" "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz" + "version" "4.5.0" + "lodash.defaults@^4.2.0": "integrity" "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" "resolved" "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz" @@ -5018,6 +5023,14 @@ optionalDependencies: "saslprep" "^1.0.0" +"mongoosastic-ts@^5.0.7": + "integrity" "sha512-os/wLY7jYt/oLDuZCXaXKU7D2acC89imk+jVa9HCPbDlDMC6A6RMW5x0GlGyXtoMtNTeFBtc5V8vuy1GgEQk1Q==" + "resolved" "https://registry.npmjs.org/mongoosastic-ts/-/mongoosastic-ts-5.0.7.tgz" + "version" "5.0.7" + dependencies: + "elasticsearch" "16.7.2" + "lodash.clonedeep" "4.5.0" + "mongoose-elasticsearch-xp@^5.8.0": "integrity" "sha512-snb8OdeIuX64vvlyQ/Hwu2vxQDLElixmxkSn3dOeuZ3TepPN2l4PQvfBLHmPT3e86HH93jHeMNU/PJtNOc4FOw==" "resolved" "https://registry.npmjs.org/mongoose-elasticsearch-xp/-/mongoose-elasticsearch-xp-5.8.0.tgz"