🔎 Elasticsearch endpoints for you know, search

models/comic.model.ts

@@ -2,6 +2,13 @@ const mongoose = require("mongoose");
 var mexp = require("mongoose-elasticsearch-xp").v7;
 const paginate = require("mongoose-paginate-v2");
 const { Client } = require("@elastic/elasticsearch");
+import ComicVineMetadataSchema from "./comicvine.metadata.model";
+import { mongoosastic } from "mongoosastic-ts";
+import {
+	MongoosasticDocument,
+	MongoosasticModel,
+	MongoosasticPluginOpts,
+} from "mongoosastic-ts/dist/types";
 
 export const eSClient = new Client({
 	node: "http://localhost:9200",
@@ -11,6 +18,23 @@ export const eSClient = new Client({
 	},
 });
 
+const RawFileDetailsSchema = mongoose.Schema({
+	_id: false,
+	name: String,
+	filePath: String,
+	fileSize: Number,
+	extension: String,
+	containedIn: String,
+	pageCount: Number,
+	cover: {
+		filePath: String,
+		stats: Object,
+	},
+	calibreMetadata: {
+		coverWriteResult: String,
+	},
+});
+
 const ComicSchema = mongoose.Schema(
 	{
 		importStatus: {
@@ -25,36 +49,25 @@ const ComicSchema = mongoose.Schema(
 		},
 		sourcedMetadata: {
 			comicInfo: { type: mongoose.Schema.Types.Mixed, default: {} },
-			comicvine: { type: mongoose.Schema.Types.Mixed, default: {} },
+			comicvine: {
+				type: ComicVineMetadataSchema,
+				es_indexed: true,
+				default: {},
+			},
 			shortboxed: {},
 			gcd: {},
 		},
-		rawFileDetails: {
+		rawFileDetails: { type: RawFileDetailsSchema, es_indexed: true },
-			name: { type: String, es_indexed: true },
-			filePath: String,
-			fileSize: Number,
-			extension: String,
-			containedIn: String,
-			pageCount: Number,
-			cover: {
-				filePath: String,
-				stats: Object,
-			},
-			calibreMetadata: {
-				coverWriteResult: String,
-			},
-		},
 		inferredMetadata: {
 			issue: {
-				name: String,
+				name: { type: String, es_indexed: true },
 				number: {
 					type: Number,
-					es_indexed: true,
 					required: false,
 					default: 0,
 				},
 				year: String,
-				subtitle: String,
+				subtitle: { type: String, es_indexed: true },
 			},
 		},
 		acquisition: {
@@ -75,9 +88,22 @@ const ComicSchema = mongoose.Schema(
 	{ timestamps: true, minimize: false }
 );
 
-ComicSchema.plugin(mexp, {
+ComicSchema.plugin(mongoosastic, {
-	client: eSClient,
+	esClient: eSClient,
 });
 ComicSchema.plugin(paginate);
-const Comic = mongoose.model("Comic", ComicSchema);
+
+const Comic = mongoose.model("Comic", ComicSchema),
+	stream = Comic.synchronize();
+let count = 0;
+
+stream.on("data", function (err, doc) {
+	count++;
+});
+stream.on("close", function () {
+	console.log("indexed " + count + " documents!");
+});
+stream.on("error", function (err) {
+	console.log(err);
+});
 export default Comic;

models/comicvine.metadata.model.ts

@@ -1,5 +1,6 @@
 const mongoose = require("mongoose");
 const Things = mongoose.Schema({
+	_id: false,
 	api_detail_url: String,
 	id: Number,
 	name: String,
@@ -7,13 +8,15 @@ const Things = mongoose.Schema({
 	count: String,
 });
 const Issue = mongoose.Schema({
+	_id: false,
 	api_detail_url: String,
 	id: Number,
 	name: String,
 	issue_number: String,
 });
 const VolumeInformation = mongoose.Schema({
-	aliases: Array,
+	_id: false,
+	aliases: [],
 	api_detail_url: String,
 	characters: [Things],
 	concepts: [Things],
@@ -60,7 +63,8 @@ const VolumeInformation = mongoose.Schema({
 	start_year: String,
 });
 
-const ComicVineMetadata = mongoose.Schema({
+const ComicVineMetadataSchema = mongoose.Schema({
+	_id: false,
 	aliases: [String],
 	api_detail_url: String,
 	cover_date: String,
@@ -75,4 +79,4 @@ const ComicVineMetadata = mongoose.Schema({
 	volumeInformation: VolumeInformation,
 });
 
-export default ComicVineMetadata;
+export default ComicVineMetadataSchema;

package-lock.json

@@ -38,6 +38,7 @@
         "moleculer-db-adapter-mongo": "^0.4.7",
         "moleculer-db-adapter-mongoose": "^0.8.9",
         "moleculer-web": "^0.10.4",
+        "mongoosastic-ts": "^5.0.7",
         "mongoose": "^5.12.7",
         "mongoose-elasticsearch-xp": "^5.8.0",
         "mongoose-paginate-v2": "^1.3.18",
@@ -2250,7 +2251,6 @@
       "version": "3.5.2",
       "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz",
       "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==",
-      "peer": true,
       "dependencies": {
         "humanize-ms": "^1.2.1"
       },
@@ -2321,7 +2321,6 @@
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
       "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
-      "peer": true,
       "engines": {
         "node": ">=0.10.0"
       }
@@ -3036,7 +3035,6 @@
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
       "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-      "peer": true,
       "dependencies": {
         "ansi-styles": "^2.2.1",
         "escape-string-regexp": "^1.0.2",
@@ -3754,7 +3752,6 @@
       "version": "16.7.2",
       "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz",
       "integrity": "sha512-1ZLKZlG2ABfYVBX2d7/JgxOsKJrM5Yu62GvshWu7ZSvhxPomCN4Gas90DS51yYI56JolY0XGhyiRlUhLhIL05Q==",
-      "peer": true,
       "dependencies": {
         "agentkeepalive": "^3.4.1",
         "chalk": "^1.0.0",
@@ -5127,7 +5124,6 @@
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
       "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
-      "peer": true,
       "dependencies": {
         "ansi-regex": "^2.0.0"
       },
@@ -5277,7 +5273,6 @@
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
       "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=",
-      "peer": true,
       "dependencies": {
         "ms": "^2.0.0"
       }
@@ -8514,6 +8509,11 @@
         "lodash._basetostring": "~4.12.0"
       }
     },
+    "node_modules/lodash.clonedeep": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+      "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8="
+    },
     "node_modules/lodash.defaults": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
@@ -9157,6 +9157,18 @@
         }
       }
     },
+    "node_modules/mongoosastic-ts": {
+      "version": "5.0.7",
+      "resolved": "https://registry.npmjs.org/mongoosastic-ts/-/mongoosastic-ts-5.0.7.tgz",
+      "integrity": "sha512-os/wLY7jYt/oLDuZCXaXKU7D2acC89imk+jVa9HCPbDlDMC6A6RMW5x0GlGyXtoMtNTeFBtc5V8vuy1GgEQk1Q==",
+      "dependencies": {
+        "elasticsearch": "16.7.2",
+        "lodash.clonedeep": "4.5.0"
+      },
+      "engines": {
+        "node": ">= 8.0"
+      }
+    },
     "node_modules/mongoose": {
       "version": "5.13.14",
       "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.13.14.tgz",
@@ -10139,6 +10151,8 @@
     },
     "node_modules/npm/node_modules/cli-columns/node_modules/string-width": {
       "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
       "dev": true,
       "inBundle": true,
       "license": "MIT",
@@ -10343,6 +10357,8 @@
     },
     "node_modules/npm/node_modules/debug/node_modules/ms": {
       "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
       "dev": true,
       "inBundle": true,
       "license": "MIT"
@@ -10514,6 +10530,8 @@
     },
     "node_modules/npm/node_modules/gauge/node_modules/string-width": {
       "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+      "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
       "dev": true,
       "inBundle": true,
       "license": "MIT",
@@ -13910,7 +13928,6 @@
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
       "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
-      "peer": true,
       "engines": {
         "node": ">=0.8.0"
       }
@@ -16706,7 +16723,6 @@
       "version": "3.5.2",
       "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz",
       "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==",
-      "peer": true,
       "requires": {
         "humanize-ms": "^1.2.1"
       }
@@ -16753,8 +16769,7 @@
     "ansi-styles": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
-      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
+      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4="
-      "peer": true
     },
     "anymatch": {
       "version": "3.1.2",
@@ -17298,7 +17313,6 @@
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
       "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-      "peer": true,
       "requires": {
         "ansi-styles": "^2.2.1",
         "escape-string-regexp": "^1.0.2",
@@ -17863,7 +17877,6 @@
       "version": "16.7.2",
       "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz",
       "integrity": "sha512-1ZLKZlG2ABfYVBX2d7/JgxOsKJrM5Yu62GvshWu7ZSvhxPomCN4Gas90DS51yYI56JolY0XGhyiRlUhLhIL05Q==",
-      "peer": true,
       "requires": {
         "agentkeepalive": "^3.4.1",
         "chalk": "^1.0.0",
@@ -18942,7 +18955,6 @@
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
       "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
-      "peer": true,
       "requires": {
         "ansi-regex": "^2.0.0"
       }
@@ -19055,7 +19067,6 @@
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
       "integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=",
-      "peer": true,
       "requires": {
         "ms": "^2.0.0"
       }
@@ -21458,6 +21469,11 @@
         "lodash._basetostring": "~4.12.0"
       }
     },
+    "lodash.clonedeep": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
+      "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8="
+    },
     "lodash.defaults": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
@@ -21868,6 +21884,15 @@
         "saslprep": "^1.0.0"
       }
     },
+    "mongoosastic-ts": {
+      "version": "5.0.7",
+      "resolved": "https://registry.npmjs.org/mongoosastic-ts/-/mongoosastic-ts-5.0.7.tgz",
+      "integrity": "sha512-os/wLY7jYt/oLDuZCXaXKU7D2acC89imk+jVa9HCPbDlDMC6A6RMW5x0GlGyXtoMtNTeFBtc5V8vuy1GgEQk1Q==",
+      "requires": {
+        "elasticsearch": "16.7.2",
+        "lodash.clonedeep": "4.5.0"
+      }
+    },
     "mongoose": {
       "version": "5.13.14",
       "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.13.14.tgz",
@@ -22562,6 +22587,8 @@
             },
             "string-width": {
               "version": "4.2.3",
+              "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+              "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
               "bundled": true,
               "dev": true,
               "requires": {
@@ -22706,6 +22733,8 @@
           "dependencies": {
             "ms": {
               "version": "2.1.2",
+              "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+              "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
               "bundled": true,
               "dev": true
             }
@@ -22837,6 +22866,8 @@
             },
             "string-width": {
               "version": "4.2.3",
+              "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+              "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
               "bundled": true,
               "dev": true,
               "requires": {
@@ -25337,8 +25368,7 @@
     "supports-color": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
-      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
+      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
-      "peer": true
     },
     "supports-hyperlinks": {
       "version": "2.2.0",

package.json

@@ -69,6 +69,7 @@
     "moleculer-db-adapter-mongo": "^0.4.7",
     "moleculer-db-adapter-mongoose": "^0.8.9",
     "moleculer-web": "^0.10.4",
+    "mongoosastic-ts": "^5.0.7",
     "mongoose": "^5.12.7",
     "mongoose-elasticsearch-xp": "^5.8.0",
     "mongoose-paginate-v2": "^1.3.18",

services/library.service.ts

@@ -190,10 +190,11 @@ export default class ImportService extends Service {
 							comicMetadata.sourcedMetadata.comicvine.volumeInformation =
 								volumeDetails.results;
 						}
-						return new Promise(async (resolve, reject) => {
+						return await Comic.create(
-							Comic.create(ctx.params, (error, data) => {
+							ctx.params,
+							(error, data) => {
 								if (data) {
-									resolve(data);
+									return data;
 								} else if (error) {
 									console.log("data", data);
 									console.log("error", error);
@@ -204,8 +205,9 @@ export default class ImportService extends Service {
 										error
 									);
 								}
-							});
+							}
-						});
+						);
+						
 					},
 				},
 				applyComicVineMetadata: {
@@ -484,14 +486,15 @@ export default class ImportService extends Service {
 									issues: [
 										{
 											$match: {
-												"sourcedMetadata.comicvine": {
+												"sourcedMetadata.comicvine.volumeInformation":
-													$gt: {},
+													{
-												},
+														$gt: {},
+													},
 											},
 										},
 										{
 											$group: {
-												_id: "$sourcedMetadata.comicvine",
+												_id: "$sourcedMetadata.comicvine.volumeInformation",
 												data: { $push: "$$ROOT._id" },
 											},
 										},

services/libraryqueue.service.ts

@@ -102,8 +102,7 @@ export default class QueueService extends Service {
 								acquisition: {
 									wanted: false,
 								}
-							},
+							}
-							{}
 						);
 						return Promise.resolve({
 							dbImportResult,

services/search.service.ts

@@ -9,7 +9,7 @@ import {
 
 import { DbMixin } from "../mixins/db.mixin";
 import Comic from "../models/comic.model";
-import { flatten, isEmpty, isUndefined } from "lodash";
+import { flatten, isEmpty, isUndefined, map } from "lodash";
 import { eSClient } from "../models/comic.model";
 const s = eSClient.helpers.msearch();
 
@@ -52,11 +52,11 @@ export default class SettingsService extends Service {
 									body: queries,
 								});
 
-								body.responses.forEach((match) => {
+								body.results.forEach((match) => {
 									console.log(match.hits.hits);
 								});
 
-								return body.responses;
+								return body.results;
 							},
 						},
 						issue: {
@@ -68,11 +68,16 @@ export default class SettingsService extends Service {
 										volumeName: string;
 										issueNumber: string;
 									};
+									pagination: {
+										size: number;
+										from: number,
+									};
 								}>
 							) => {
-								console.log(ctx.params.query);
+								console.log(ctx.params);
+								const { query, pagination } = ctx.params;
 								let eSQuery = {};
-								if (isEmpty(ctx.params.query)) {
+								if (isEmpty(query)) {
 									Object.assign(eSQuery, {
 										match_all: {},
 									});
@@ -80,24 +85,23 @@ export default class SettingsService extends Service {
 									Object.assign(eSQuery, {
 										match: {
 											"rawFileDetails.name":
-												ctx.params.query.volumeName,
+												query.volumeName,
 										},
 									});
 								}
-								console.log("eSQuery", eSQuery);
+								console.log(query);
 								const result = await eSClient.search(
 									{
 										index: "comics",
 										body: {
 											query: eSQuery,
 										},
-										size: 50,
+										...pagination,
 									},
 									{ hydrate: true }
 								);
-								console.log(result.body)
+								
-								const { hits } = result.body;
+								return result;
-								return hits;
 							},
 						},
 					},

yarn.lock

@@ -2408,7 +2408,7 @@
   "resolved" "https://registry.npmjs.org/efrt-unpack/-/efrt-unpack-2.2.0.tgz"
   "version" "2.2.0"
 
-"elasticsearch@^12.1.3 || ^13.0.0 || ^14.0.0 || ^16.0.0":
+"elasticsearch@^12.1.3 || ^13.0.0 || ^14.0.0 || ^16.0.0", "elasticsearch@16.7.2":
   "integrity" "sha512-1ZLKZlG2ABfYVBX2d7/JgxOsKJrM5Yu62GvshWu7ZSvhxPomCN4Gas90DS51yYI56JolY0XGhyiRlUhLhIL05Q=="
   "resolved" "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz"
   "version" "16.7.2"
@@ -4602,6 +4602,11 @@
   dependencies:
     "lodash._basetostring" "~4.12.0"
 
+"lodash.clonedeep@4.5.0":
+  "integrity" "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8="
+  "resolved" "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz"
+  "version" "4.5.0"
+
 "lodash.defaults@^4.2.0":
   "integrity" "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw="
   "resolved" "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz"
@@ -5018,6 +5023,14 @@
   optionalDependencies:
     "saslprep" "^1.0.0"
 
+"mongoosastic-ts@^5.0.7":
+  "integrity" "sha512-os/wLY7jYt/oLDuZCXaXKU7D2acC89imk+jVa9HCPbDlDMC6A6RMW5x0GlGyXtoMtNTeFBtc5V8vuy1GgEQk1Q=="
+  "resolved" "https://registry.npmjs.org/mongoosastic-ts/-/mongoosastic-ts-5.0.7.tgz"
+  "version" "5.0.7"
+  dependencies:
+    "elasticsearch" "16.7.2"
+    "lodash.clonedeep" "4.5.0"
+
 "mongoose-elasticsearch-xp@^5.8.0":
   "integrity" "sha512-snb8OdeIuX64vvlyQ/Hwu2vxQDLElixmxkSn3dOeuZ3TepPN2l4PQvfBLHmPT3e86HH93jHeMNU/PJtNOc4FOw=="
   "resolved" "https://registry.npmjs.org/mongoose-elasticsearch-xp/-/mongoose-elasticsearch-xp-5.8.0.tgz"