frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

🔎 Elasticsearch endpoints for you know, search

Browse Source
This commit is contained in:
Rishi Ghan
2022-03-07 03:04:51 -08:00
parent f264004c30
commit 3237c93b94
8 changed files with 144 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
models/comic.model.ts
View File

@@ -2,6 +2,13 @@ const mongoose = require("mongoose");
var mexp = require("mongoose-elasticsearch-xp").v7;
const paginate = require("mongoose-paginate-v2");
const { Client } = require("@elastic/elasticsearch");
import ComicVineMetadataSchema from "./comicvine.metadata.model";
import { mongoosastic } from "mongoosastic-ts";
import {
MongoosasticDocument,
MongoosasticModel,
MongoosasticPluginOpts,
} from "mongoosastic-ts/dist/types";
export const eSClient = new Client({
node: "http://localhost:9200",
@@ -11,6 +18,23 @@ export const eSClient = new Client({
},
});
const RawFileDetailsSchema = mongoose.Schema({
_id: false,
name: String,
filePath: String,
fileSize: Number,
extension: String,
containedIn: String,
pageCount: Number,
cover: {
filePath: String,
stats: Object,
},
calibreMetadata: {
coverWriteResult: String,
},
});
const ComicSchema = mongoose.Schema(
{
importStatus: {
@@ -25,36 +49,25 @@ const ComicSchema = mongoose.Schema(
},
sourcedMetadata: {
comicInfo: { type: mongoose.Schema.Types.Mixed, default: {} },
comicvine: { type: mongoose.Schema.Types.Mixed, default: {} },
comicvine: {
type: ComicVineMetadataSchema,
es_indexed: true,
default: {},
},
shortboxed: {},
gcd: {},
},
rawFileDetails: {
name: { type: String, es_indexed: true },
filePath: String,
fileSize: Number,
extension: String,
containedIn: String,
pageCount: Number,
cover: {
filePath: String,
stats: Object,
},
calibreMetadata: {
coverWriteResult: String,
},
},
rawFileDetails: { type: RawFileDetailsSchema, es_indexed: true },
inferredMetadata: {
issue: {
name: String,
name: { type: String, es_indexed: true },
number: {
type: Number,
es_indexed: true,
required: false,
default: 0,
},
year: String,
subtitle: String,
subtitle: { type: String, es_indexed: true },
},
},
acquisition: {
@@ -75,9 +88,22 @@ const ComicSchema = mongoose.Schema(
{ timestamps: true, minimize: false }
);
ComicSchema.plugin(mexp, {
client: eSClient,
ComicSchema.plugin(mongoosastic, {
esClient: eSClient,
});
ComicSchema.plugin(paginate);
const Comic = mongoose.model("Comic", ComicSchema);
const Comic = mongoose.model("Comic", ComicSchema),
stream = Comic.synchronize();
let count = 0;
stream.on("data", function (err, doc) {
count++;
});
stream.on("close", function () {
console.log("indexed " + count + " documents!");
});
stream.on("error", function (err) {
console.log(err);
});
export default Comic;

10
models/comicvine.metadata.model.ts
View File

@@ -1,5 +1,6 @@
const mongoose = require("mongoose");
const Things = mongoose.Schema({
_id: false,
api_detail_url: String,
id: Number,
name: String,
@@ -7,13 +8,15 @@ const Things = mongoose.Schema({
count: String,
});
const Issue = mongoose.Schema({
_id: false,
api_detail_url: String,
id: Number,
name: String,
issue_number: String,
});
const VolumeInformation = mongoose.Schema({
aliases: Array,
_id: false,
aliases: [],
api_detail_url: String,
characters: [Things],
concepts: [Things],
@@ -60,7 +63,8 @@ const VolumeInformation = mongoose.Schema({
start_year: String,
});
const ComicVineMetadata = mongoose.Schema({
const ComicVineMetadataSchema = mongoose.Schema({
_id: false,
aliases: [String],
api_detail_url: String,
cover_date: String,
@@ -75,4 +79,4 @@ const ComicVineMetadata = mongoose.Schema({
volumeInformation: VolumeInformation,
});
export default ComicVineMetadata;
export default ComicVineMetadataSchema;

62
package-lock.json generated
View File

@@ -38,6 +38,7 @@
"moleculer-db-adapter-mongo": "^0.4.7",
"moleculer-db-adapter-mongoose": "^0.8.9",
"moleculer-web": "^0.10.4",
"mongoosastic-ts": "^5.0.7",
"mongoose": "^5.12.7",
"mongoose-elasticsearch-xp": "^5.8.0",
"mongoose-paginate-v2": "^1.3.18",
@@ -2250,7 +2251,6 @@
"version": "3.5.2",
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz",
"integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==",
"peer": true,
"dependencies": {
"humanize-ms": "^1.2.1"
},
@@ -2321,7 +2321,6 @@
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
"integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -3036,7 +3035,6 @@
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
"integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
"peer": true,
"dependencies": {
"ansi-styles": "^2.2.1",
"escape-string-regexp": "^1.0.2",
@@ -3754,7 +3752,6 @@
"version": "16.7.2",
"resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz",
"integrity": "sha512-1ZLKZlG2ABfYVBX2d7/JgxOsKJrM5Yu62GvshWu7ZSvhxPomCN4Gas90DS51yYI56JolY0XGhyiRlUhLhIL05Q==",
"peer": true,
"dependencies": {
"agentkeepalive": "^3.4.1",
"chalk": "^1.0.0",
@@ -5127,7 +5124,6 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
"integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
"peer": true,
"dependencies": {
"ansi-regex": "^2.0.0"
},
@@ -5277,7 +5273,6 @@
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
"integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=",
"peer": true,
"dependencies": {
"ms": "^2.0.0"
}
@@ -8514,6 +8509,11 @@
"lodash._basetostring": "~4.12.0"
}
},
"node_modules/lodash.clonedeep": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
"integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8="
},
"node_modules/lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
@@ -9157,6 +9157,18 @@
}
}
},
"node_modules/mongoosastic-ts": {
"version": "5.0.7",
"resolved": "https://registry.npmjs.org/mongoosastic-ts/-/mongoosastic-ts-5.0.7.tgz",
"integrity": "sha512-os/wLY7jYt/oLDuZCXaXKU7D2acC89imk+jVa9HCPbDlDMC6A6RMW5x0GlGyXtoMtNTeFBtc5V8vuy1GgEQk1Q==",
"dependencies": {
"elasticsearch": "16.7.2",
"lodash.clonedeep": "4.5.0"
},
"engines": {
"node": ">= 8.0"
}
},
"node_modules/mongoose": {
"version": "5.13.14",
"resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.13.14.tgz",
@@ -10139,6 +10151,8 @@
},
"node_modules/npm/node_modules/cli-columns/node_modules/string-width": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
"dev": true,
"inBundle": true,
"license": "MIT",
@@ -10343,6 +10357,8 @@
},
"node_modules/npm/node_modules/debug/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
"dev": true,
"inBundle": true,
"license": "MIT"
@@ -10514,6 +10530,8 @@
},
"node_modules/npm/node_modules/gauge/node_modules/string-width": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
"dev": true,
"inBundle": true,
"license": "MIT",
@@ -13910,7 +13928,6 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
"integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
"peer": true,
"engines": {
"node": ">=0.8.0"
}
@@ -16706,7 +16723,6 @@
"version": "3.5.2",
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz",
"integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==",
"peer": true,
"requires": {
"humanize-ms": "^1.2.1"
}
@@ -16753,8 +16769,7 @@
"ansi-styles": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
"integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
"peer": true
"integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4="
},
"anymatch": {
"version": "3.1.2",
@@ -17298,7 +17313,6 @@
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
"integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
"peer": true,
"requires": {
"ansi-styles": "^2.2.1",
"escape-string-regexp": "^1.0.2",
@@ -17863,7 +17877,6 @@
"version": "16.7.2",
"resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz",
"integrity": "sha512-1ZLKZlG2ABfYVBX2d7/JgxOsKJrM5Yu62GvshWu7ZSvhxPomCN4Gas90DS51yYI56JolY0XGhyiRlUhLhIL05Q==",
"peer": true,
"requires": {
"agentkeepalive": "^3.4.1",
"chalk": "^1.0.0",
@@ -18942,7 +18955,6 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
"integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
"peer": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@@ -19055,7 +19067,6 @@
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
"integrity": "sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0=",
"peer": true,
"requires": {
"ms": "^2.0.0"
}
@@ -21458,6 +21469,11 @@
"lodash._basetostring": "~4.12.0"
}
},
"lodash.clonedeep": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
"integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8="
},
"lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
@@ -21868,6 +21884,15 @@
"saslprep": "^1.0.0"
}
},
"mongoosastic-ts": {
"version": "5.0.7",
"resolved": "https://registry.npmjs.org/mongoosastic-ts/-/mongoosastic-ts-5.0.7.tgz",
"integrity": "sha512-os/wLY7jYt/oLDuZCXaXKU7D2acC89imk+jVa9HCPbDlDMC6A6RMW5x0GlGyXtoMtNTeFBtc5V8vuy1GgEQk1Q==",
"requires": {
"elasticsearch": "16.7.2",
"lodash.clonedeep": "4.5.0"
}
},
"mongoose": {
"version": "5.13.14",
"resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.13.14.tgz",
@@ -22562,6 +22587,8 @@
},
"string-width": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
"bundled": true,
"dev": true,
"requires": {
@@ -22706,6 +22733,8 @@
"dependencies": {
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
"bundled": true,
"dev": true
}
@@ -22837,6 +22866,8 @@
},
"string-width": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
"bundled": true,
"dev": true,
"requires": {
@@ -25337,8 +25368,7 @@
"supports-color": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
"integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
"peer": true
"integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
},
"supports-hyperlinks": {
"version": "2.2.0",

1
package.json
View File

@@ -69,6 +69,7 @@
"moleculer-db-adapter-mongo": "^0.4.7",
"moleculer-db-adapter-mongoose": "^0.8.9",
"moleculer-web": "^0.10.4",
"mongoosastic-ts": "^5.0.7",
"mongoose": "^5.12.7",
"mongoose-elasticsearch-xp": "^5.8.0",
"mongoose-paginate-v2": "^1.3.18",

21
services/library.service.ts
View File

@@ -190,10 +190,11 @@ export default class ImportService extends Service {
comicMetadata.sourcedMetadata.comicvine.volumeInformation =
volumeDetails.results;
}
return new Promise(async (resolve, reject) => {
Comic.create(ctx.params, (error, data) => {
return await Comic.create(
ctx.params,
(error, data) => {
if (data) {
resolve(data);
return data;
} else if (error) {
console.log("data", data);
console.log("error", error);
@@ -204,8 +205,9 @@ export default class ImportService extends Service {
error
);
}
});
});
}
);
},
},
applyComicVineMetadata: {
@@ -484,14 +486,15 @@ export default class ImportService extends Service {
issues: [
{
$match: {
"sourcedMetadata.comicvine": {
$gt: {},
},
"sourcedMetadata.comicvine.volumeInformation":
{
$gt: {},
},
},
},
{
$group: {
_id: "$sourcedMetadata.comicvine",
_id: "$sourcedMetadata.comicvine.volumeInformation",
data: { $push: "$$ROOT._id" },
},
},

3
services/libraryqueue.service.ts
View File

@@ -102,8 +102,7 @@ export default class QueueService extends Service {
acquisition: {
wanted: false,
}
},
{}
}
);
return Promise.resolve({
dbImportResult,

26
services/search.service.ts
View File

@@ -9,7 +9,7 @@ import {
import { DbMixin } from "../mixins/db.mixin";
import Comic from "../models/comic.model";
import { flatten, isEmpty, isUndefined } from "lodash";
import { flatten, isEmpty, isUndefined, map } from "lodash";
import { eSClient } from "../models/comic.model";
const s = eSClient.helpers.msearch();
@@ -52,11 +52,11 @@ export default class SettingsService extends Service {
body: queries,
});
body.responses.forEach((match) => {
body.results.forEach((match) => {
console.log(match.hits.hits);
});
return body.responses;
return body.results;
},
},
issue: {
@@ -68,11 +68,16 @@ export default class SettingsService extends Service {
volumeName: string;
issueNumber: string;
};
pagination: {
size: number;
from: number,
};
}>
) => {
console.log(ctx.params.query);
console.log(ctx.params);
const { query, pagination } = ctx.params;
let eSQuery = {};
if (isEmpty(ctx.params.query)) {
if (isEmpty(query)) {
Object.assign(eSQuery, {
match_all: {},
});
@@ -80,24 +85,23 @@ export default class SettingsService extends Service {
Object.assign(eSQuery, {
match: {
"rawFileDetails.name":
ctx.params.query.volumeName,
query.volumeName,
},
});
}
console.log("eSQuery", eSQuery);
console.log(query);
const result = await eSClient.search(
{
index: "comics",
body: {
query: eSQuery,
},
size: 50,
...pagination,
},
{ hydrate: true }
);
console.log(result.body)
const { hits } = result.body;
return hits;
return result;
},
},
},

15
yarn.lock
View File

@@ -2408,7 +2408,7 @@
"resolved" "https://registry.npmjs.org/efrt-unpack/-/efrt-unpack-2.2.0.tgz"
"version" "2.2.0"
"elasticsearch@^12.1.3 || ^13.0.0 || ^14.0.0 || ^16.0.0":
"elasticsearch@^12.1.3 || ^13.0.0 || ^14.0.0 || ^16.0.0", "elasticsearch@16.7.2":
"integrity" "sha512-1ZLKZlG2ABfYVBX2d7/JgxOsKJrM5Yu62GvshWu7ZSvhxPomCN4Gas90DS51yYI56JolY0XGhyiRlUhLhIL05Q=="
"resolved" "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.2.tgz"
"version" "16.7.2"
@@ -4602,6 +4602,11 @@
dependencies:
"lodash._basetostring" "~4.12.0"
"lodash.clonedeep@4.5.0":
"integrity" "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8="
"resolved" "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz"
"version" "4.5.0"
"lodash.defaults@^4.2.0":
"integrity" "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw="
"resolved" "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz"
@@ -5018,6 +5023,14 @@
optionalDependencies:
"saslprep" "^1.0.0"
"mongoosastic-ts@^5.0.7":
"integrity" "sha512-os/wLY7jYt/oLDuZCXaXKU7D2acC89imk+jVa9HCPbDlDMC6A6RMW5x0GlGyXtoMtNTeFBtc5V8vuy1GgEQk1Q=="
"resolved" "https://registry.npmjs.org/mongoosastic-ts/-/mongoosastic-ts-5.0.7.tgz"
"version" "5.0.7"
dependencies:
"elasticsearch" "16.7.2"
"lodash.clonedeep" "4.5.0"
"mongoose-elasticsearch-xp@^5.8.0":
"integrity" "sha512-snb8OdeIuX64vvlyQ/Hwu2vxQDLElixmxkSn3dOeuZ3TepPN2l4PQvfBLHmPT3e86HH93jHeMNU/PJtNOc4FOw=="
"resolved" "https://registry.npmjs.org/mongoose-elasticsearch-xp/-/mongoose-elasticsearch-xp-5.8.0.tgz"