From 06d0c6aa723e216c3e28226063073e231cb8ac10 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Wed, 2 Jun 2021 09:02:32 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=92=BD=20mongo=20import=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/comic.model.ts | 47 +++++++++++++++-------------------- package-lock.json | 19 ++++++++++++++ package.json | 3 ++- services/api.service.ts | 23 +++++++++++++---- services/import.service.ts | 26 +++++++++++++++++++ utils/comicinfo.xml | 24 ++++++++++++++++++ utils/uncompression.utils.ts | 15 ++++++----- utils/validation.utils.ts | 10 +++++++- utils/xml.utils.ts | 48 ++++++++++++++++++++++++++++++++++++ 9 files changed, 175 insertions(+), 40 deletions(-) create mode 100644 utils/comicinfo.xml create mode 100644 utils/xml.utils.ts diff --git a/models/comic.model.ts b/models/comic.model.ts index 545e872..c4bf51a 100644 --- a/models/comic.model.ts +++ b/models/comic.model.ts @@ -2,10 +2,9 @@ const mongoose = require("mongoose"); const paginate = require("mongoose-paginate"); const ComicSchema = mongoose.Schema({ - name: String, - type: String, - import: { + importStatus: { isImported: Boolean, + tagged: Boolean, matchedResult: { score: String, }, @@ -14,28 +13,21 @@ const ComicSchema = mongoose.Schema({ tags: [], }, - comicStructure: { - cover: { - thumb: String, - medium: String, - large: String, - }, - collection: { - publishDate: String, - type: String, // issue, series, trade paperback - metadata: { - publisher: String, - issueNumber: String, - description: String, - synopsis: String, - team: { - writer: String, - inker: String, - penciler: String, - colorist: String, - }, - }, - }, + comicInfo: { + blackAndWhite: String, + characters: [String], + count: String, + genre: String, + manga: String, + month: String, + number: String, + pageCount: String, + pages: [], + publisher: String, + summary: String, + title: String, + writer: String, + year: String, }, sourcedMetadata: { comicvine: {}, @@ -43,13 +35,14 @@ const ComicSchema = mongoose.Schema({ gcd: {}, }, rawFileDetails: { - fileName: String, + name: String, path: String, - extension: String, + fileSize: Number, }, acquisition: { wanted: Boolean, release: {}, + directconnect: {}, torrent: { sourceApplication: String, magnet: String, diff --git a/package-lock.json b/package-lock.json index 61a6e1a..d6a0c80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7778,6 +7778,11 @@ "sparse-bitfield": "^3.0.3" } }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "saxes": { "version": "3.1.11", "resolved": "https://registry.npmjs.org/saxes/-/saxes-3.1.11.tgz", @@ -9440,6 +9445,20 @@ "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==", "dev": true }, + "xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + }, "xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", diff --git a/package.json b/package.json index a1f0543..43782b1 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,8 @@ "socket.io": "^4.1.1", "socket.io-stream": "^0.9.1", "typescript": "^3.8.3", - "unzipper": "^0.10.11" + "unzipper": "^0.10.11", + "xml2js": "^0.4.23" }, "engines": { "node": ">= 10.x.x" diff --git a/services/api.service.ts b/services/api.service.ts index c67867a..bbee1d6 100644 --- a/services/api.service.ts +++ b/services/api.service.ts @@ -22,6 +22,20 @@ export default class ApiService extends Service { // Access to any actions in all services under "/api" URL "**", ], + cors: { + origin: "*", + methods: [ + "GET", + "OPTIONS", + "POST", + "PUT", + "DELETE", + ], + allowedHeaders: ["*"], + exposedHeaders: [], + credentials: false, + maxAge: 3600, + }, use: [], mergeParams: true, autoAliases: true, @@ -59,7 +73,6 @@ export default class ApiService extends Service { logRequestParams: null, // Logging the response data. Set to any log level to enable it. E.g. "info" logResponseData: null, - // Serve assets from "public" folder assets: { folder: "public", // Options to `server-static` module @@ -105,10 +118,10 @@ export default class ApiService extends Service { folder ); - client.emit("comicBookCoverMetadata", { - data: comicBookCoverMetadata, - status: "Done!", - }); + client.emit( + "comicBookCoverMetadata", + comicBookCoverMetadata + ); }); case "single": diff --git a/services/import.service.ts b/services/import.service.ts index 0349c64..85f65e1 100644 --- a/services/import.service.ts +++ b/services/import.service.ts @@ -3,6 +3,7 @@ import { Context, Service, ServiceBroker, ServiceSchema } from "moleculer"; import { DbMixin } from "../mixins/db.mixin"; import Comic from "../models/comic.model"; import { walkFolder } from "../utils/uncompression.utils"; +import { convertXMLToJSON } from "../utils/xml.utils"; export default class ProductsService extends Service { // @ts-ignore @@ -41,6 +42,31 @@ export default class ProductsService extends Service { ); }, }, + convertXMLToJSON: { + rest: "POST /convertXmlToJson", + params: {}, + async handler(ctx: Context<{}>) { + return convertXMLToJSON("lagos"); + }, + }, + rawImportToDB: { + rest: "POST /rawImportToDB", + params: { payload: "object" }, + async handler(ctx: Context<{ payload: object }>) { + return new Promise((resolve, reject) => { + Comic.create( + ctx.params.payload, + (error, data) => { + if (data) { + resolve(data); + } else if (error) { + reject(new Error(error)); + } + } + ); + }); + }, + }, }, methods: {}, }, diff --git a/utils/comicinfo.xml b/utils/comicinfo.xml new file mode 100644 index 0000000..c2ca4ff --- /dev/null +++ b/utils/comicinfo.xml @@ -0,0 +1,24 @@ + + + Title of the Book + A description of the book + 1 + 3 + 2010 + 4 + Author name + self + educational + No + No + Superman + 5 + + + + + + + + diff --git a/utils/uncompression.utils.ts b/utils/uncompression.utils.ts index 6e19399..84b9a6b 100644 --- a/utils/uncompression.utils.ts +++ b/utils/uncompression.utils.ts @@ -205,12 +205,15 @@ export const unzip = async ( } if (fileName !== "" && entry.type !== "Directory") { logger.info(`Attempting to write ${fileName}`); - entry.pipe(createWriteStream(paths.targetPath + "/" + fileName)); - extractedFiles.push({ - name: fileName, - fileSize: size, - path: paths.targetPath, - }); + entry + .pipe(createWriteStream(paths.targetPath + "/" + fileName)) + .on("finish", () => { + extractedFiles.push({ + name: fileName, + fileSize: size, + path: paths.targetPath, + }); + }); } entry.autodrain(); } diff --git a/utils/validation.utils.ts b/utils/validation.utils.ts index 0df5894..fcaeeed 100644 --- a/utils/validation.utils.ts +++ b/utils/validation.utils.ts @@ -6,11 +6,11 @@ import { IFolderData, } from "../interfaces/folder.interface"; const Validator = require("fastest-validator"); +import { logger } from "./logger.utils"; export const validateComicBookMetadata = ( comicBookMetadataObject: IExtractedComicBookCoverFile ): boolean => { - console.log(comicBookMetadataObject); const validator = new Validator(); const sch = { name: { type: "string" }, @@ -18,5 +18,13 @@ export const validateComicBookMetadata = ( path: { type: "string" }, }; const check = validator.compile(sch); + if (check(comicBookMetadataObject)) { + logger.info(`Valid comic book metadata: ${comicBookMetadataObject}`); + } else { + logger.error( + `Comic book metadata was invalid: + ${comicBookMetadataObject}` + ); + } return check(comicBookMetadataObject); }; diff --git a/utils/xml.utils.ts b/utils/xml.utils.ts new file mode 100644 index 0000000..647f55b --- /dev/null +++ b/utils/xml.utils.ts @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2021 Rishi Ghan + * + The MIT License (MIT) + +Copyright (c) 2021 Rishi Ghan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + */ + +import xml2js from "xml2js"; +import fs from "fs"; +import { logger } from "../utils/logger.utils"; + +export const convertXMLToJSON = (xmlPayload) => { + const parser = new xml2js.Parser({ + mergeAttrs: true, + trim: true, + normalizeTags: true, + }); + const xml = fs.readFileSync(__dirname + "/comicinfo.xml", "utf8"); + return parser + .parseStringPromise(xml) + .then((result) => { + return result; + }) + .catch((error) => { + logger.error(error); + }); +};