diff --git a/models/comic.model.ts b/models/comic.model.ts index 3c16297..4a1fa49 100644 --- a/models/comic.model.ts +++ b/models/comic.model.ts @@ -12,23 +12,23 @@ const ComicSchema = mongoose.Schema({ userAddedMetadata: { tags: [], }, - comicInfo: { - blackAndWhite: String, - characters: [String], - count: String, - genre: String, - manga: String, - month: String, - number: String, - pageCount: String, - pages: [], - publisher: String, - summary: String, - title: String, - writer: String, - year: String, - }, sourcedMetadata: { + comicInfo: { + blackAndWhite: String, + characters: [String], + count: String, + genre: String, + manga: String, + month: String, + number: String, + pageCount: String, + pages: [], + publisher: String, + summary: String, + title: String, + writer: String, + year: String, + }, comicvine: {}, shortboxed: {}, gcd: {}, @@ -37,7 +37,11 @@ const ComicSchema = mongoose.Schema({ name: String, path: String, fileSize: Number, + extension: String, containedIn: String, + calibreMetadata :{ + coverWriteResult: String, + } }, acquisition: { wanted: Boolean, @@ -53,7 +57,7 @@ const ComicSchema = mongoose.Schema({ sourceApplication: String, }, }, -}); +}, { timestamps: true}); ComicSchema.plugin(paginate); const Comic = mongoose.model("Comic", ComicSchema); diff --git a/package-lock.json b/package-lock.json index 586b459..306a413 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,7 @@ "jest": "^25.1.0", "jest-cli": "^25.1.0", "moleculer-repl": "^0.6.2", - "threetwo-ui-typings": "^1.0.1-0", + "threetwo-ui-typings": "^1.0.2-0", "ts-jest": "^25.3.0", "ts-node": "^8.8.1" }, @@ -11303,9 +11303,9 @@ "dev": true }, "node_modules/threetwo-ui-typings": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.1.tgz", - "integrity": "sha512-g5FWa069AT1WfUHHEVFOQ1q6cfK+9UOzewfKblheuDBSsGN/e89MJMEVfuInBaHgxHM32/P+mNUFBqnBoeRSLQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.2.tgz", + "integrity": "sha512-sK5cb/fApFKseQNgcnGmAnPNxDDXT+dQ/Blei1N4q0mduO3kZfJTnlMYaXjO4FDXLP+jkiwsv0bf7PXm0DgF7g==", "dev": true, "dependencies": { "typescript": "^4.3.2" @@ -21081,9 +21081,9 @@ "dev": true }, "threetwo-ui-typings": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.1.tgz", - "integrity": "sha512-g5FWa069AT1WfUHHEVFOQ1q6cfK+9UOzewfKblheuDBSsGN/e89MJMEVfuInBaHgxHM32/P+mNUFBqnBoeRSLQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.2.tgz", + "integrity": "sha512-sK5cb/fApFKseQNgcnGmAnPNxDDXT+dQ/Blei1N4q0mduO3kZfJTnlMYaXjO4FDXLP+jkiwsv0bf7PXm0DgF7g==", "dev": true, "requires": { "typescript": "^4.3.2" diff --git a/package.json b/package.json index ade7349..274f175 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "jest": "^25.1.0", "jest-cli": "^25.1.0", "moleculer-repl": "^0.6.2", - "threetwo-ui-typings": "^1.0.1-0", + "threetwo-ui-typings": "^1.0.2-0", "ts-jest": "^25.3.0", "ts-node": "^8.8.1" }, diff --git a/services/api.service.ts b/services/api.service.ts index ca5649a..bac4c24 100644 --- a/services/api.service.ts +++ b/services/api.service.ts @@ -14,7 +14,6 @@ export default class ApiService extends Service { // More info about settings: https://moleculer.services/docs/0.14/moleculer-web.html settings: { port: process.env.PORT || 3000, - routes: [ { path: "/api", @@ -39,7 +38,6 @@ export default class ApiService extends Service { use: [], mergeParams: true, autoAliases: true, - aliases: {}, // Calling options. More info: https://moleculer.services/docs/0.14/moleculer-web.html#Calling-options @@ -55,11 +53,7 @@ export default class ApiService extends Service { limit: "1MB", }, }, - - // Mapping policy setting. More info: https://moleculer.services/docs/0.14/moleculer-web.html#Mapping-policy mappingPolicy: "all", // Available values: "all", "restrict" - - // Enable/disable logging logging: true, }, { @@ -71,15 +65,11 @@ export default class ApiService extends Service { use: [ApiGateway.serveStatic("comics")], }, ], - // Do not log client side errors (does not log an error response when the error.code is 400<=X<500) log4XXResponses: false, - // Logging the request parameters. Set to any log level to enable it. E.g. "info" logRequestParams: null, - // Logging the response data. Set to any log level to enable it. E.g. "info" logResponseData: null, assets: { folder: "public", - // Options to `server-static` module options: {}, }, }, @@ -104,7 +94,7 @@ export default class ApiService extends Service { this.logger.info("Client connected via websocket!"); client.on( - "importComicsInDB", + "importComicsToDB", async ({ action, params, opts }, done) => { this.logger.info( "Received request from client! Action:", diff --git a/services/imagetransformation.service.ts b/services/imagetransformation.service.ts index 830805e..e48fa35 100644 --- a/services/imagetransformation.service.ts +++ b/services/imagetransformation.service.ts @@ -8,12 +8,7 @@ import { } from "moleculer"; import { resizeImage, - calculateLevenshteinDistance, } from "../utils/imagetransformation.utils"; -import https from "https"; -import fs from "fs"; -import path from "path"; - export default class ProductsService extends Service { // @ts-ignore public constructor( @@ -58,44 +53,6 @@ export default class ProductsService extends Service { return { resizeOperationStatus: resizeResult }; }, }, - calculateLevenshteinDistance: { - rest: "POST /calculateLevenshteinDistance", - params: {}, - async handler( - ctx: Context<{ - imagePath: string; - imagePath2: string; - options: { - match_id: string, - }; - }> - ) { - const fileName = ctx.params.options.match_id + "_" + path.basename( - ctx.params.imagePath - ); - return new Promise((resolve, reject) => { - https.get(ctx.params.imagePath2, (response) => { - const fileStream = response.pipe( - fs.createWriteStream( - `./userdata/temporary/${fileName}` - ) - ); - fileStream.on("finish", async () => { - const levenshteinDistance = await calculateLevenshteinDistance( - ctx.params.imagePath, - path.resolve( - `./userdata/temporary/${fileName}` - ) - ); - resolve(levenshteinDistance); - }); - - }).end(); - }); - - - }, - }, }, methods: {}, }, diff --git a/services/import.service.ts b/services/import.service.ts index fcc8e59..4baa4a0 100644 --- a/services/import.service.ts +++ b/services/import.service.ts @@ -1,4 +1,5 @@ "use strict"; +import { isNil } from "lodash"; import { Context, Service, @@ -10,6 +11,9 @@ import { DbMixin } from "../mixins/db.mixin"; import Comic from "../models/comic.model"; import { walkFolder } from "../utils/file.utils"; import { convertXMLToJSON } from "../utils/xml.utils"; +import https from "https"; +const ObjectId = require("mongoose").Types.ObjectId; + export default class ProductsService extends Service { public constructor( @@ -74,8 +78,55 @@ export default class ProductsService extends Service { }); }, }, - getRecentlyImportedComicBooks: { - rest: "POST /getRecentlyImportedComicBooks", + applyComicVineMetadata: { + rest: "POST /applyComicVineMetadata", + params: {}, + async handler(ctx: Context<{ match: { volume: { api_detail_url: string}, volumeInformation: object}, comicObjectId: string }>) { + // 1. find mongo object by id + // 2. import payload into sourcedMetadata.comicvine + const comicObjectId = new ObjectId(ctx.params.comicObjectId); + const matchedResult = ctx.params.match; + console.log(matchedResult.volume.api_detail_url); + let volumeDetailsPromise; + if (!isNil(matchedResult.volume)) { + volumeDetailsPromise = new Promise((resolve, reject) => { + return https.get(`${matchedResult.volume.api_detail_url}?api_key=a5fa0663683df8145a85d694b5da4b87e1c92c69&format=json&limit=1&offset=0&field_list=id,name,deck,image,first_issue,last_issue,publisher,count_of_issues`, (resp) => { + + let data = ''; + resp.on('data', (chunk) => { + data += chunk; + }); + + resp.on('end', () => { + const volumeInformation = JSON.parse(data); + resolve(volumeInformation); + }); + + }).on("error", (err) => { + console.log("Error: " + err.message); + reject(err); + }); + }); + } + return new Promise(async (resolve, reject) => { + const volumeDetails = await volumeDetailsPromise; + matchedResult.volumeInformation = volumeDetails.results; + Comic.findByIdAndUpdate(comicObjectId, { sourcedMetadata: { comicvine: matchedResult } }, { new: true }, (err, result) => { + if (err) { + console.log(err); + reject(err); + } else { + // 3. Fetch and append volume information + resolve(result); + } + }); + }); + }, + + }, + + getComicBooks: { + rest: "POST /getComicBooks", params: {}, async handler( ctx: Context<{ paginationOptions: object }> diff --git a/utils/imagetransformation.utils.ts b/utils/imagetransformation.utils.ts index 68b17cb..e3e7bcf 100644 --- a/utils/imagetransformation.utils.ts +++ b/utils/imagetransformation.utils.ts @@ -1,9 +1,6 @@ const sharp = require("sharp"); -const imghash = require("imghash"); -const leven = require("leven"); -import path from "path"; -import { isNull, reject } from "lodash"; import { logger } from "./logger.utils"; +import { ISharpResizedImageStats } from "threetwo-ui-typings"; export const extractMetadataFromImage = async ( imageFilePath: string @@ -21,35 +18,20 @@ export const resizeImage = async ( outputPath: string, newWidth: number, newHeight?: number -): Promise => { - return await sharp(imageFile) - .resize(newWidth) - .toFile(`${outputPath}`, (err, info) => { - if (err) { - logger.error("Failed to resize image:"); - logger.error(err); - return err; - } +): Promise => { + return new Promise((resolve, reject) => { + sharp(imageFile) + .resize(newWidth) + .toFile(`${outputPath}`, (err, info) => { + if (err) { + logger.error("Failed to resize image:"); + logger.error(err); + reject(err); + } - logger.info("Image file resized with the following parameters:"); - logger.info(info); - return info; - }); -}; - -export const calculateLevenshteinDistance = async ( - imagePath1: string, - imagePath2: string -): Promise> => { - console.log("AGANTUK", imagePath1) - const hash1 = await imghash.hash(imagePath1); - const hash2 = await imghash.hash(imagePath2); - console.log("HASHISH", hash1) - if (!isNull(hash1) && !isNull(hash2)) { - return new Promise((resolve, reject) => { - resolve({ levenshteinDistance: leven(hash1, hash2) }); - }); - } else { - reject("Can't calculate the Levenshtein distance") - } + logger.info("Image file resized with the following parameters:"); + logger.info(info); + resolve(info); + }); + }); }; diff --git a/utils/uncompression.utils.ts b/utils/uncompression.utils.ts index 02156d1..9a63dc7 100644 --- a/utils/uncompression.utils.ts +++ b/utils/uncompression.utils.ts @@ -31,7 +31,7 @@ SOFTWARE. * Initial: 2021/05/04 Rishi Ghan */ -import { createReadStream, createWriteStream, readFileSync } from "fs"; +import { createReadStream, createWriteStream, readFileSync, stat } from "fs"; const fse = require("fs-extra"); import path from "path"; import { each, isEmpty, map, flatten } from "lodash"; @@ -42,6 +42,7 @@ import { IExtractedComicBookCoverFile, IExtractionOptions, IFolderData, + ISharpResizedImageStats, } from "threetwo-ui-typings"; import { logger } from "./logger.utils"; import { validateComicBookMetadata } from "../utils/validation.utils"; @@ -84,6 +85,7 @@ export const extractCoverFromFile = async ( } catch (error) { logger.error(`${error}: Couldn't create directory.`); } + // extract the cover let result: string; const targetCoverImageFilePath = path.resolve(constructedPaths.targetPath + "/" + walkedFolder.name + "_cover.jpg") result = await calibre.run( @@ -95,15 +97,17 @@ export const extractCoverFromFile = async ( ); // create renditions const renditionPath = constructedPaths.targetPath + "/" + walkedFolder.name + "_200px.jpg"; - await resizeImage(targetCoverImageFilePath, path.resolve(renditionPath), 200); - + const stats:ISharpResizedImageStats = await resizeImage(targetCoverImageFilePath, path.resolve(renditionPath), 200); + resolve({ name: walkedFolder.name, - path: constructedPaths.targetPath + "/" + walkedFolder.name + "_200px.jpg", //renditionPath - fileSize: 0, + path: renditionPath, + fileSize: stats.size, + extension: path.extname(constructedPaths.inputFilePath), containedIn: walkedFolder.containedIn, - //originalPath: - //extension: + calibreMetadata: { + coverWriteResult: result, + } }); } catch (error) { console.log(error); @@ -200,6 +204,7 @@ export const unrar = async ( mode: 0o2775, }; try { + // read the file into a buffer const fileBuffer = await readFile( paths.inputFilePath ).catch((err) => console.error("Failed to read file", err)); @@ -238,8 +243,12 @@ export const unrar = async ( resolve({ name: `${extractedFiles[0].fileHeader.name}`, path: paths.targetPath, + extension: path.extname(extractedFiles[0].fileHeader.name), fileSize: extractedFiles[0].fileHeader.packSize, containedIn: walkedFolder.containedIn, + calibreMetadata: { + coverWriteResult: "", + } }); } catch (error) { logger.error(`${error}: Couldn't write file.`); diff --git a/yarn.lock b/yarn.lock index 6df0374..164a97f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5906,10 +5906,10 @@ "resolved" "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz" "version" "0.2.0" -"threetwo-ui-typings@^1.0.1-0": - "integrity" "sha512-g5FWa069AT1WfUHHEVFOQ1q6cfK+9UOzewfKblheuDBSsGN/e89MJMEVfuInBaHgxHM32/P+mNUFBqnBoeRSLQ==" - "resolved" "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.1.tgz" - "version" "1.0.1" +"threetwo-ui-typings@^1.0.2-0": + "integrity" "sha512-sK5cb/fApFKseQNgcnGmAnPNxDDXT+dQ/Blei1N4q0mduO3kZfJTnlMYaXjO4FDXLP+jkiwsv0bf7PXm0DgF7g==" + "resolved" "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.2.tgz" + "version" "1.0.2" dependencies: "typescript" "^4.3.2"