frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/rishighan/threetwo-import-service

Browse Source
This commit is contained in:
Rishi Ghan
2021-08-10 11:42:39 -07:00
parent 4c28dcdf72 affac51d62
commit 2663fc822a
9 changed files with 119 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
models/comic.model.ts
View File

@@ -12,23 +12,23 @@ const ComicSchema = mongoose.Schema({
userAddedMetadata: {
tags: [],
},
comicInfo: {
blackAndWhite: String,
characters: [String],
count: String,
genre: String,
manga: String,
month: String,
number: String,
pageCount: String,
pages: [],
publisher: String,
summary: String,
title: String,
writer: String,
year: String,
},
sourcedMetadata: {
comicInfo: {
blackAndWhite: String,
characters: [String],
count: String,
genre: String,
manga: String,
month: String,
number: String,
pageCount: String,
pages: [],
publisher: String,
summary: String,
title: String,
writer: String,
year: String,
},
comicvine: {},
shortboxed: {},
gcd: {},
@@ -37,7 +37,11 @@ const ComicSchema = mongoose.Schema({
name: String,
path: String,
fileSize: Number,
extension: String,
containedIn: String,
calibreMetadata :{
coverWriteResult: String,
}
},
acquisition: {
wanted: Boolean,
@@ -53,7 +57,7 @@ const ComicSchema = mongoose.Schema({
sourceApplication: String,
},
},
});
}, { timestamps: true});
ComicSchema.plugin(paginate);
const Comic = mongoose.model("Comic", ComicSchema);

14
package-lock.json generated
View File

@@ -50,7 +50,7 @@
"jest": "^25.1.0",
"jest-cli": "^25.1.0",
"moleculer-repl": "^0.6.2",
"threetwo-ui-typings": "^1.0.1-0",
"threetwo-ui-typings": "^1.0.2-0",
"ts-jest": "^25.3.0",
"ts-node": "^8.8.1"
},
@@ -11303,9 +11303,9 @@
"dev": true
},
"node_modules/threetwo-ui-typings": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.1.tgz",
"integrity": "sha512-g5FWa069AT1WfUHHEVFOQ1q6cfK+9UOzewfKblheuDBSsGN/e89MJMEVfuInBaHgxHM32/P+mNUFBqnBoeRSLQ==",
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.2.tgz",
"integrity": "sha512-sK5cb/fApFKseQNgcnGmAnPNxDDXT+dQ/Blei1N4q0mduO3kZfJTnlMYaXjO4FDXLP+jkiwsv0bf7PXm0DgF7g==",
"dev": true,
"dependencies": {
"typescript": "^4.3.2"
@@ -21081,9 +21081,9 @@
"dev": true
},
"threetwo-ui-typings": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.1.tgz",
"integrity": "sha512-g5FWa069AT1WfUHHEVFOQ1q6cfK+9UOzewfKblheuDBSsGN/e89MJMEVfuInBaHgxHM32/P+mNUFBqnBoeRSLQ==",
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.2.tgz",
"integrity": "sha512-sK5cb/fApFKseQNgcnGmAnPNxDDXT+dQ/Blei1N4q0mduO3kZfJTnlMYaXjO4FDXLP+jkiwsv0bf7PXm0DgF7g==",
"dev": true,
"requires": {
"typescript": "^4.3.2"

2
package.json
View File

@@ -27,7 +27,7 @@
"jest": "^25.1.0",
"jest-cli": "^25.1.0",
"moleculer-repl": "^0.6.2",
"threetwo-ui-typings": "^1.0.1-0",
"threetwo-ui-typings": "^1.0.2-0",
"ts-jest": "^25.3.0",
"ts-node": "^8.8.1"
},

12
services/api.service.ts
View File

@@ -14,7 +14,6 @@ export default class ApiService extends Service {
// More info about settings: https://moleculer.services/docs/0.14/moleculer-web.html
settings: {
port: process.env.PORT || 3000,
routes: [
{
path: "/api",
@@ -39,7 +38,6 @@ export default class ApiService extends Service {
use: [],
mergeParams: true,
autoAliases: true,
aliases: {},
// Calling options. More info: https://moleculer.services/docs/0.14/moleculer-web.html#Calling-options
@@ -55,11 +53,7 @@ export default class ApiService extends Service {
limit: "1MB",
},
},
// Mapping policy setting. More info: https://moleculer.services/docs/0.14/moleculer-web.html#Mapping-policy
mappingPolicy: "all", // Available values: "all", "restrict"
// Enable/disable logging
logging: true,
},
{
@@ -71,15 +65,11 @@ export default class ApiService extends Service {
use: [ApiGateway.serveStatic("comics")],
},
],
// Do not log client side errors (does not log an error response when the error.code is 400<=X<500)
log4XXResponses: false,
// Logging the request parameters. Set to any log level to enable it. E.g. "info"
logRequestParams: null,
// Logging the response data. Set to any log level to enable it. E.g. "info"
logResponseData: null,
assets: {
folder: "public",
// Options to `server-static` module
options: {},
},
},
@@ -104,7 +94,7 @@ export default class ApiService extends Service {
this.logger.info("Client connected via websocket!");
client.on(
"importComicsInDB",
"importComicsToDB",
async ({ action, params, opts }, done) => {
this.logger.info(
"Received request from client! Action:",

43
services/imagetransformation.service.ts
View File

@@ -8,12 +8,7 @@ import {
} from "moleculer";
import {
resizeImage,
calculateLevenshteinDistance,
} from "../utils/imagetransformation.utils";
import https from "https";
import fs from "fs";
import path from "path";
export default class ProductsService extends Service {
// @ts-ignore
public constructor(
@@ -58,44 +53,6 @@ export default class ProductsService extends Service {
return { resizeOperationStatus: resizeResult };
},
},
calculateLevenshteinDistance: {
rest: "POST /calculateLevenshteinDistance",
params: {},
async handler(
ctx: Context<{
imagePath: string;
imagePath2: string;
options: {
match_id: string,
};
}>
) {
const fileName = ctx.params.options.match_id + "_" + path.basename(
ctx.params.imagePath
);
return new Promise((resolve, reject) => {
https.get(ctx.params.imagePath2, (response) => {
const fileStream = response.pipe(
fs.createWriteStream(
`./userdata/temporary/${fileName}`
)
);
fileStream.on("finish", async () => {
const levenshteinDistance = await calculateLevenshteinDistance(
ctx.params.imagePath,
path.resolve(
`./userdata/temporary/${fileName}`
)
);
resolve(levenshteinDistance);
});
}).end();
});
},
},
},
methods: {},
},

55
services/import.service.ts
View File

@@ -1,4 +1,5 @@
"use strict";
import { isNil } from "lodash";
import {
Context,
Service,
@@ -10,6 +11,9 @@ import { DbMixin } from "../mixins/db.mixin";
import Comic from "../models/comic.model";
import { walkFolder } from "../utils/file.utils";
import { convertXMLToJSON } from "../utils/xml.utils";
import https from "https";
const ObjectId = require("mongoose").Types.ObjectId;
export default class ProductsService extends Service {
public constructor(
@@ -74,8 +78,55 @@ export default class ProductsService extends Service {
});
},
},
getRecentlyImportedComicBooks: {
rest: "POST /getRecentlyImportedComicBooks",
applyComicVineMetadata: {
rest: "POST /applyComicVineMetadata",
params: {},
async handler(ctx: Context<{ match: { volume: { api_detail_url: string}, volumeInformation: object}, comicObjectId: string }>) {
// 1. find mongo object by id
// 2. import payload into sourcedMetadata.comicvine
const comicObjectId = new ObjectId(ctx.params.comicObjectId);
const matchedResult = ctx.params.match;
console.log(matchedResult.volume.api_detail_url);
let volumeDetailsPromise;
if (!isNil(matchedResult.volume)) {
volumeDetailsPromise = new Promise((resolve, reject) => {
return https.get(`${matchedResult.volume.api_detail_url}?api_key=a5fa0663683df8145a85d694b5da4b87e1c92c69&format=json&limit=1&offset=0&field_list=id,name,deck,image,first_issue,last_issue,publisher,count_of_issues`, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
const volumeInformation = JSON.parse(data);
resolve(volumeInformation);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(err);
});
});
}
return new Promise(async (resolve, reject) => {
const volumeDetails = await volumeDetailsPromise;
matchedResult.volumeInformation = volumeDetails.results;
Comic.findByIdAndUpdate(comicObjectId, { sourcedMetadata: { comicvine: matchedResult } }, { new: true }, (err, result) => {
if (err) {
console.log(err);
reject(err);
} else {
// 3. Fetch and append volume information
resolve(result);
}
});
});
},
},
getComicBooks: {
rest: "POST /getComicBooks",
params: {},
async handler(
ctx: Context<{ paginationOptions: object }>

50
utils/imagetransformation.utils.ts
View File

@@ -1,9 +1,6 @@
const sharp = require("sharp");
const imghash = require("imghash");
const leven = require("leven");
import path from "path";
import { isNull, reject } from "lodash";
import { logger } from "./logger.utils";
import { ISharpResizedImageStats } from "threetwo-ui-typings";
export const extractMetadataFromImage = async (
imageFilePath: string
@@ -21,35 +18,20 @@ export const resizeImage = async (
outputPath: string,
newWidth: number,
newHeight?: number
): Promise<unknown> => {
return await sharp(imageFile)
.resize(newWidth)
.toFile(`${outputPath}`, (err, info) => {
if (err) {
logger.error("Failed to resize image:");
logger.error(err);
return err;
}
): Promise<ISharpResizedImageStats> => {
return new Promise((resolve, reject) => {
sharp(imageFile)
.resize(newWidth)
.toFile(`${outputPath}`, (err, info) => {
if (err) {
logger.error("Failed to resize image:");
logger.error(err);
reject(err);
}
logger.info("Image file resized with the following parameters:");
logger.info(info);
return info;
});
};
export const calculateLevenshteinDistance = async (
imagePath1: string,
imagePath2: string
): Promise<Record<string, unknown>> => {
console.log("AGANTUK", imagePath1)
const hash1 = await imghash.hash(imagePath1);
const hash2 = await imghash.hash(imagePath2);
console.log("HASHISH", hash1)
if (!isNull(hash1) && !isNull(hash2)) {
return new Promise((resolve, reject) => {
resolve({ levenshteinDistance: leven(hash1, hash2) });
});
} else {
reject("Can't calculate the Levenshtein distance")
}
logger.info("Image file resized with the following parameters:");
logger.info(info);
resolve(info);
});
});
};

23
utils/uncompression.utils.ts
View File

@@ -31,7 +31,7 @@ SOFTWARE.
* Initial: 2021/05/04 Rishi Ghan
*/
import { createReadStream, createWriteStream, readFileSync } from "fs";
import { createReadStream, createWriteStream, readFileSync, stat } from "fs";
const fse = require("fs-extra");
import path from "path";
import { each, isEmpty, map, flatten } from "lodash";
@@ -42,6 +42,7 @@ import {
IExtractedComicBookCoverFile,
IExtractionOptions,
IFolderData,
ISharpResizedImageStats,
} from "threetwo-ui-typings";
import { logger } from "./logger.utils";
import { validateComicBookMetadata } from "../utils/validation.utils";
@@ -84,6 +85,7 @@ export const extractCoverFromFile = async (
} catch (error) {
logger.error(`${error}: Couldn't create directory.`);
}
// extract the cover
let result: string;
const targetCoverImageFilePath = path.resolve(constructedPaths.targetPath + "/" + walkedFolder.name + "_cover.jpg")
result = await calibre.run(
@@ -95,15 +97,17 @@ export const extractCoverFromFile = async (
);
// create renditions
const renditionPath = constructedPaths.targetPath + "/" + walkedFolder.name + "_200px.jpg";
await resizeImage(targetCoverImageFilePath, path.resolve(renditionPath), 200);
const stats:ISharpResizedImageStats = await resizeImage(targetCoverImageFilePath, path.resolve(renditionPath), 200);
resolve({
name: walkedFolder.name,
path: constructedPaths.targetPath + "/" + walkedFolder.name + "_200px.jpg", //renditionPath
fileSize: 0,
path: renditionPath,
fileSize: stats.size,
extension: path.extname(constructedPaths.inputFilePath),
containedIn: walkedFolder.containedIn,
//originalPath:
//extension:
calibreMetadata: {
coverWriteResult: result,
}
});
} catch (error) {
console.log(error);
@@ -200,6 +204,7 @@ export const unrar = async (
mode: 0o2775,
};
try {
// read the file into a buffer
const fileBuffer = await readFile(
paths.inputFilePath
).catch((err) => console.error("Failed to read file", err));
@@ -238,8 +243,12 @@ export const unrar = async (
resolve({
name: `${extractedFiles[0].fileHeader.name}`,
path: paths.targetPath,
extension: path.extname(extractedFiles[0].fileHeader.name),
fileSize: extractedFiles[0].fileHeader.packSize,
containedIn: walkedFolder.containedIn,
calibreMetadata: {
coverWriteResult: "",
}
});
} catch (error) {
logger.error(`${error}: Couldn't write file.`);

8
yarn.lock
View File

@@ -5906,10 +5906,10 @@
"resolved" "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz"
"version" "0.2.0"
"threetwo-ui-typings@^1.0.1-0":
"integrity" "sha512-g5FWa069AT1WfUHHEVFOQ1q6cfK+9UOzewfKblheuDBSsGN/e89MJMEVfuInBaHgxHM32/P+mNUFBqnBoeRSLQ=="
"resolved" "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.1.tgz"
"version" "1.0.1"
"threetwo-ui-typings@^1.0.2-0":
"integrity" "sha512-sK5cb/fApFKseQNgcnGmAnPNxDDXT+dQ/Blei1N4q0mduO3kZfJTnlMYaXjO4FDXLP+jkiwsv0bf7PXm0DgF7g=="
"resolved" "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.2.tgz"
"version" "1.0.2"
dependencies:
"typescript" "^4.3.2"