frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

✏️ Formatting

Browse Source
This commit is contained in:
Rishi Ghan
2021-09-14 23:13:34 -07:00
parent e9fb359ca3
commit 16d9b50f21
5 changed files with 66 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
queue/consumer.ts
View File

@@ -1,35 +0,0 @@
const amqp = require("amqplib/callback_api");
export const connectQueue = (socketConnection) => {
amqp.connect("amqp://localhost", (error0, connection) => {
if (error0) {
throw error0;
}
connection.createChannel((error1, channel) => {
if (error1) {
throw error1;
}
const queue = "comicBookCovers";
channel.assertQueue(queue, {
durable: false,
});
console.log(
" [*] Waiting for comic book cover data in %s. To exit press CTRL+C",
queue
);
channel.consume(
queue,
(data) => {
console.log("data", data);
//Socket Trigger All Clients
socketConnection.emit("coverExtracted", JSON.parse(data.content.toString()));
},
{
noAck: true,
}
);
});
});
};

4
queue/importQueue.ts
View File

@@ -1,8 +1,6 @@
import { logger } from "../utils/logger.utils"; import { logger } from "../utils/logger.utils";
//RabbitMQ //RabbitMQ
const amqp = require("amqplib/callback_api"); const amqp = require("amqplib/callback_api");
const rabbitUrl = "amqp://localhost"; const rabbitUrl = "amqp://localhost";
export const sendRabbitMQ = (queueName, data) => { export const sendRabbitMQ = (queueName, data) => {
@@ -16,7 +14,6 @@ export const sendRabbitMQ = (queueName, data) => {
if (error1) { if (error1) {
throw error1; throw error1;
} }
channel.prefetch(1);
const queue = queueName; const queue = queueName;
// Checks for “queueName (updateStock)” queue. If it doesn’t exist, then it creates one. // Checks for “queueName (updateStock)” queue. If it doesn’t exist, then it creates one.
channel.assertQueue(queue, { channel.assertQueue(queue, {
@@ -31,4 +28,3 @@ export const sendRabbitMQ = (queueName, data) => {
}, 500); }, 500);
}); });
}; };
module.exports = sendRabbitMQ;

13
services/api.service.ts
View File

@@ -1,7 +1,5 @@
import { Service, ServiceBroker, Context } from "moleculer"; import { Service, ServiceBroker, Context } from "moleculer";
import ApiGateway from "moleculer-web"; import ApiGateway from "moleculer-web";
import { connectQueue } from "../queue/consumer";
const IO = require("socket.io")();
export default class ApiService extends Service { export default class ApiService extends Service {
public constructor(broker: ServiceBroker) { public constructor(broker: ServiceBroker) {
super(broker); super(broker);
@@ -71,20 +69,11 @@ export default class ApiService extends Service {
}, },
}, },
events: { events: {
"**"(payload, sender, event) {
if (this.io)
this.io.emit("event", {
sender,
event,
payload,
});
},
}, },
methods: {}, methods: {},
started(): any { started(): any {
this.io = IO.listen(this.server);
connectQueue(this.io);
}, },
}); });
} }

114
services/import.service.ts
View File

@@ -13,7 +13,7 @@ import { walkFolder } from "../utils/file.utils";
import { convertXMLToJSON } from "../utils/xml.utils"; import { convertXMLToJSON } from "../utils/xml.utils";
import https from "https"; import https from "https";
import { logger } from "../utils/logger.utils"; import { logger } from "../utils/logger.utils";
const rabbitmq = require("../queue/importQueue"); import { sendRabbitMQ } from "../queue/importQueue";
import { import {
IExtractComicBookCoverErrorResponse, IExtractComicBookCoverErrorResponse,
IExtractedComicBookCoverFile, IExtractedComicBookCoverFile,
@@ -66,6 +66,11 @@ export default class ImportService extends Service {
}, },
importComicsToDb: { importComicsToDb: {
rest: "POST /importComicsToDB", rest: "POST /importComicsToDB",
bulkhead: {
enabled: true,
concurrency: 50,
maxQueueSize: 100,
},
params: {}, params: {},
async handler( async handler(
ctx: Context<{ ctx: Context<{
@@ -82,57 +87,64 @@ export default class ImportService extends Service {
]; ];
}> }>
) { ) {
const { extractionOptions, walkedFolders } = try {
ctx.params; const { extractionOptions, walkedFolders } =
map(walkedFolders, async (folder, idx) => { ctx.params;
let comicExists = await Comic.exists({ map(walkedFolders, async (folder, idx) => {
"rawFileDetails.name": `${folder.name}`, let comicExists = await Comic.exists({
}); "rawFileDetails.name": `${folder.name}`,
if (!comicExists) { });
let comicBookCoverMetadata: if (!comicExists) {
| IExtractedComicBookCoverFile // 1. Extract cover and cover metadata
| IExtractComicBookCoverErrorResponse let comicBookCoverMetadata:
| IExtractedComicBookCoverFile[] = await extractCoverFromFile( | IExtractedComicBookCoverFile
extractionOptions, | IExtractComicBookCoverErrorResponse
folder | IExtractedComicBookCoverFile[] =
); await extractCoverFromFile(
extractionOptions,
folder
);
// const dbImportResult = // 2. Add to mongo
// await this.broker.call( const dbImportResult =
// "import.rawImportToDB", await this.broker.call(
// { "import.rawImportToDB",
// importStatus: { {
// isImported: true, importStatus: {
// tagged: false, isImported: true,
// matchedResult: { tagged: false,
// score: "0", matchedResult: {
// }, score: "0",
// }, },
// rawFileDetails: },
// comicBookCoverMetadata, rawFileDetails:
// sourcedMetadata: { comicBookCoverMetadata,
// comicvine: {}, sourcedMetadata: {
// }, comicvine: {},
// }, },
// {} },
// ); {}
rabbitmq( );
"comicBookCovers", // 3. Send to the queue
JSON.stringify({ sendRabbitMQ(
comicBookCoverMetadata, "comicBookCovers",
}) JSON.stringify({
); comicBookCoverMetadata,
} else { dbImportResult,
logger.info( })
`Comic: \"${folder.name}\" already exists in the database` );
); } else {
rabbitmq("comicBookCovers", logger.info(
JSON.stringify({ `Comic: \"${folder.name}\" already exists in the database`
name: folder.name, );
}) }
); });
} } catch (error) {
}); logger.error(
"Error importing comic books",
error
);
}
}, },
}, },
rawImportToDB: { rawImportToDB: {

4
utils/logger.utils.ts
View File

@@ -2,12 +2,12 @@ import { default as Pino } from "pino";
import { default as pinopretty } from "pino-pretty"; import { default as pinopretty } from "pino-pretty";
export const logger = Pino({ export const logger = Pino({
name: "threetwo!", name: "Threetwo!",
prettyPrint: { colorize: true }, prettyPrint: { colorize: true },
// crlf: false, // crlf: false,
// errorLikeObjectKeys: ["err", "error"], // errorLikeObjectKeys: ["err", "error"],
// errorProps: "", // errorProps: "",
// levelFirst: false, // --levelFirst // levelFirst: false,
messageKey: "msg", // --messageKey messageKey: "msg", // --messageKey
levelKey: "level", // --levelKey levelKey: "level", // --levelKey
// messageFormat: false, // --messageFormat // messageFormat: false, // --messageFormat