frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

✏️ Formatting

Browse Source
This commit is contained in:
Rishi Ghan
2021-09-14 23:13:34 -07:00
parent e9fb359ca3
commit 16d9b50f21
5 changed files with 66 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
queue/consumer.ts
View File

@@ -1,35 +0,0 @@
const amqp = require("amqplib/callback_api");
export const connectQueue = (socketConnection) => {
amqp.connect("amqp://localhost", (error0, connection) => {
if (error0) {
throw error0;
}
connection.createChannel((error1, channel) => {
if (error1) {
throw error1;
}
const queue = "comicBookCovers";
channel.assertQueue(queue, {
durable: false,
});
console.log(
" [*] Waiting for comic book cover data in %s. To exit press CTRL+C",
queue
);
channel.consume(
queue,
(data) => {
console.log("data", data);
//Socket Trigger All Clients
socketConnection.emit("coverExtracted", JSON.parse(data.content.toString()));
},
{
noAck: true,
}
);
});
});
};

4
queue/importQueue.ts
View File

@@ -1,8 +1,6 @@
import { logger } from "../utils/logger.utils";
//RabbitMQ
const amqp = require("amqplib/callback_api");
const rabbitUrl = "amqp://localhost";
export const sendRabbitMQ = (queueName, data) => {
@@ -16,7 +14,6 @@ export const sendRabbitMQ = (queueName, data) => {
if (error1) {
throw error1;
}
channel.prefetch(1);
const queue = queueName;
// Checks for “queueName (updateStock)” queue. If it doesn’t exist, then it creates one.
channel.assertQueue(queue, {
@@ -31,4 +28,3 @@ export const sendRabbitMQ = (queueName, data) => {
}, 500);
});
};
module.exports = sendRabbitMQ;

13
services/api.service.ts
View File

@@ -1,7 +1,5 @@
import { Service, ServiceBroker, Context } from "moleculer";
import ApiGateway from "moleculer-web";
import { connectQueue } from "../queue/consumer";
const IO = require("socket.io")();
export default class ApiService extends Service {
public constructor(broker: ServiceBroker) {
super(broker);
@@ -71,20 +69,11 @@ export default class ApiService extends Service {
},
},
events: {
"**"(payload, sender, event) {
if (this.io)
this.io.emit("event", {
sender,
event,
payload,
});
},
},
methods: {},
started(): any {
this.io = IO.listen(this.server);
connectQueue(this.io);
},
});
}

114
services/import.service.ts
View File

@@ -13,7 +13,7 @@ import { walkFolder } from "../utils/file.utils";
import { convertXMLToJSON } from "../utils/xml.utils";
import https from "https";
import { logger } from "../utils/logger.utils";
const rabbitmq = require("../queue/importQueue");
import { sendRabbitMQ } from "../queue/importQueue";
import {
IExtractComicBookCoverErrorResponse,
IExtractedComicBookCoverFile,
@@ -66,6 +66,11 @@ export default class ImportService extends Service {
},
importComicsToDb: {
rest: "POST /importComicsToDB",
bulkhead: {
enabled: true,
concurrency: 50,
maxQueueSize: 100,
},
params: {},
async handler(
ctx: Context<{
@@ -82,57 +87,64 @@ export default class ImportService extends Service {
];
}>
) {
const { extractionOptions, walkedFolders } =
ctx.params;
map(walkedFolders, async (folder, idx) => {
let comicExists = await Comic.exists({
"rawFileDetails.name": `${folder.name}`,
});
if (!comicExists) {
let comicBookCoverMetadata:
| IExtractedComicBookCoverFile
| IExtractComicBookCoverErrorResponse
| IExtractedComicBookCoverFile[] = await extractCoverFromFile(
extractionOptions,
folder
);
try {
const { extractionOptions, walkedFolders } =
ctx.params;
map(walkedFolders, async (folder, idx) => {
let comicExists = await Comic.exists({
"rawFileDetails.name": `${folder.name}`,
});
if (!comicExists) {
// 1. Extract cover and cover metadata
let comicBookCoverMetadata:
| IExtractedComicBookCoverFile
| IExtractComicBookCoverErrorResponse
| IExtractedComicBookCoverFile[] =
await extractCoverFromFile(
extractionOptions,
folder
);
// const dbImportResult =
// await this.broker.call(
// "import.rawImportToDB",
// {
// importStatus: {
// isImported: true,
// tagged: false,
// matchedResult: {
// score: "0",
// },
// },
// rawFileDetails:
// comicBookCoverMetadata,
// sourcedMetadata: {
// comicvine: {},
// },
// },
// {}
// );
rabbitmq(
"comicBookCovers",
JSON.stringify({
comicBookCoverMetadata,
})
);
} else {
logger.info(
`Comic: \"${folder.name}\" already exists in the database`
);
rabbitmq("comicBookCovers",
JSON.stringify({
name: folder.name,
})
);
}
});
// 2. Add to mongo
const dbImportResult =
await this.broker.call(
"import.rawImportToDB",
{
importStatus: {
isImported: true,
tagged: false,
matchedResult: {
score: "0",
},
},
rawFileDetails:
comicBookCoverMetadata,
sourcedMetadata: {
comicvine: {},
},
},
{}
);
// 3. Send to the queue
sendRabbitMQ(
"comicBookCovers",
JSON.stringify({
comicBookCoverMetadata,
dbImportResult,
})
);
} else {
logger.info(
`Comic: \"${folder.name}\" already exists in the database`
);
}
});
} catch (error) {
logger.error(
"Error importing comic books",
error
);
}
},
},
rawImportToDB: {

4
utils/logger.utils.ts
View File

@@ -2,12 +2,12 @@ import { default as Pino } from "pino";
import { default as pinopretty } from "pino-pretty";
export const logger = Pino({
name: "threetwo!",
name: "Threetwo!",
prettyPrint: { colorize: true },
// crlf: false,
// errorLikeObjectKeys: ["err", "error"],
// errorProps: "",
// levelFirst: false, // --levelFirst
// levelFirst: false,
messageKey: "msg", // --messageKey
levelKey: "level", // --levelKey
// messageFormat: false, // --messageFormat