frishi/threetwo-acquisition-service
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

🔧 Fixes to search query builder

Browse Source
This commit is contained in:
Rishi Ghan
2024-06-12 21:41:29 -05:00
parent 60e5b6f61b
commit 55494abdc0
2 changed files with 177 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
services/autodownload.service.ts
View File

@@ -13,16 +13,9 @@ interface Comic {
};
}
interface PaginatedResult {
wantedComics: Comic[];
total: number;
page: number;
limit: number;
pages: number;
}
export default class AutoDownloadService extends Service {
private kafkaProducer: any;
private readonly BATCH_SIZE = 100; // Adjust based on your system capacity
// @ts-ignore
public constructor(
@@ -41,13 +34,23 @@ export default class AutoDownloadService extends Service {
const limit = this.BATCH_SIZE;
while (true) {
const result: PaginatedResult = await this.broker.call(
const comics: Comic[] = await this.broker.call(
"library.getComicsMarkedAsWanted",
{ page, limit },
);
if (!result || !result.wantedComics) {
this.logger.error("Invalid response structure", result);
// Log the entire result object for debugging
this.logger.info(
"Received comics from getComicsMarkedAsWanted:",
JSON.stringify(comics, null, 2),
);
// Check if result structure is correct
if (!Array.isArray(comics)) {
this.logger.error(
"Invalid response structure",
JSON.stringify(comics, null, 2),
);
throw new Errors.MoleculerError(
"Invalid response structure from getComicsMarkedAsWanted",
500,
@@ -56,41 +59,22 @@ export default class AutoDownloadService extends Service {
}
this.logger.info(
`Fetched ${result.wantedComics.length} comics from page ${page} of ${result.pages}`,
`Fetched ${comics.length} comics from page ${page}`,
);
for (const comic of result.wantedComics) {
if (comic.wanted.markEntireVolumeWanted) {
const issues: any = await this.broker.call(
"comicvine.getIssuesForVolume",
{
volumeId: comic.wanted.volume.id,
},
);
for (const issue of issues) {
await this.produceJobToKafka(
comic.wanted.volume.name,
issue,
);
}
} else if (
comic.wanted.issues &&
comic.wanted.issues.length > 0
) {
for (const issue of comic.wanted.issues) {
await this.produceJobToKafka(
comic.wanted.volume?.name,
issue,
);
}
}
// Enqueue the jobs in batches
for (const comic of comics) {
await this.produceJobToKafka(comic);
}
if (page >= result.pages) break;
if (comics.length < limit) break; // End loop if fewer comics than the limit were fetched
page += 1;
}
return { success: true, message: "Processing started." };
return {
success: true,
message: "Jobs enqueued for background processing.",
};
} catch (error) {
this.logger.error("Error in searchWantedComics:", error);
throw new Errors.MoleculerError(
@@ -104,13 +88,17 @@ export default class AutoDownloadService extends Service {
},
},
methods: {
produceJobToKafka: async (volumeName: string, issue: any) => {
const job = { volumeName, issue };
produceJobToKafka: async (comic: Comic) => {
const job = { comic };
try {
await this.kafkaProducer.send({
topic: "comic-search-jobs",
messages: [{ value: JSON.stringify(job) }],
});
this.logger.info("Produced job to Kafka:", job);
} catch (error) {
this.logger.error("Error producing job to Kafka:", error);
}
},
},
async started() {

143
services/comicprocessor.service.ts
View File

@@ -1,8 +1,9 @@
"use strict";
import { Service, ServiceBroker, ServiceSchema } from "moleculer";
import { Kafka, EachMessagePayload, logLevel } from "kafkajs";
import { isUndefined } from "lodash";
import io from "socket.io-client";
import { isUndefined } from "lodash";
interface SearchResult {
result: {
id: string;
@@ -17,12 +18,14 @@ interface SearchResultPayload {
updatedResult: SearchResult;
instanceId: string;
}
export default class ComicProcessorService extends Service {
private kafkaConsumer: any;
private socketIOInstance: any;
private kafkaProducer: any;
private prowlarrResultsMap: Map<string, any> = new Map();
private airDCPPSearchResults: Array<any> = [];
private airDCPPSearchResults: Map<string, any[]> = new Map();
private issuesToSearch: any = [];
// @ts-ignore
public constructor(
@@ -42,18 +45,46 @@ export default class ComicProcessorService extends Service {
};
},
processJob: async (job: any) => {
this.logger.info("Processing job:", job);
const { volumeName, issue } = job;
const { year } = this.parseStringDate(issue.cover_date || issue.coverDate);
try {
this.logger.info("Processing job:", JSON.stringify(job, null, 2));
const { comic } = job;
const { volume, issues, markEntireVolumeWanted } = comic.wanted;
// If entire volume is marked as wanted, get their details from CV
if (markEntireVolumeWanted) {
this.issuesToSearch = await this.broker.call(
"comicvine.getIssuesForVolume",
{ volumeId: volume.id },
);
this.logger.info(
`The entire volume with id: ${volume.id} was marked as wanted.`,
);
this.logger.info(`Fetched issues for ${volume.id}:`);
this.logger.info(`${this.issuesToSearch.length} issues to search`);
} else {
// Or proceed with `issues` from the wanted object.
this.issuesToSearch = issues;
}
for (const issue of this.issuesToSearch) {
// issue number
const inferredIssueNumber = issue.issueNumber
? issue.issueNumber
: issue.issue_number;
// year
const { year } = this.parseStringDate(issue.coverDate);
const inferredYear = year ? issue?.coverDate : issue.year;
const settings: any = await this.broker.call("settings.getSettings", {
settingsKey: "directConnect",
});
const hubs = settings.client.hubs.map((hub: any) => hub.value);
const dcppSearchQuery = {
query: {
pattern: `${volumeName.replace(/#/g, "")} ${
issue.issue_number || issue.issueNumber
} ${year}`,
pattern: `${volume.name.replace(
/#/g,
"",
)} ${inferredIssueNumber} ${inferredYear}`,
extensions: ["cbz", "cbr", "cb7"],
},
hub_urls: hubs,
@@ -81,7 +112,7 @@ export default class ComicProcessorService extends Service {
apiKey: "c4f42e265fb044dc81f7e88bd41c3367",
offset: 0,
categories: [7030],
query: `${volumeName} ${issue.issueNumber} ${year}`,
query: `${volume.name} ${issue.issueNumber} ${year}`,
host: "localhost",
limit: 100,
type: "search",
@@ -93,12 +124,18 @@ export default class ComicProcessorService extends Service {
"Prowlarr search results:",
JSON.stringify(prowlarrResults, null, 4),
);
// Store prowlarr results in map using unique key
const key = `${volumeName}-${issue.issueNumber}-${year}`;
const key = `${volume.name}-${issue.issueNumber}-${year}`;
this.prowlarrResultsMap.set(key, prowlarrResults);
}
} catch (error) {
this.logger.error("Error processing job:", error);
}
},
produceResultsToKafka: async (dcppResults: any, prowlarrResults: any) => {
const results = { dcppResults, prowlarrResults };
try {
await this.kafkaProducer.send({
topic: "comic-search-results",
messages: [{ value: JSON.stringify(results) }],
@@ -107,6 +144,19 @@ export default class ComicProcessorService extends Service {
"Produced results to Kafka:",
JSON.stringify(results, null, 4),
);
// socket event for UI
await this.broker.call("socket.broadcast", {
namespace: "/",
event: "searchResultsAvailable",
args: [
{
dcppResults,
},
],
});
} catch (error) {
this.logger.error("Error producing results to Kafka:", error);
}
},
},
async started() {
@@ -117,9 +167,22 @@ export default class ComicProcessorService extends Service {
});
this.kafkaConsumer = kafka.consumer({ groupId: "comic-processor-group" });
this.kafkaProducer = kafka.producer();
this.kafkaConsumer.on("consumer.crash", (event: any) => {
this.logger.error("Consumer crash:", event);
});
this.kafkaConsumer.on("consumer.connect", () => {
this.logger.info("Consumer connected");
});
this.kafkaConsumer.on("consumer.disconnect", () => {
this.logger.info("Consumer disconnected");
});
this.kafkaConsumer.on("consumer.network.request_timeout", () => {
this.logger.warn("Consumer network request timeout");
});
await this.kafkaConsumer.connect();
await this.kafkaProducer.connect();
this.logger.info("Kafka consumer and producer connected successfully.");
await this.kafkaConsumer.subscribe({
topic: "comic-search-jobs",
@@ -130,10 +193,6 @@ export default class ComicProcessorService extends Service {
eachMessage: async ({ topic, partition, message }: EachMessagePayload) => {
if (message.value) {
const job = JSON.parse(message.value.toString());
this.logger.info(
"Consumed job from Kafka:",
JSON.stringify(job, null, 4),
);
await this.processJob(job);
} else {
this.logger.warn("Received message with null value");
@@ -149,6 +208,7 @@ export default class ComicProcessorService extends Service {
this.logger.info("Socket.IO connected successfully.");
});
// Handle searchResultAdded event
this.socketIOInstance.on(
"searchResultAdded",
({ groupedResult, instanceId }: SearchResultPayload) => {
@@ -156,13 +216,14 @@ export default class ComicProcessorService extends Service {
"Received search result added:",
JSON.stringify(groupedResult, null, 4),
);
this.airDCPPSearchResults.push({
groupedResult: groupedResult.result,
instanceId,
});
if (!this.airDCPPSearchResults.has(instanceId)) {
this.airDCPPSearchResults.set(instanceId, []);
}
this.airDCPPSearchResults.get(instanceId).push(groupedResult.result);
},
);
// Handle searchResultUpdated event
this.socketIOInstance.on(
"searchResultUpdated",
async ({ updatedResult, instanceId }: SearchResultPayload) => {
@@ -170,50 +231,30 @@ export default class ComicProcessorService extends Service {
"Received search result update:",
JSON.stringify(updatedResult, null, 4),
);
if (
!isUndefined(updatedResult.result) &&
!isUndefined(this.airDCPPSearchResults.result)
) {
const toReplaceIndex = this.airDCPPSearchResults.findIndex(
(element: any) => {
return element?.result.id === updatedResult.result.id;
},
const resultsForInstance = this.airDCPPSearchResults.get(instanceId);
if (resultsForInstance) {
const toReplaceIndex = resultsForInstance.findIndex(
(element: any) => element.id === updatedResult.result.id,
);
this.airDCPPSearchResults[toReplaceIndex] = {
result: updatedResult.result,
instanceId,
};
if (toReplaceIndex !== -1) {
resultsForInstance[toReplaceIndex] = updatedResult.result;
}
}
},
);
this.socketIOInstance.on("searchComplete", async () => {
// Ensure results are not empty before producing to Kafka
if (this.airDCPPSearchResults.length > 0) {
const results = this.airDCPPSearchResults.reduce((acc: any, item: any) => {
const key = item.instanceId;
if (!acc[key]) {
acc[key] = [];
}
acc[key].push(item);
return acc;
}, {});
await this.produceResultsToKafka(results, []);
} else {
this.logger.warn(
"AirDC++ search results are empty, not producing to Kafka.",
);
}
// Handle searchComplete event
this.socketIOInstance.on("searchComplete", async (instanceId: string) => {
this.logger.info(`Search complete for instance ID ${instanceId}`);
await this.produceResultsToKafka(instanceId);
});
},
async stopped() {
await this.kafkaConsumer.disconnect();
await this.kafkaProducer.disconnect();
this.logger.info("Kafka consumer and producer disconnected successfully.");
// Close Socket.IO connection
if (this.socketIOInstance) {
this.socketIOInstance.close();
this.logger.info("Socket.IO disconnected successfully.");
}
},
});