From 7dbe2b27015661f1f417d2b70856a01fb18f8eb5 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Sun, 3 Mar 2024 12:22:40 -0500 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=8F=97=EF=B8=8F=20Added=20torrent=20a?= =?UTF-8?q?ttrs=20to=20comic=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/comic.model.ts | 14 +++++++------- models/settings.model.ts | 6 ++++++ services/library.service.ts | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/models/comic.model.ts b/models/comic.model.ts index 8778cd6..7dbd06f 100644 --- a/models/comic.model.ts +++ b/models/comic.model.ts @@ -115,13 +115,13 @@ const ComicSchema = mongoose.Schema( default: [], }, }, - torrent: { - downloads: [], - sourceApplication: String, - magnet: String, - tracker: String, - status: String, - }, + torrent: [ + { + infoHash: String, + name: String, + announce: [String], + }, + ], usenet: { sourceApplication: String, }, diff --git a/models/settings.model.ts b/models/settings.model.ts index a60f75d..73a4422 100644 --- a/models/settings.model.ts +++ b/models/settings.model.ts @@ -23,6 +23,12 @@ const SettingsScehma = mongoose.Schema({ host: HostSchema, }, }, + prowlarr: { + client: { + host: HostSchema, + apiKey: String, + }, + }, }); const Settings = mongoose.model("Settings", SettingsScehma); diff --git a/services/library.service.ts b/services/library.service.ts index ce198d8..26ae9b2 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -303,6 +303,7 @@ export default class ImportService extends Service { ); switch (ctx.params.importType) { case "new": + console.log(comicMetadata); return await Comic.create(comicMetadata); case "update": return await Comic.findOneAndUpdate( @@ -424,6 +425,41 @@ export default class ImportService extends Service { }); }, }, + applyTorrentDownloadMetadata: { + rest: "POST /applyTorrentDownloadMetadata", + handler: async ( + ctx: Context<{ + torrentToDownload: any; + comicObjectId: String; + infoHash: String; + name: String; + announce: [String]; + }> + ) => { + const { + name, + torrentToDownload, + comicObjectId, + announce, + infoHash, + } = ctx.params; + console.log(JSON.stringify(ctx.params, null, 4)); + + return await Comic.findByIdAndUpdate( + new ObjectId(comicObjectId), + { + $push: { + "acquisition.torrent": { + infoHash, + name, + announce, + }, + }, + }, + { new: true, safe: true, upsert: true } + ); + }, + }, getComicBooks: { rest: "POST /getComicBooks", params: {}, -- 2.49.1 From 8f0c2f4302e1097401bb814913c482e444fc79ee Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Tue, 12 Mar 2024 16:39:44 -0500 Subject: [PATCH 2/5] =?UTF-8?q?=E2=9A=99=EF=B8=8F=20getAllSettings=20is=20?= =?UTF-8?q?parameterized?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/settings.service.ts | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/services/settings.service.ts b/services/settings.service.ts index 7159a00..35666bf 100644 --- a/services/settings.service.ts +++ b/services/settings.service.ts @@ -28,12 +28,32 @@ export default class SettingsService extends Service { rest: "GET /getAllSettings", params: {}, async handler(ctx: Context<{ settingsKey: string }>) { - const settings = await Settings.find({}); - if (isEmpty(settings)) { + const { settingsKey } = ctx.params; + + // Initialize a projection object. Include everything by default. + let projection = settingsKey + ? { _id: 0, [settingsKey]: 1 } + : {}; + + // Find the settings with the dynamic projection + const settings = await Settings.find({}, projection); + + // Check if settings are empty + if (settings.length === 0) { return {}; } - console.log(settings[0]); - return settings[0]; + + // If settingsKey is provided, return the specific part of the settings. + // Otherwise, return the entire settings document. + if (settingsKey) { + // Check if the specific key exists in the settings document. + // Since `settings` is an array, we access the first element. + // Then, we use the settingsKey to return only that part of the document. + return settings[0][settingsKey] || {}; + } else { + // Return the entire settings document + return settings[0]; + } }, }, -- 2.49.1 From aea7a24f766d339e5dec036e2147f5310313bd27 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Sun, 24 Mar 2024 17:31:31 -0400 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=A7=B2=20Added=20a=20job=20for=20dele?= =?UTF-8?q?ted=20torrents=20clean-up?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/jobqueue.service.ts | 47 +++++++++++++++++++++++++++++++++- services/library.service.ts | 49 +++++++++++++++++++++++++++--------- services/settings.service.ts | 1 - 3 files changed, 83 insertions(+), 14 deletions(-) diff --git a/services/jobqueue.service.ts b/services/jobqueue.service.ts index 5e1cc08..a6eea9d 100644 --- a/services/jobqueue.service.ts +++ b/services/jobqueue.service.ts @@ -51,9 +51,10 @@ export default class JobQueueService extends Service { } }, }, + enqueue: { queue: true, - rest: "/GET enqueue", + rest: "GET /enqueue", handler: async ( ctx: Context<{ queueName: string; description: string }> ) => { @@ -73,6 +74,49 @@ export default class JobQueueService extends Service { return job.id; }, }, + checkForDeletedTorrents: { + queue: true, + rest: "GET /checkForDeletedTorrents", + handler: async (ctx: Context<{}>) => { + const job = await this.localQueue( + ctx, + "deletedTorrents", + "bird", + { + repeat: { + every: 10000, // Repeat every 10000 ms + limit: 100, // Limit to 100 repeats + }, + } + ); + return job; + }, + }, + deletedTorrents: { + rest: "GET /deletedTorrents", + handler: async ( + ctx: Context<{ + birdName: String; + }> + ) => { + console.info( + `Scheduled job for deleting torrents from mongo fired.` + ); + // 1. query mongo for infohashes + const infoHashes = await this.broker.call( + "library.getInfoHashes", + {} + ); + // 2. query qbittorrent to see if they exist + const torrents: any = await this.broker.call( + "qbittorrent.getTorrentRealTimeStats", + { infoHashes } + ); + console.log("sudarshan", torrents); + // 3. If they do, don't do anything + // 4. If they don't purge them from mongo + }, + }, // Comic Book Import Job Queue "enqueue.async": { handler: async ( @@ -437,6 +481,7 @@ export default class JobQueueService extends Service { }); }, }, + methods: {}, }); } } diff --git a/services/library.service.ts b/services/library.service.ts index 26ae9b2..066aed3 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -444,20 +444,45 @@ export default class ImportService extends Service { infoHash, } = ctx.params; console.log(JSON.stringify(ctx.params, null, 4)); - - return await Comic.findByIdAndUpdate( - new ObjectId(comicObjectId), - { - $push: { - "acquisition.torrent": { - infoHash, - name, - announce, + try { + return await Comic.findByIdAndUpdate( + new ObjectId(comicObjectId), + { + $push: { + "acquisition.torrent": { + infoHash, + name, + announce, + }, }, }, - }, - { new: true, safe: true, upsert: true } - ); + { new: true, safe: true, upsert: true } + ); + } catch (err) { + console.log(err); + } + }, + }, + getInfoHashes: { + rest: "GET /getInfoHashes", + handler: async (ctx: Context<{}>) => { + try { + return await Comic.aggregate([ + { + $unwind: "$acquisition.torrent", + }, + { + $group: { + _id: "$_id", + infoHashes: { + $push: "$acquisition.torrent.infoHash", + }, + }, + }, + ]); + } catch (err) { + return err; + } }, }, getComicBooks: { diff --git a/services/settings.service.ts b/services/settings.service.ts index 35666bf..361ca1f 100644 --- a/services/settings.service.ts +++ b/services/settings.service.ts @@ -38,7 +38,6 @@ export default class SettingsService extends Service { // Find the settings with the dynamic projection const settings = await Settings.find({}, projection); - // Check if settings are empty if (settings.length === 0) { return {}; } -- 2.49.1 From f053dcb7899f13397a215b6826ff58b3a1478d60 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Wed, 27 Mar 2024 22:22:40 -0500 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=A7=B2=20Massaging=20data=20to=20be?= =?UTF-8?q?=20sent=20to=20UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/jobqueue.service.ts | 49 +++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/services/jobqueue.service.ts b/services/jobqueue.service.ts index a6eea9d..b782351 100644 --- a/services/jobqueue.service.ts +++ b/services/jobqueue.service.ts @@ -12,7 +12,6 @@ import { import { isNil, isUndefined } from "lodash"; import { pubClient } from "../config/redis.config"; import path from "path"; - const { MoleculerError } = require("moleculer").Errors; console.log(process.env.REDIS_URI); @@ -74,33 +73,44 @@ export default class JobQueueService extends Service { return job.id; }, }, - checkForDeletedTorrents: { + getTorrentData: { queue: true, - rest: "GET /checkForDeletedTorrents", - handler: async (ctx: Context<{}>) => { + rest: "GET /getTorrentData", + handler: async (ctx: Context<{ trigger: string }>) => { + const { trigger } = ctx.params; + console.log(`Recieved ${trigger} as the trigger...`); + + const jobOptions = { + jobId: "retrieveTorrentData", + name: "bossy", + repeat: { + every: 10000, // Repeat every 10000 ms + limit: 100, // Limit to 100 repeats + }, + }; + const job = await this.localQueue( ctx, - "deletedTorrents", + "fetchTorrentDataJob", "bird", - { - repeat: { - every: 10000, // Repeat every 10000 ms - limit: 100, // Limit to 100 repeats - }, - } + jobOptions ); return job; }, }, - deletedTorrents: { - rest: "GET /deletedTorrents", + fetchTorrentDataJob: { + rest: "GET /fetchTorrentDataJob", handler: async ( ctx: Context<{ birdName: String; }> ) => { + const repeatableJob = await this.$resolve( + "jobqueue" + ).getRepeatableJobs(); + console.info(repeatableJob); console.info( - `Scheduled job for deleting torrents from mongo fired.` + `Scheduled job for fetching torrent data fired.` ); // 1. query mongo for infohashes const infoHashes = await this.broker.call( @@ -112,7 +122,16 @@ export default class JobQueueService extends Service { "qbittorrent.getTorrentRealTimeStats", { infoHashes } ); - console.log("sudarshan", torrents); + // 4. Emit the LS_COVER_EXTRACTION_FAILED event with the necessary details + await this.broker.call("socket.broadcast", { + namespace: "/", + event: "AS_TORRENT_DATA", + args: [ + { + torrents, + }, + ], + }); // 3. If they do, don't do anything // 4. If they don't purge them from mongo }, -- 2.49.1 From b35e2140b5023070cb3f799a010ffeec9ae8bd07 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Fri, 29 Mar 2024 19:36:16 -0400 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=A7=B2=20Created=20a=20dedicated=20qu?= =?UTF-8?q?eue=20for=20torrent=20ops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/jobqueue.service.ts | 68 +-------------------- services/library.service.ts | 2 +- services/torrentjobs.service.ts | 101 ++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 66 deletions(-) create mode 100644 services/torrentjobs.service.ts diff --git a/services/jobqueue.service.ts b/services/jobqueue.service.ts index b782351..830e18c 100644 --- a/services/jobqueue.service.ts +++ b/services/jobqueue.service.ts @@ -55,13 +55,13 @@ export default class JobQueueService extends Service { queue: true, rest: "GET /enqueue", handler: async ( - ctx: Context<{ queueName: string; description: string }> + ctx: Context<{ action: string; description: string }> ) => { - const { queueName, description } = ctx.params; + const { action, description } = ctx.params; // Enqueue the job const job = await this.localQueue( ctx, - queueName, + action, ctx.params, { priority: 10, @@ -73,69 +73,7 @@ export default class JobQueueService extends Service { return job.id; }, }, - getTorrentData: { - queue: true, - rest: "GET /getTorrentData", - handler: async (ctx: Context<{ trigger: string }>) => { - const { trigger } = ctx.params; - console.log(`Recieved ${trigger} as the trigger...`); - const jobOptions = { - jobId: "retrieveTorrentData", - name: "bossy", - repeat: { - every: 10000, // Repeat every 10000 ms - limit: 100, // Limit to 100 repeats - }, - }; - - const job = await this.localQueue( - ctx, - "fetchTorrentDataJob", - "bird", - jobOptions - ); - return job; - }, - }, - fetchTorrentDataJob: { - rest: "GET /fetchTorrentDataJob", - handler: async ( - ctx: Context<{ - birdName: String; - }> - ) => { - const repeatableJob = await this.$resolve( - "jobqueue" - ).getRepeatableJobs(); - console.info(repeatableJob); - console.info( - `Scheduled job for fetching torrent data fired.` - ); - // 1. query mongo for infohashes - const infoHashes = await this.broker.call( - "library.getInfoHashes", - {} - ); - // 2. query qbittorrent to see if they exist - const torrents: any = await this.broker.call( - "qbittorrent.getTorrentRealTimeStats", - { infoHashes } - ); - // 4. Emit the LS_COVER_EXTRACTION_FAILED event with the necessary details - await this.broker.call("socket.broadcast", { - namespace: "/", - event: "AS_TORRENT_DATA", - args: [ - { - torrents, - }, - ], - }); - // 3. If they do, don't do anything - // 4. If they don't purge them from mongo - }, - }, // Comic Book Import Job Queue "enqueue.async": { handler: async ( diff --git a/services/library.service.ts b/services/library.service.ts index 066aed3..904186d 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -222,7 +222,7 @@ export default class ImportService extends Service { }, sessionId, importType: "new", - queueName: "enqueue.async", + action: "enqueue.async", }); } else { console.log( diff --git a/services/torrentjobs.service.ts b/services/torrentjobs.service.ts new file mode 100644 index 0000000..c6e5f51 --- /dev/null +++ b/services/torrentjobs.service.ts @@ -0,0 +1,101 @@ +"use strict"; +import axios from "axios"; +import { + Context, + Service, + ServiceBroker, + ServiceSchema, + Errors, +} from "moleculer"; +import { DbMixin } from "../mixins/db.mixin"; +import Comic from "../models/comic.model"; +const ObjectId = require("mongoose").Types.ObjectId; +import { isNil, isUndefined } from "lodash"; +import BullMqMixin from "moleculer-bullmq"; +const { MoleculerError } = require("moleculer").Errors; + +export default class ImageTransformation extends Service { + // @ts-ignore + public constructor( + public broker: ServiceBroker, + schema: ServiceSchema<{}> = { name: "imagetransformation" } + ) { + super(broker); + this.parseServiceSchema({ + name: "torrentjobs", + mixins: [DbMixin("comics", Comic), BullMqMixin], + settings: { + bullmq: { + client: process.env.REDIS_URI, + }, + }, + hooks: {}, + actions: { + getTorrentData: { + queue: true, + rest: "GET /getTorrentData", + handler: async (ctx: Context<{ trigger: string }>) => { + const { trigger } = ctx.params; + console.log(`Recieved ${trigger} as the trigger...`); + + const jobOptions = { + jobId: "retrieveTorrentData", + name: "bossy", + repeat: { + every: 10000, // Repeat every 10000 ms + limit: 100, // Limit to 100 repeats + }, + }; + + const job = await this.localQueue( + ctx, + "fetchTorrentData", + ctx.params, + jobOptions + ); + return job; + }, + }, + fetchTorrentData: { + rest: "GET /fetchTorrentData", + handler: async ( + ctx: Context<{ + birdName: String; + }> + ) => { + const repeatableJob = await this.$resolve( + "torrentjobs" + ).getRepeatableJobs(); + console.info(repeatableJob); + console.info( + `Scheduled job for fetching torrent data fired.` + ); + // 1. query mongo for infohashes + const infoHashes = await this.broker.call( + "library.getInfoHashes", + {} + ); + // 2. query qbittorrent to see if they exist + const torrents: any = await this.broker.call( + "qbittorrent.getTorrentRealTimeStats", + { infoHashes } + ); + // 4. Emit the LS_COVER_EXTRACTION_FAILED event with the necessary details + await this.broker.call("socket.broadcast", { + namespace: "/", + event: "AS_TORRENT_DATA", + args: [ + { + torrents, + }, + ], + }); + // 3. If they do, don't do anything + // 4. If they don't purge them from mongo + }, + }, + }, + methods: {}, + }); + } +} -- 2.49.1