From 019493a6f357cb17ffe5b97d36fcb6f5612e21e5 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Wed, 26 Jan 2022 00:40:30 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20Refactoring=20core-service=20to?= =?UTF-8?q?=20autowatch=20and=20import=20comics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mixins/db.mixin.ts | 3 +- package.json | 2 +- services/api.service.ts | 171 ++++++++++++++++--------------- services/import.service.ts | 17 +-- services/libraryqueue.service.ts | 1 + utils/file.utils.ts | 4 +- utils/uncompression.utils.ts | 6 +- 7 files changed, 108 insertions(+), 96 deletions(-) diff --git a/mixins/db.mixin.ts b/mixins/db.mixin.ts index 0637b16..c6e0353 100644 --- a/mixins/db.mixin.ts +++ b/mixins/db.mixin.ts @@ -3,7 +3,7 @@ const mkdir = require("mkdirp").sync; const DbService = require("moleculer-db"); const MongoAdapter = require("moleculer-db-adapter-mongoose"); -export const DbMixin = (collection, model) => { +export const DbMixin = (collection, model) => { if(process.env.MONGO_URI) { return { mixins: [DbService], @@ -12,6 +12,7 @@ export const DbMixin = (collection, model) => { pass: process.env.MONGO_INITDB_ROOT_PASSWORD, keepAlive: true, useUnifiedTopology: true, + family: 4, }), model, collection, diff --git a/package.json b/package.json index 44c15ad..1b85bbc 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "@types/string-similarity": "^4.0.0", "7zip-bin": "^5.1.1", "7zip-min": "^1.4.0", - "chokidar": "^3.5.2", + "chokidar": "^3.5.3", "dotenv": "^10.0.0", "fs-extra": "^10.0.0", "imghash": "^0.0.9", diff --git a/services/api.service.ts b/services/api.service.ts index d3ecc1f..503b17f 100644 --- a/services/api.service.ts +++ b/services/api.service.ts @@ -90,7 +90,6 @@ export default class ApiService extends Service { methods: {}, started(): any { - // Add a connect listener io.on("connection", (client) => { console.log("Client connected via websocket!"); @@ -112,94 +111,102 @@ export default class ApiService extends Service { client.on("disconnect", () => { console.log("Client disconnected"); }); - }); - // Filewatcher - const fileWatcher = chokidar.watch(path.resolve("./comics"), { - ignored: /(^|[\/\\])\../, // ignore dotfiles - persistent: true, - ignoreInitial: true, - atomic: true, - awaitWriteFinish: { - stabilityThreshold: 2000, - pollInterval: 100, - }, - }); - const fileCopyDelaySeconds = 10; - const checkFileCopyComplete = (path, previousPath) => { - fs.stat(path, async (err, stat) => { - if (err) { - throw err; + // Filewatcher + const fileWatcher = chokidar.watch( + path.resolve("./comics"), + { + ignored: /(^|[\/\\])\../, // ignore dotfiles + persistent: true, + usePolling: true, + ignoreInitial: true, + atomic: true, + depth: 1, + awaitWriteFinish: { + stabilityThreshold: 2000, + pollInterval: 100, + }, } - if ( - stat.mtime.getTime() === - previousPath.mtime.getTime() - ) { - console.log("File detected, starting import..."); - // this walking business needs to go, SACURATAYYY, SACURATAYYY!! This dude needs to go. - const walkedFolders: IFolderData = - await broker.call("import.walkFolders", { - basePathToWalk: path, - }); - const extractionOptions: IExtractionOptions = { - extractTarget: "cover", - targetExtractionFolder: "./userdata/covers", - extractionMode: "single", - }; - await this.broker.call( - "import.processAndImportToDB", - { walkedFolders, extractionOptions } - ); - } else { - setTimeout( - checkFileCopyComplete, - fileCopyDelaySeconds * 1000, - path, - stat - ); - } - }); - }; - fileWatcher - .on("add", async (path, stats) => { - console.log("Watcher detected new files."); - console.log( - `File ${path} has been added with stats: ${JSON.stringify( - stats - )}` - ); - - console.log("File copy started..."); - fs.stat(path, function (err, stat) { + ); + const fileCopyDelaySeconds = 10; + const checkFileCopyComplete = (path, previousPath) => { + fs.stat(path, async (err, stat) => { if (err) { - console.log( - "Error watching file for copy completion. ERR: " + - err.message - ); - console.log( - "Error file not processed. PATH: " + path - ); throw err; } - setTimeout( - checkFileCopyComplete, - fileCopyDelaySeconds * 1000, - path, - stat - ); + if ( + stat.mtime.getTime() === + previousPath.mtime.getTime() + ) { + console.log( + "File detected, starting import..." + ); + // this walking business needs to go, SACURATAYYY, SACURATAYYY!! This dude needs to go. + const walkedFolder: IFolderData = + await broker.call("import.walkFolders", { + basePathToWalk: path, + }); + console.log(walkedFolder); + await this.broker.call( + "import.processAndImportToDB", + { walkedFolder } + ); + } else { + setTimeout( + checkFileCopyComplete, + fileCopyDelaySeconds * 1000, + path, + stat + ); + } }); - }) - .on("change", (path, stats) => - console.log( - `File ${path} has been changed. Stats: ${stats}` + }; + fileWatcher + .on("add", async (path, stats) => { + console.log("Watcher detected new files."); + console.log( + `File ${path} has been added with stats: ${JSON.stringify( + stats + )}` + ); + + console.log("File copy started..."); + fs.stat(path, function (err, stat) { + if (err) { + console.log( + "Error watching file for copy completion. ERR: " + + err.message + ); + console.log( + "Error file not processed. PATH: " + + path + ); + throw err; + } + setTimeout( + checkFileCopyComplete, + fileCopyDelaySeconds * 1000, + path, + stat + ); + client.emit("action", { + type: "LS_COMIC_ADDED", + result: path, + }); + }); + }) + .on("change", (path, stats) => + console.log( + `File ${path} has been changed. Stats: ${stats}` + ) ) - ) - .on("unlink", (path) => - console.log(`File ${path} has been removed`) - ) - .on("addDir", (path) => - console.log(`Directory ${path} has been added`) - ); + .on("unlink", (path) => + console.log(`File ${path} has been removed`) + ) + .on("addDir", (path) => + console.log(`Directory ${path} has been added`) + ); + }); }, }); } diff --git a/services/import.service.ts b/services/import.service.ts index a180f51..b007455 100644 --- a/services/import.service.ts +++ b/services/import.service.ts @@ -27,6 +27,7 @@ import klaw from "klaw"; import path from "path"; import { COMICS_DIRECTORY, USERDATA_DIRECTORY } from "../constants/directories"; +console.log(process.env.MONGO_URI); export default class ImportService extends Service { public constructor(public broker: ServiceBroker) { super(broker); @@ -128,8 +129,7 @@ export default class ImportService extends Service { params: {}, async handler( ctx: Context<{ - extractionOptions: any; - walkedFolders: { + walkedFolder: { name: string; path: string; extension: string; @@ -141,10 +141,9 @@ export default class ImportService extends Service { }> ) { try { - const { extractionOptions, walkedFolders } = - ctx.params; + const { walkedFolder } = ctx.params; let comicExists = await Comic.exists({ - "rawFileDetails.name": `${walkedFolders.name}`, + "rawFileDetails.name": `${walkedFolder.name}`, }); // rough flow of import process // 1. Walk folder @@ -157,7 +156,7 @@ export default class ImportService extends Service { | IExtractedComicBookCoverFile | IExtractComicBookCoverErrorResponse | IExtractedComicBookCoverFile[] = await extractCoverFromFile2( - extractionOptions + walkedFolder[0] ); // 2. Add to mongo @@ -185,7 +184,7 @@ export default class ImportService extends Service { }; } else { console.info( - `Comic: \"${walkedFolders.name}\" already exists in the database` + `Comic: \"${walkedFolder.name}\" already exists in the database` ); } } catch (error) { @@ -459,7 +458,9 @@ export default class ImportService extends Service { }); resp.on("end", () => { - console.log(`${apiDetailURL} returned data.`) + console.log( + `${apiDetailURL} returned data.` + ); const volumeInformation = JSON.parse(data); resolve(volumeInformation.results); diff --git a/services/libraryqueue.service.ts b/services/libraryqueue.service.ts index 73102b0..86b3a24 100644 --- a/services/libraryqueue.service.ts +++ b/services/libraryqueue.service.ts @@ -14,6 +14,7 @@ import { extractCoverFromFile2 } from "../utils/uncompression.utils"; import { io } from "./api.service"; const REDIS_URI = process.env.REDIS_URI || `redis://0.0.0.0:6379`; +console.log(`REDIS -> ${REDIS_URI}`); export default class LibraryQueueService extends Service { public constructor(public broker: ServiceBroker) { super(broker); diff --git a/utils/file.utils.ts b/utils/file.utils.ts index 9822ec4..525d3e7 100644 --- a/utils/file.utils.ts +++ b/utils/file.utils.ts @@ -17,7 +17,7 @@ export const walkFolder = async (folder: string, formats: string[]): Promise => { try { - const { filePath, size} = fileObject; + console.log("ASDASD!@#!#!@#!@#"); + console.log(fileObject); + const { filePath, fileSize} = fileObject; const calibre = new Calibre(); console.info(`Initiating extraction process for path ${filePath}`); @@ -111,7 +113,7 @@ export const extractCoverFromFile2 = async ( return { name: fileNameWithoutExtension, path: filePath, - fileSize: size, + fileSize, extension: path.extname(filePath), cover: { filePath: path.relative(process.cwd(),renditionPath),