From ece7b99feb3aa705296f612ef57cedde5c9c2afe Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Sat, 18 Sep 2021 09:26:26 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=AE=F0=9F=8F=BC=E2=80=8D=E2=99=82?= =?UTF-8?q?=EF=B8=8F=20chokidar=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 109 ++++++++++++++++++++++++++++++++++--- package.json | 3 +- queue/importQueue.ts | 2 +- services/api.service.ts | 48 +++++++++++++++- services/import.service.ts | 12 +++- 5 files changed, 160 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 89990ad..5e2cf45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,7 +35,7 @@ "sharp": "^0.28.1", "socket.io": "^4.1.1", "socket.io-stream": "^0.5.3", - "threetwo-ui-typings": "^1.0.4", + "threetwo-ui-typings": "^1.0.5", "typescript": "^3.8.3", "xml2js": "^0.4.23" }, @@ -44,6 +44,7 @@ "@types/unzipper": "^0.10.3", "@typescript-eslint/eslint-plugin": "^2.26.0", "@typescript-eslint/parser": "^2.26.0", + "chokidar": "^3.5.2", "eslint": "^6.8.0", "eslint-plugin-import": "^2.20.2", "eslint-plugin-prefer-arrow": "^1.2.2", @@ -2452,6 +2453,15 @@ "tweetnacl": "^0.14.3" } }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/bitsyntax": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/bitsyntax/-/bitsyntax-0.1.0.tgz", @@ -2783,6 +2793,27 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, + "node_modules/chokidar": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.2.tgz", + "integrity": "sha512-ekGhOnNVPgT77r4K/U3GDhu+FQ2S8TnK/s2KbIGXi0SZWuwkZ2QNyfWdZW+TVfn84DpEP7rLeCt2UI6bJ8GwbQ==", + "dev": true, + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -5551,6 +5582,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/is-boolean-object": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", @@ -10194,6 +10237,18 @@ "node": ">= 6" } }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, "node_modules/realpath-native": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/realpath-native/-/realpath-native-2.0.0.tgz", @@ -11948,9 +12003,9 @@ "dev": true }, "node_modules/threetwo-ui-typings": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.4.tgz", - "integrity": "sha512-Hpke7x19mctO4q8/jSdtT4I08J3zpl5IjXsIC5N6ti7xXf6QAF04tHxhtWsAHgREQzxf05rMXNtjtMlHIKYr8w==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.5.tgz", + "integrity": "sha512-3SxAFXpWWxP3umyO7QkH5s4ySDeXC08qIASrx6M3AkzLJCt2uCqYKumAxM9H78ETasEh0oOXzgusy6BvVMIRYA==", "dependencies": { "typescript": "^4.3.2" } @@ -14856,6 +14911,12 @@ "tweetnacl": "^0.14.3" } }, + "binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true + }, "bitsyntax": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/bitsyntax/-/bitsyntax-0.1.0.tgz", @@ -15123,6 +15184,22 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, + "chokidar": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.2.tgz", + "integrity": "sha512-ekGhOnNVPgT77r4K/U3GDhu+FQ2S8TnK/s2KbIGXi0SZWuwkZ2QNyfWdZW+TVfn84DpEP7rLeCt2UI6bJ8GwbQ==", + "dev": true, + "requires": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + } + }, "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -17319,6 +17396,15 @@ "has-bigints": "^1.0.1" } }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "requires": { + "binary-extensions": "^2.0.0" + } + }, "is-boolean-object": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", @@ -20802,6 +20888,15 @@ "util-deprecate": "^1.0.1" } }, + "readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "requires": { + "picomatch": "^2.2.1" + } + }, "realpath-native": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/realpath-native/-/realpath-native-2.0.0.tgz", @@ -22213,9 +22308,9 @@ "dev": true }, "threetwo-ui-typings": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.4.tgz", - "integrity": "sha512-Hpke7x19mctO4q8/jSdtT4I08J3zpl5IjXsIC5N6ti7xXf6QAF04tHxhtWsAHgREQzxf05rMXNtjtMlHIKYr8w==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/threetwo-ui-typings/-/threetwo-ui-typings-1.0.5.tgz", + "integrity": "sha512-3SxAFXpWWxP3umyO7QkH5s4ySDeXC08qIASrx6M3AkzLJCt2uCqYKumAxM9H78ETasEh0oOXzgusy6BvVMIRYA==", "requires": { "typescript": "^4.3.2" }, diff --git a/package.json b/package.json index 3a61987..3f979af 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "@types/unzipper": "^0.10.3", "@typescript-eslint/eslint-plugin": "^2.26.0", "@typescript-eslint/parser": "^2.26.0", + "chokidar": "^3.5.2", "eslint": "^6.8.0", "eslint-plugin-import": "^2.20.2", "eslint-plugin-prefer-arrow": "^1.2.2", @@ -63,7 +64,7 @@ "sharp": "^0.28.1", "socket.io": "^4.1.1", "socket.io-stream": "^0.5.3", - "threetwo-ui-typings": "^1.0.4", + "threetwo-ui-typings": "^1.0.5", "typescript": "^3.8.3", "xml2js": "^0.4.23" }, diff --git a/queue/importQueue.ts b/queue/importQueue.ts index 619e0b9..097294c 100644 --- a/queue/importQueue.ts +++ b/queue/importQueue.ts @@ -3,7 +3,7 @@ import { logger } from "../utils/logger.utils"; const amqp = require("amqplib/callback_api"); const rabbitUrl = "amqp://localhost"; -export const sendRabbitMQ = (queueName, data) => { +export const sendToRabbitMQ = (queueName, data) => { // connect to local rabbitmq instance amqp.connect(rabbitUrl, (error0, connection) => { if (error0) { diff --git a/services/api.service.ts b/services/api.service.ts index 3115960..ee1ab7b 100644 --- a/services/api.service.ts +++ b/services/api.service.ts @@ -1,5 +1,9 @@ import { Service, ServiceBroker, Context } from "moleculer"; import ApiGateway from "moleculer-web"; +import chokidar from "chokidar"; +import { logger } from "../utils/logger.utils"; +import path from "path"; +import { IExtractionOptions, IFolderData } from "threetwo-ui-typings"; export default class ApiService extends Service { public constructor(broker: ServiceBroker) { super(broker); @@ -72,12 +76,50 @@ export default class ApiService extends Service { options: {}, }, }, - events: { - - }, + events: {}, methods: {}, started(): any { + const fileWatcher = chokidar.watch(path.resolve("./comics"), { + ignored: /(^|[\/\\])\../, // ignore dotfiles + persistent: true, + ignoreInitial: true, + atomic: true, + awaitWriteFinish: { + stabilityThreshold: 2000, + pollInterval: 100, + }, + }); + fileWatcher + .on("add", async (path, stats) => { + logger.info( + `File ${path} has been added with stats: ${JSON.stringify( + stats + )}` + ); + const walkedFolders:IFolderData = await broker.call("import.walkFolders", {basePathToWalk: path}); + const extractionOptions: IExtractionOptions = { + extractTarget: "cover", + targetExtractionFolder: "./userdata/covers", + extractionMode: "single", + paginationOptions: { + pageLimit: 25, + page: 1, + }, + }; + this.broker.call("import.importComicsToDb", {walkedFolders, extractionOptions }); + }) + .on("change", (path, stats) => + logger.info( + `File ${path} has been changed. Stats: ${stats}` + ) + ) + .on("unlink", (path) => + logger.info(`File ${path} has been removed`) + ) + .on("addDir", (path) => + logger.info(`Directory ${path} has been added`) + ); }, }); } diff --git a/services/import.service.ts b/services/import.service.ts index 4de3c66..2ca43bb 100644 --- a/services/import.service.ts +++ b/services/import.service.ts @@ -13,7 +13,7 @@ import { walkFolder } from "../utils/file.utils"; import { convertXMLToJSON } from "../utils/xml.utils"; import https from "https"; import { logger } from "../utils/logger.utils"; -import { sendRabbitMQ } from "../queue/importQueue"; +import { sendToRabbitMQ } from "../queue/importQueue"; import { IExtractComicBookCoverErrorResponse, IExtractedComicBookCoverFile, @@ -126,7 +126,7 @@ export default class ImportService extends Service { {} ); // 3. Send to the queue - sendRabbitMQ( + sendToRabbitMQ( "comicBookCovers", JSON.stringify({ comicBookCoverMetadata, @@ -312,6 +312,14 @@ export default class ImportService extends Service { return await Comic.findById(ctx.params.id); }, }, + axn: { + rest: "GET /axn", + params: {}, + async handler(ctx: Context<{path, stats}>) { + logger.info(ctx.params); + return {"pandurang": "hari"}; + } + }, }, methods: { getComicVineVolumeMetadata: (apiDetailURL) =>