From 9a41f340991f589bd69e5e518b8c49b3c9c95592 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Thu, 8 Dec 2022 11:15:10 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=8F=97=EF=B8=8F=20Massive=20refactoring?= =?UTF-8?q?=20around=20uncompression=20and=20resizing=20methods?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 449 ++++++++++++++++---------------- package.json | 2 +- services/importqueue.service.ts | 52 +++- services/library.service.ts | 8 +- services/socket.service.ts | 4 + utils/uncompression.utils.ts | 13 +- 6 files changed, 282 insertions(+), 246 deletions(-) diff --git a/package-lock.json b/package-lock.json index 74dc798..f7992e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,7 +34,7 @@ "lodash": "^4.17.21", "mkdirp": "^0.5.5", "moleculer": "^0.14.23", - "moleculer-bull": "github:rishighan/moleculer-bull", + "moleculer-bull": "github:rishighan/moleculer-bull#1.0.0", "moleculer-db": "^0.8.17", "moleculer-db-adapter-mongo": "^0.4.7", "moleculer-db-adapter-mongoose": "^0.8.12", @@ -890,6 +890,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -1377,6 +1382,78 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.2.0.tgz", + "integrity": "sha512-Z9LFPzfoJi4mflGWV+rv7o7ZbMU5oAU9VmzCgL240KnqDW65Y2HFCT3MW06/ITJSnbVLacmcEJA8phywK7JinQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.2.0.tgz", + "integrity": "sha512-vq0tT8sjZsy4JdSqmadWVw6f66UXqUCabLmUVHZwUFzMgtgoIIQjT4VVRHKvlof3P/dMCkbMJ5hB1oJ9OWHaaw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.2.0.tgz", + "integrity": "sha512-SaJ3Qq4lX9Syd2xEo9u3qPxi/OB+5JO/ngJKK97XDpa1C587H9EWYO6KD8995DAjSinWvdHKRrCOXVUC5fvGOg==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.2.0.tgz", + "integrity": "sha512-hlxxLdRmPyq16QCutUtP8Tm6RDWcyaLsRssaHROatgnkOxdleMTgetf9JsdncL8vLh7FVy/RN9i3XR5dnb9cRA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.2.0.tgz", + "integrity": "sha512-94y5PJrSOqUNcFKmOl7z319FelCLAE0rz/jPCWS+UtdMZvpa4jrQd+cJPQCLp2Fes1yAW/YUQj/Di6YVT3c3Iw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.2.0.tgz", + "integrity": "sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2672,14 +2749,14 @@ } }, "node_modules/bull": { - "version": "4.8.1", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.8.1.tgz", - "integrity": "sha512-ojH5AfOchKQsQwwE+thViS1pMpvREGC+Ov1+3HXsQqn5Q27ZSGkgMriMqc6c9J9rvQ/+D732pZE+TN1+2LRWVg==", + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/bull/-/bull-4.10.2.tgz", + "integrity": "sha512-xa65xtWjQsLqYU/eNaXxq9VRG8xd6qNsQEjR7yjYuae05xKrzbVMVj2QgrYsTMmSs/vsqJjHqHSRRiW1+IkGXQ==", "dependencies": { "cron-parser": "^4.2.1", "debuglog": "^1.0.0", "get-port": "^5.1.1", - "ioredis": "^4.28.5", + "ioredis": "^5.0.0", "lodash": "^4.17.21", "msgpackr": "^1.5.2", "p-timeout": "^3.2.0", @@ -2687,7 +2764,7 @@ "uuid": "^8.3.0" }, "engines": { - "node": ">=10.1" + "node": ">=12" } }, "node_modules/bytes": { @@ -3143,14 +3220,14 @@ "peer": true }, "node_modules/cron-parser": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.3.0.tgz", - "integrity": "sha512-mK6qJ6k9Kn0/U7Cv6LKQnReUW3GqAW4exgwmHJGb3tPgcy0LrS+PeqxPPiwL8uW/4IJsMsCZrCc4vf1nnXMjzA==", + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.0.tgz", + "integrity": "sha512-BdAELR+MCT2ZWsIBhZKDuUqIUCBjHHulPJnm53OfdRLA4EWBjva3R+KM5NeidJuGsNXdEcZkjC7SCnkW5rAFSA==", "dependencies": { - "luxon": "^1.28.0" + "luxon": "^3.1.0" }, "engines": { - "node": ">=0.8" + "node": ">=12.0.0" } }, "node_modules/cron-validate": { @@ -3255,7 +3332,7 @@ "node_modules/debuglog": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=", + "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==", "engines": { "node": "*" } @@ -3759,9 +3836,9 @@ } }, "node_modules/engine.io": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.2.0.tgz", - "integrity": "sha512-4KzwW3F3bk+KlzSOY57fj/Jx6LyRQ1nbcyIadehl+AnXjKT7gDO0ORdRi/84ixvMKTym6ZKuxvbzN62HDDU1Lg==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.2.1.tgz", + "integrity": "sha512-ECceEFcAaNRybd3lsGQKas3ZlMVjN3cyWwMP25D2i0zWfyiytVbTpRPa34qrr+FHddtpBVOmq4H/DCv1O0lZRA==", "dependencies": { "@types/cookie": "^0.4.1", "@types/cors": "^2.8.12", @@ -5539,30 +5616,36 @@ } }, "node_modules/ioredis": { - "version": "4.28.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", - "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", + "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", "dependencies": { + "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", - "debug": "^4.3.1", - "denque": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.0.1", "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", "lodash.isarguments": "^3.1.0", - "p-map": "^2.1.0", - "redis-commands": "1.7.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" }, "engines": { - "node": ">=6" + "node": ">=12.22.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/ioredis" } }, + "node_modules/ioredis/node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/ip-regex": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", @@ -7345,11 +7428,6 @@ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" }, - "node_modules/lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" - }, "node_modules/lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", @@ -7424,11 +7502,11 @@ } }, "node_modules/luxon": { - "version": "1.28.0", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", - "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.1.1.tgz", + "integrity": "sha512-Ah6DloGmvseB/pX1cAmjbFvyU/pKuwQMQqz7d0yvuDlVYLTs2WeDHQMpC8tGjm1da+BriHROW/OEIT/KfYg6xw==", "engines": { - "node": "*" + "node": ">=12" } }, "node_modules/make-dir": { @@ -7936,11 +8014,11 @@ } }, "node_modules/moleculer-bull": { - "version": "0.2.8", - "resolved": "git+ssh://git@github.com/rishighan/moleculer-bull.git#7e3c245b5a2776c0d255a013bd121c79eeafe923", + "version": "0.2.9", + "resolved": "git+ssh://git@github.com/rishighan/moleculer-bull.git#487020c3f3b4879bbf1bb40c75f19a259d17dd59", "license": "MIT", "dependencies": { - "bull": "^4.8.0", + "bull": "^4.10.2", "lodash": "^4.17.21" }, "engines": { @@ -8255,103 +8333,34 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/msgpackr": { - "version": "1.5.6", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.5.6.tgz", - "integrity": "sha512-Y1Ia1AYKcz30JOAUyyC0jCicI7SeP8NK+SVCGZIeLg2oQs28wSwW2GbHXktk4ZZmrq9/v2jU0JAbvbp2d1ewpg==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.1.tgz", + "integrity": "sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==", "optionalDependencies": { - "msgpackr-extract": "^1.1.4" + "msgpackr-extract": "^2.2.0" } }, "node_modules/msgpackr-extract": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-1.1.4.tgz", - "integrity": "sha512-WQbHvsThprXh+EqZYy+SQFEs7z6bNM7a0vgirwUfwUcphWGT2mdPcpyLCNiRsN6w5q5VKJUMblHY+tNEyceb9Q==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz", + "integrity": "sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==", "hasInstallScript": true, "optional": true, "dependencies": { - "node-gyp-build-optional-packages": "^4.3.2" + "node-gyp-build-optional-packages": "5.0.3" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" }, "optionalDependencies": { - "msgpackr-extract-darwin-arm64": "1.1.0", - "msgpackr-extract-darwin-x64": "1.1.0", - "msgpackr-extract-linux-arm": "1.1.0", - "msgpackr-extract-linux-arm64": "1.1.0", - "msgpackr-extract-linux-x64": "1.1.0", - "msgpackr-extract-win32-x64": "1.1.0" + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "2.2.0", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "2.2.0", + "@msgpackr-extract/msgpackr-extract-linux-arm": "2.2.0", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "2.2.0", + "@msgpackr-extract/msgpackr-extract-linux-x64": "2.2.0", + "@msgpackr-extract/msgpackr-extract-win32-x64": "2.2.0" } }, - "node_modules/msgpackr-extract-darwin-arm64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-1.1.0.tgz", - "integrity": "sha512-s1kHoT12tS2cCQOv+Wl3I+/cYNJXBPtwQqGA+dPYoXmchhXiE0Nso+BIfvQ5PxbmAyjj54Q5o7PnLTqVquNfZA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/msgpackr-extract-darwin-x64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-1.1.0.tgz", - "integrity": "sha512-yx/H/i12IKg4eWGu/eKdKzJD4jaYvvujQSaVmeOMCesbSQnWo5X6YR9TFjoiNoU9Aexk1KufzL9gW+1DozG1yw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/msgpackr-extract-linux-arm": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-1.1.0.tgz", - "integrity": "sha512-0VvSCqi12xpavxl14gMrauwIzHqHbmSChUijy/uo3mpjB1Pk4vlisKpZsaOZvNJyNKj0ACi5jYtbWnnOd7hYGw==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/msgpackr-extract-linux-arm64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-1.1.0.tgz", - "integrity": "sha512-AxFle3fHNwz2V4CYDIGFxI6o/ZuI0lBKg0uHI8EcCMUmDE5mVAUWYge5WXmORVvb8sVWyVgFlmi3MTu4Ve6tNQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/msgpackr-extract-linux-x64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-1.1.0.tgz", - "integrity": "sha512-O+XoyNFWpdB8oQL6O/YyzffPpmG5rTNrr1nKLW70HD2ENJUhcITzbV7eZimHPzkn8LAGls1tBaMTHQezTBpFOw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/msgpackr-extract-win32-x64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-1.1.0.tgz", - "integrity": "sha512-6AJdM5rNsL4yrskRfhujVSPEd6IBpgvsnIT/TPowKNLQ62iIdryizPY2PJNFiW3AJcY249AHEiDBXS1cTDPxzA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ] - }, "node_modules/nanoclone": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/nanoclone/-/nanoclone-0.2.1.tgz", @@ -8470,14 +8479,14 @@ } }, "node_modules/node-gyp-build-optional-packages": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-4.3.2.tgz", - "integrity": "sha512-P5Ep3ISdmwcCkZIaBaQamQtWAG0facC89phWZgi5Z3hBU//J6S48OIvyZWSPPf6yQMklLZiqoosWAZUj7N+esA==", + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz", + "integrity": "sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==", "optional": true, "bin": { - "node-gyp-build-optional": "optional.js", "node-gyp-build-optional-packages": "bin.js", - "node-gyp-build-test": "build-test.js" + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" } }, "node_modules/node-int64": { @@ -11223,14 +11232,6 @@ "node": ">=4" } }, - "node_modules/p-map": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", - "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", - "engines": { - "node": ">=6" - } - }, "node_modules/p-timeout": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", @@ -14845,6 +14846,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -15212,6 +15218,42 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-2.2.0.tgz", + "integrity": "sha512-Z9LFPzfoJi4mflGWV+rv7o7ZbMU5oAU9VmzCgL240KnqDW65Y2HFCT3MW06/ITJSnbVLacmcEJA8phywK7JinQ==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-2.2.0.tgz", + "integrity": "sha512-vq0tT8sjZsy4JdSqmadWVw6f66UXqUCabLmUVHZwUFzMgtgoIIQjT4VVRHKvlof3P/dMCkbMJ5hB1oJ9OWHaaw==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-2.2.0.tgz", + "integrity": "sha512-SaJ3Qq4lX9Syd2xEo9u3qPxi/OB+5JO/ngJKK97XDpa1C587H9EWYO6KD8995DAjSinWvdHKRrCOXVUC5fvGOg==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-2.2.0.tgz", + "integrity": "sha512-hlxxLdRmPyq16QCutUtP8Tm6RDWcyaLsRssaHROatgnkOxdleMTgetf9JsdncL8vLh7FVy/RN9i3XR5dnb9cRA==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-2.2.0.tgz", + "integrity": "sha512-94y5PJrSOqUNcFKmOl7z319FelCLAE0rz/jPCWS+UtdMZvpa4jrQd+cJPQCLp2Fes1yAW/YUQj/Di6YVT3c3Iw==", + "optional": true + }, + "@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-2.2.0.tgz", + "integrity": "sha512-XrC0JzsqQSvOyM3t04FMLO6z5gCuhPE6k4FXuLK5xf52ZbdvcFe1yBmo7meCew9B8G2f0T9iu9t3kfTYRYROgA==", + "optional": true + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -16221,14 +16263,14 @@ "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=" }, "bull": { - "version": "4.8.1", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.8.1.tgz", - "integrity": "sha512-ojH5AfOchKQsQwwE+thViS1pMpvREGC+Ov1+3HXsQqn5Q27ZSGkgMriMqc6c9J9rvQ/+D732pZE+TN1+2LRWVg==", + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/bull/-/bull-4.10.2.tgz", + "integrity": "sha512-xa65xtWjQsLqYU/eNaXxq9VRG8xd6qNsQEjR7yjYuae05xKrzbVMVj2QgrYsTMmSs/vsqJjHqHSRRiW1+IkGXQ==", "requires": { "cron-parser": "^4.2.1", "debuglog": "^1.0.0", "get-port": "^5.1.1", - "ioredis": "^4.28.5", + "ioredis": "^5.0.0", "lodash": "^4.17.21", "msgpackr": "^1.5.2", "p-timeout": "^3.2.0", @@ -16586,11 +16628,11 @@ "peer": true }, "cron-parser": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.3.0.tgz", - "integrity": "sha512-mK6qJ6k9Kn0/U7Cv6LKQnReUW3GqAW4exgwmHJGb3tPgcy0LrS+PeqxPPiwL8uW/4IJsMsCZrCc4vf1nnXMjzA==", + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.7.0.tgz", + "integrity": "sha512-BdAELR+MCT2ZWsIBhZKDuUqIUCBjHHulPJnm53OfdRLA4EWBjva3R+KM5NeidJuGsNXdEcZkjC7SCnkW5rAFSA==", "requires": { - "luxon": "^1.28.0" + "luxon": "^3.1.0" } }, "cron-validate": { @@ -16682,7 +16724,7 @@ "debuglog": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", - "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=" + "integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==" }, "decimal.js": { "version": "10.3.1", @@ -17063,9 +17105,9 @@ } }, "engine.io": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.2.0.tgz", - "integrity": "sha512-4KzwW3F3bk+KlzSOY57fj/Jx6LyRQ1nbcyIadehl+AnXjKT7gDO0ORdRi/84ixvMKTym6ZKuxvbzN62HDDU1Lg==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.2.1.tgz", + "integrity": "sha512-ECceEFcAaNRybd3lsGQKas3ZlMVjN3cyWwMP25D2i0zWfyiytVbTpRPa34qrr+FHddtpBVOmq4H/DCv1O0lZRA==", "requires": { "@types/cookie": "^0.4.1", "@types/cors": "^2.8.12", @@ -18485,21 +18527,26 @@ } }, "ioredis": { - "version": "4.28.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", - "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", + "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", "requires": { + "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", - "debug": "^4.3.1", - "denque": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.0.1", "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", "lodash.isarguments": "^3.1.0", - "p-map": "^2.1.0", - "redis-commands": "1.7.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" + }, + "dependencies": { + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + } } }, "ip-regex": { @@ -19857,11 +19904,6 @@ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" }, - "lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" - }, "lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", @@ -19927,9 +19969,9 @@ } }, "luxon": { - "version": "1.28.0", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", - "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.1.1.tgz", + "integrity": "sha512-Ah6DloGmvseB/pX1cAmjbFvyU/pKuwQMQqz7d0yvuDlVYLTs2WeDHQMpC8tGjm1da+BriHROW/OEIT/KfYg6xw==" }, "make-dir": { "version": "3.1.0", @@ -20293,10 +20335,10 @@ } }, "moleculer-bull": { - "version": "git+ssh://git@github.com/rishighan/moleculer-bull.git#7e3c245b5a2776c0d255a013bd121c79eeafe923", - "from": "moleculer-bull@github:rishighan/moleculer-bull", + "version": "git+ssh://git@github.com/rishighan/moleculer-bull.git#487020c3f3b4879bbf1bb40c75f19a259d17dd59", + "from": "moleculer-bull@rishighan/moleculer-bull#1.0.0", "requires": { - "bull": "^4.8.0", + "bull": "^4.10.2", "lodash": "^4.17.21" } }, @@ -20506,64 +20548,28 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "msgpackr": { - "version": "1.5.6", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.5.6.tgz", - "integrity": "sha512-Y1Ia1AYKcz30JOAUyyC0jCicI7SeP8NK+SVCGZIeLg2oQs28wSwW2GbHXktk4ZZmrq9/v2jU0JAbvbp2d1ewpg==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.8.1.tgz", + "integrity": "sha512-05fT4J8ZqjYlR4QcRDIhLCYKUOHXk7C/xa62GzMKj74l3up9k2QZ3LgFc6qWdsPHl91QA2WLWqWc8b8t7GLNNw==", "requires": { - "msgpackr-extract": "^1.1.4" + "msgpackr-extract": "^2.2.0" } }, "msgpackr-extract": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-1.1.4.tgz", - "integrity": "sha512-WQbHvsThprXh+EqZYy+SQFEs7z6bNM7a0vgirwUfwUcphWGT2mdPcpyLCNiRsN6w5q5VKJUMblHY+tNEyceb9Q==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz", + "integrity": "sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==", "optional": true, "requires": { - "msgpackr-extract-darwin-arm64": "1.1.0", - "msgpackr-extract-darwin-x64": "1.1.0", - "msgpackr-extract-linux-arm": "1.1.0", - "msgpackr-extract-linux-arm64": "1.1.0", - "msgpackr-extract-linux-x64": "1.1.0", - "msgpackr-extract-win32-x64": "1.1.0", - "node-gyp-build-optional-packages": "^4.3.2" + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "2.2.0", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "2.2.0", + "@msgpackr-extract/msgpackr-extract-linux-arm": "2.2.0", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "2.2.0", + "@msgpackr-extract/msgpackr-extract-linux-x64": "2.2.0", + "@msgpackr-extract/msgpackr-extract-win32-x64": "2.2.0", + "node-gyp-build-optional-packages": "5.0.3" } }, - "msgpackr-extract-darwin-arm64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-1.1.0.tgz", - "integrity": "sha512-s1kHoT12tS2cCQOv+Wl3I+/cYNJXBPtwQqGA+dPYoXmchhXiE0Nso+BIfvQ5PxbmAyjj54Q5o7PnLTqVquNfZA==", - "optional": true - }, - "msgpackr-extract-darwin-x64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-1.1.0.tgz", - "integrity": "sha512-yx/H/i12IKg4eWGu/eKdKzJD4jaYvvujQSaVmeOMCesbSQnWo5X6YR9TFjoiNoU9Aexk1KufzL9gW+1DozG1yw==", - "optional": true - }, - "msgpackr-extract-linux-arm": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-1.1.0.tgz", - "integrity": "sha512-0VvSCqi12xpavxl14gMrauwIzHqHbmSChUijy/uo3mpjB1Pk4vlisKpZsaOZvNJyNKj0ACi5jYtbWnnOd7hYGw==", - "optional": true - }, - "msgpackr-extract-linux-arm64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-1.1.0.tgz", - "integrity": "sha512-AxFle3fHNwz2V4CYDIGFxI6o/ZuI0lBKg0uHI8EcCMUmDE5mVAUWYge5WXmORVvb8sVWyVgFlmi3MTu4Ve6tNQ==", - "optional": true - }, - "msgpackr-extract-linux-x64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-1.1.0.tgz", - "integrity": "sha512-O+XoyNFWpdB8oQL6O/YyzffPpmG5rTNrr1nKLW70HD2ENJUhcITzbV7eZimHPzkn8LAGls1tBaMTHQezTBpFOw==", - "optional": true - }, - "msgpackr-extract-win32-x64": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-1.1.0.tgz", - "integrity": "sha512-6AJdM5rNsL4yrskRfhujVSPEd6IBpgvsnIT/TPowKNLQ62iIdryizPY2PJNFiW3AJcY249AHEiDBXS1cTDPxzA==", - "optional": true - }, "nanoclone": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/nanoclone/-/nanoclone-0.2.1.tgz", @@ -20658,9 +20664,9 @@ } }, "node-gyp-build-optional-packages": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-4.3.2.tgz", - "integrity": "sha512-P5Ep3ISdmwcCkZIaBaQamQtWAG0facC89phWZgi5Z3hBU//J6S48OIvyZWSPPf6yQMklLZiqoosWAZUj7N+esA==", + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz", + "integrity": "sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==", "optional": true }, "node-int64": { @@ -22600,11 +22606,6 @@ "p-limit": "^1.1.0" } }, - "p-map": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", - "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" - }, "p-timeout": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", diff --git a/package.json b/package.json index af9034e..840d333 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "lodash": "^4.17.21", "mkdirp": "^0.5.5", "moleculer": "^0.14.23", - "moleculer-bull": "github:rishighan/moleculer-bull", + "moleculer-bull": "github:rishighan/moleculer-bull#1.0.0", "moleculer-db": "^0.8.17", "moleculer-db-adapter-mongo": "^0.4.7", "moleculer-db-adapter-mongoose": "^0.8.12", diff --git a/services/importqueue.service.ts b/services/importqueue.service.ts index 0709811..1d20ed1 100644 --- a/services/importqueue.service.ts +++ b/services/importqueue.service.ts @@ -38,7 +38,7 @@ import { Context, Service, ServiceBroker, ServiceSchema } from "moleculer"; import BullMQMixin, { SandboxedJob } from "moleculer-bull"; import { DbMixin } from "../mixins/db.mixin"; import Comic from "../models/comic.model"; -import { extractFromArchive } from "../utils/uncompression.utils"; +import { extractFromArchive, uncompressEntireArchive } from "../utils/uncompression.utils"; const REDIS_URI = process.env.REDIS_URI || `redis://localhost:6379`; const EventEmitter = require("events"); @@ -57,7 +57,7 @@ export default class QueueService extends Service { settings: { bullmq: { maxStalledCount: 0, - } + }, }, hooks: {}, @@ -68,11 +68,9 @@ export default class QueueService extends Service { console.info("New job received!", job.data); console.info(`Processing queue...`); // extract the cover - const result = await extractFromArchive( job.data.fileObject.filePath ); - const { name, filePath, @@ -87,7 +85,6 @@ export default class QueueService extends Service { const { inferredIssueDetails } = refineQuery( result.name ); - console.log( "Issue metadata inferred: ", JSON.stringify(inferredIssueDetails, null, 2) @@ -126,7 +123,7 @@ export default class QueueService extends Service { }, directconnect: { downloads: [], - } + }, }, }); return { @@ -145,8 +142,29 @@ export default class QueueService extends Service { }; }, }, + "process.uncompressAndResize": { + concurrency: 2, + async process(job: SandboxedJob) { + console.log(``); + return await uncompressEntireArchive(job.data.filePath, job.data.options); + }, + }, }, actions: { + uncompressResize: { + rest: "POST /uncompressResize", + params: {}, + async handler( + ctx: Context<{ + data: { filePath: string; options: any }; + }> + ) { + return await this.createJob( + "process.uncompressAndResize", + ctx.params + ); + }, + }, processImport: { rest: "POST /processImport", params: {}, @@ -164,7 +182,6 @@ export default class QueueService extends Service { rest: "POST /pauseImportQueue", params: {}, handler: async (ctx: Context<{ action: string }>) => { - console.log(ctx.params); switch (ctx.params.action) { case "pause": const foo = await this.getQueue( @@ -183,11 +200,6 @@ export default class QueueService extends Service { } }, }, - unarchiveComicBook: { - rest: "POST /unarchiveComicBook", - params: {}, - handler: async (ctx: Context<{}>) => {}, - }, }, methods: {}, async started(): Promise { @@ -208,19 +220,31 @@ export default class QueueService extends Service { event: "action", args: [{ type: "LS_COVER_EXTRACTED", result: res }], //optional }); - console.info(`Job with the id '${job.id}' completed.`); + console.info(`Import Job with the id '${job.id}' completed.`); } ); await this.getQueue("process.import").on( "stalled", async (job) => { console.warn( - `The job with the id '${job.id} got stalled!` + `Import job '${job.id} stalled!` ); console.log(`${JSON.stringify(job, null, 2)}`); console.log(`is stalled.`); } ); + + await this.getQueue("process.uncompressAndResize").on( + "completed", + async (job, res) => { + await this.broker.call("socket.broadcast", { + namespace: "/", + event: "action", + args: [{ type: "COMICBOOK_EXTRACTION_SUCCESS", result: res }] + }); + console.info(`Uncompression Job ${job.id} completed.`) + } + ); }, }); } diff --git a/services/library.service.ts b/services/library.service.ts index f6c726d..9ff64dd 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -98,10 +98,10 @@ export default class ImportService extends Service { handler: async ( ctx: Context<{ filePath: string; options: any }> ) => { - return await uncompressEntireArchive( - ctx.params.filePath, - ctx.params.options - ); + await broker.call("importqueue.uncompressResize", { + filePath: ctx.params.filePath, + options: ctx.params.options, + }); }, }, newImport: { diff --git a/services/socket.service.ts b/services/socket.service.ts index 164b902..edaf6d8 100644 --- a/services/socket.service.ts +++ b/services/socket.service.ts @@ -56,6 +56,10 @@ export default class SocketService extends Service { {} ); break; + // uncompress archive events + case "COMICBOOK_EXTRACTION_SUCCESS": + console.log(data); + return data; } }, }, diff --git a/utils/uncompression.utils.ts b/utils/uncompression.utils.ts index 7687e1f..123853b 100644 --- a/utils/uncompression.utils.ts +++ b/utils/uncompression.utils.ts @@ -399,7 +399,7 @@ export const uncompressZipArchive = async (filePath: string, options: any) => { mode: 0o2775, }; const { fileNameWithoutExtension } = getFileConstituents(filePath); - const targetDirectory = `${USERDATA_DIRECTORY}/expanded/${fileNameWithoutExtension}`; + const targetDirectory = `${USERDATA_DIRECTORY}/expanded/${options.purpose}/${fileNameWithoutExtension}`; await createDirectory(directoryOptions, targetDirectory); await p7zip.extract(filePath, targetDirectory, [], "", false); @@ -413,7 +413,7 @@ export const uncompressRarArchive = async (filePath: string, options: any) => { }; const { fileNameWithoutExtension, extension } = getFileConstituents(filePath); - const targetDirectory = `${USERDATA_DIRECTORY}/expanded/${fileNameWithoutExtension}`; + const targetDirectory = `${USERDATA_DIRECTORY}/expanded/${options.purpose}/${fileNameWithoutExtension}`; await createDirectory(directoryOptions, targetDirectory); const archive = new Unrar({ @@ -469,9 +469,16 @@ export const resizeImageDirectory = async ( return await Promise.all(resizePromises); }; +/** + * Method that resizes an image in a specified location based on parameters provided + * @param {string} directoryPath + * @param {any} file + * @param {any} options + * @returns {any} + */ export const resizeImage = (directoryPath: string, file: any, options: any) => { const { baseWidth } = options.imageResizeOptions; - const sharpResizeInstance = sharp().resize(baseWidth).toFormat("png"); + const sharpResizeInstance = sharp().resize(baseWidth).toFormat("jpg"); return new Promise((resolve, reject) => { const resizedStream = createReadStream( `${directoryPath}/${file.name}${file.extension}`