frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

🔧 Refactor for docker-compose

Browse Source
This commit is contained in:
Rishi Ghan
2024-05-24 14:22:59 -04:00
parent e01421f17b
commit e61ecb1143
9 changed files with 270 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
config/redis.config.ts
View File

@@ -1,14 +1,14 @@
// Import the Redis library // Import the Redis library
import Redis from "ioredis"; import IORedis from "ioredis";
// Environment variable for Redis URI // Environment variable for Redis URI
const redisURI = process.env.REDIS_URI || "redis://localhost:6379"; const redisURI = process.env.REDIS_URI || "redis://localhost:6379";
console.log(`process.env.REDIS_URI is ${process.env.REDIS_URI}`) console.log(`process.env.REDIS_URI is ${process.env.REDIS_URI}`);
// Creating the publisher client // Creating the publisher client
const pubClient = new Redis(redisURI); const pubClient = new IORedis(redisURI);
// Creating the subscriber client // Creating the subscriber client
const subClient = new Redis(redisURI); const subClient = new IORedis(redisURI);
// Handle connection events for the publisher // Handle connection events for the publisher
pubClient.on("connect", () => { pubClient.on("connect", () => {

1
docker-compose.env
View File

@@ -6,6 +6,7 @@ SERVICEDIR=dist/services
COMICS_DIRECTORY=/Users/rishi/work/threetwo-core-service/comics COMICS_DIRECTORY=/Users/rishi/work/threetwo-core-service/comics
USERDATA_DIRECTORY=/Users/rishi/work/threetwo-core-service/userdata USERDATA_DIRECTORY=/Users/rishi/work/threetwo-core-service/userdata
REDIS_URI=redis://redis:6379 REDIS_URI=redis://redis:6379
KAFKA_BROKER=kafka1:9092
ELASTICSEARCH_URI=http://elasticsearch:9200 ELASTICSEARCH_URI=http://elasticsearch:9200
MONGO_URI=mongodb://db:27017/threetwo MONGO_URI=mongodb://db:27017/threetwo
UNRAR_BIN_PATH=/opt/homebrew/bin/unrar UNRAR_BIN_PATH=/opt/homebrew/bin/unrar

46
docker-compose.yml
View File

@@ -23,8 +23,8 @@ services:
- db - db
- redis - redis
- elasticsearch - elasticsearch
- kafka - kafka1
- zookeeper - zoo1
environment: environment:
name: core-services name: core-services
SERVICES: api,library,imagetransformation,opds,search,settings,jobqueue,socket,torrentjobs SERVICES: api,library,imagetransformation,opds,search,settings,jobqueue,socket,torrentjobs
@@ -35,28 +35,43 @@ services:
networks: networks:
- proxy - proxy
zookeeper: zoo1:
image: zookeeper:latest image: confluentinc/cp-zookeeper:7.3.2
container_name: zookeeper hostname: zoo1
container_name: zoo1
ports: ports:
- "2181:2181" - "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_SERVERS: zoo1:2888:3888
networks: networks:
- proxy - proxy
kafka: kafka1:
image: apache/kafka:latest image: confluentinc/cp-kafka:7.3.2
container_name: kafka hostname: kafka1
container_name: kafka1
ports: ports:
- "9092:9092" - "9092:9092"
- "29092:29092"
- "9999:9999"
environment: environment:
KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9093,OUTSIDE://localhost:9092 KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1} :9092,DOCKER://host.docker.internal:29092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
KAFKA_LISTENERS: INSIDE://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
volumes: KAFKA_BROKER_ID: 1
- /var/run/docker.sock:/var/run/docker.sock KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state. change.logger=INFO"
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_JMX_PORT: 9999
KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
depends_on: depends_on:
- zookeeper - zoo1
networks: networks:
- proxy - proxy
@@ -73,6 +88,7 @@ services:
redis: redis:
image: "bitnami/redis:latest" image: "bitnami/redis:latest"
container_name: redis container_name: redis
hostname: redis
environment: environment:
ALLOW_EMPTY_PASSWORD: "yes" ALLOW_EMPTY_PASSWORD: "yes"
networks: networks:

186
package-lock.json generated
View File

@@ -31,7 +31,6 @@
"http-response-stream": "^1.0.9", "http-response-stream": "^1.0.9",
"image-js": "^0.34.0", "image-js": "^0.34.0",
"imghash": "^0.0.9", "imghash": "^0.0.9",
"ioredis": "^5.4.1",
"jsdom": "^21.1.0", "jsdom": "^21.1.0",
"klaw": "^4.1.0", "klaw": "^4.1.0",
"leven": "^3.1.0", "leven": "^3.1.0",
@@ -48,6 +47,7 @@
"nats": "^1.3.2", "nats": "^1.3.2",
"opds-extra": "^3.0.10", "opds-extra": "^3.0.10",
"p7zip-threetwo": "^1.0.4", "p7zip-threetwo": "^1.0.4",
"redis": "^4.6.14",
"sanitize-filename-ts": "^1.0.2", "sanitize-filename-ts": "^1.0.2",
"sharp": "^0.33.3", "sharp": "^0.33.3",
"threetwo-ui-typings": "^1.0.14", "threetwo-ui-typings": "^1.0.14",
@@ -63,6 +63,7 @@
"eslint-plugin-import": "^2.20.2", "eslint-plugin-import": "^2.20.2",
"eslint-plugin-prefer-arrow": "^1.2.2", "eslint-plugin-prefer-arrow": "^1.2.2",
"install": "^0.13.0", "install": "^0.13.0",
"ioredis": "^5.4.1",
"jest": "^29.5.0", "jest": "^29.5.0",
"jest-cli": "^29.5.0", "jest-cli": "^29.5.0",
"moleculer-repl": "^0.7.0", "moleculer-repl": "^0.7.0",
@@ -2656,6 +2657,64 @@
"resolved": "https://registry.npmjs.org/@npcz/magic/-/magic-1.3.14.tgz", "resolved": "https://registry.npmjs.org/@npcz/magic/-/magic-1.3.14.tgz",
"integrity": "sha512-Jt+fjEVAVoDJh9N+nrQ/IQSC6MFLpIDag8VXxvdVGGG5mrGK2HH4X5KqC9zgzb20fqk2vBM9g2QzyczylKVvqg==" "integrity": "sha512-Jt+fjEVAVoDJh9N+nrQ/IQSC6MFLpIDag8VXxvdVGGG5mrGK2HH4X5KqC9zgzb20fqk2vBM9g2QzyczylKVvqg=="
}, },
"node_modules/@redis/bloom": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/client": {
"version": "1.5.16",
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.16.tgz",
"integrity": "sha512-X1a3xQ5kEMvTib5fBrHKh6Y+pXbeKXqziYuxOUo1ojQNECg4M5Etd1qqyhMap+lFUOAh8S7UYevgJHOm4A+NOg==",
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
"yallist": "4.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/@redis/client/node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"node_modules/@redis/graph": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
"integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/json": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz",
"integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/search": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz",
"integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/time-series": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz",
"integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@root/walk": { "node_modules/@root/walk": {
"version": "1.1.0", "version": "1.1.0",
"resolved": "https://registry.npmjs.org/@root/walk/-/walk-1.1.0.tgz", "resolved": "https://registry.npmjs.org/@root/walk/-/walk-1.1.0.tgz",
@@ -4623,58 +4682,6 @@
"url": "https://github.com/sponsors/sindresorhus" "url": "https://github.com/sponsors/sindresorhus"
} }
}, },
"node_modules/bullmq": {
"version": "3.15.8",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.15.8.tgz",
"integrity": "sha512-k3uimHGhl5svqD7SEak+iI6c5DxeLOaOXzCufI9Ic0ST3nJr69v71TGR4cXCTXdgCff3tLec5HgoBnfyWjgn5A==",
"dependencies": {
"cron-parser": "^4.6.0",
"glob": "^8.0.3",
"ioredis": "^5.3.2",
"lodash": "^4.17.21",
"msgpackr": "^1.6.2",
"semver": "^7.3.7",
"tslib": "^2.0.0",
"uuid": "^9.0.0"
}
},
"node_modules/bullmq/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/bullmq/node_modules/glob": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
"integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^5.0.1",
"once": "^1.3.0"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/bullmq/node_modules/minimatch": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
"integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=10"
}
},
"node_modules/bytes": { "node_modules/bytes": {
"version": "3.1.2", "version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -7102,6 +7109,14 @@
"url": "https://github.com/sponsors/ljharb" "url": "https://github.com/sponsors/ljharb"
} }
}, },
"node_modules/generic-pool": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
"integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
"engines": {
"node": ">= 4"
}
},
"node_modules/gensync": { "node_modules/gensync": {
"version": "1.0.0-beta.2", "version": "1.0.0-beta.2",
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -10094,6 +10109,58 @@
"node": ">= 10.x.x" "node": ">= 10.x.x"
} }
}, },
"node_modules/moleculer-bullmq/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/moleculer-bullmq/node_modules/bullmq": {
"version": "3.15.8",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.15.8.tgz",
"integrity": "sha512-k3uimHGhl5svqD7SEak+iI6c5DxeLOaOXzCufI9Ic0ST3nJr69v71TGR4cXCTXdgCff3tLec5HgoBnfyWjgn5A==",
"dependencies": {
"cron-parser": "^4.6.0",
"glob": "^8.0.3",
"ioredis": "^5.3.2",
"lodash": "^4.17.21",
"msgpackr": "^1.6.2",
"semver": "^7.3.7",
"tslib": "^2.0.0",
"uuid": "^9.0.0"
}
},
"node_modules/moleculer-bullmq/node_modules/glob": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
"integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^5.0.1",
"once": "^1.3.0"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/moleculer-bullmq/node_modules/minimatch": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
"integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=10"
}
},
"node_modules/moleculer-db": { "node_modules/moleculer-db": {
"version": "0.8.24", "version": "0.8.24",
"resolved": "https://registry.npmjs.org/moleculer-db/-/moleculer-db-0.8.24.tgz", "resolved": "https://registry.npmjs.org/moleculer-db/-/moleculer-db-0.8.24.tgz",
@@ -14076,6 +14143,19 @@
"recursive-watch": "bin.js" "recursive-watch": "bin.js"
} }
}, },
"node_modules/redis": {
"version": "4.6.14",
"resolved": "https://registry.npmjs.org/redis/-/redis-4.6.14.tgz",
"integrity": "sha512-GrNg/e33HtsQwNXL7kJT+iNFPSwE1IPmd7wzV3j4f2z0EYxZfZE7FVTmUysgAtqQQtg5NXF5SNLR9OdO/UHOfw==",
"dependencies": {
"@redis/bloom": "1.2.0",
"@redis/client": "1.5.16",
"@redis/graph": "1.1.1",
"@redis/json": "1.0.6",
"@redis/search": "1.1.6",
"@redis/time-series": "1.0.5"
}
},
"node_modules/redis-errors": { "node_modules/redis-errors": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",

3
package.json
View File

@@ -27,6 +27,7 @@
"eslint-plugin-import": "^2.20.2", "eslint-plugin-import": "^2.20.2",
"eslint-plugin-prefer-arrow": "^1.2.2", "eslint-plugin-prefer-arrow": "^1.2.2",
"install": "^0.13.0", "install": "^0.13.0",
"ioredis": "^5.4.1",
"jest": "^29.5.0", "jest": "^29.5.0",
"jest-cli": "^29.5.0", "jest-cli": "^29.5.0",
"moleculer-repl": "^0.7.0", "moleculer-repl": "^0.7.0",
@@ -61,7 +62,6 @@
"http-response-stream": "^1.0.9", "http-response-stream": "^1.0.9",
"image-js": "^0.34.0", "image-js": "^0.34.0",
"imghash": "^0.0.9", "imghash": "^0.0.9",
"ioredis": "^5.4.1",
"jsdom": "^21.1.0", "jsdom": "^21.1.0",
"klaw": "^4.1.0", "klaw": "^4.1.0",
"leven": "^3.1.0", "leven": "^3.1.0",
@@ -78,6 +78,7 @@
"nats": "^1.3.2", "nats": "^1.3.2",
"opds-extra": "^3.0.10", "opds-extra": "^3.0.10",
"p7zip-threetwo": "^1.0.4", "p7zip-threetwo": "^1.0.4",
"redis": "^4.6.14",
"sanitize-filename-ts": "^1.0.2", "sanitize-filename-ts": "^1.0.2",
"sharp": "^0.33.3", "sharp": "^0.33.3",
"threetwo-ui-typings": "^1.0.14", "threetwo-ui-typings": "^1.0.14",

8
scripts/start.sh
View File

@@ -1,21 +1,25 @@
#!/bin/bash #!/bin/bash
# Check if the first argument is 'dev', use ts-node; otherwise, use node echo "Starting script with mode: $MODE"
MODE=$1
# Extract the host and port from MONGO_URI # Extract the host and port from MONGO_URI
HOST_PORT=$(echo $MONGO_URI | sed -e 's/mongodb:\/\///' -e 's/\/.*$//') HOST_PORT=$(echo $MONGO_URI | sed -e 's/mongodb:\/\///' -e 's/\/.*$//')
# Assuming the script is called from the project root # Assuming the script is called from the project root
PROJECT_ROOT=$(pwd) PROJECT_ROOT=$(pwd)
echo "Project root: $PROJECT_ROOT"
CONFIG_PATH="$PROJECT_ROOT/moleculer.config.ts" CONFIG_PATH="$PROJECT_ROOT/moleculer.config.ts"
echo "Configuration path: $CONFIG_PATH"
# Set the correct path for moleculer-runner based on the mode # Set the correct path for moleculer-runner based on the mode
if [ "$MODE" == "dev" ]; then if [ "$MODE" == "dev" ]; then
# For development: use ts-node # For development: use ts-node
MOLECULER_RUNNER="ts-node $PROJECT_ROOT/node_modules/moleculer/bin/moleculer-runner.js --hot --repl --config $CONFIG_PATH $PROJECT_ROOT/services/**/*.service.ts" MOLECULER_RUNNER="ts-node $PROJECT_ROOT/node_modules/moleculer/bin/moleculer-runner.js --hot --repl --config $CONFIG_PATH $PROJECT_ROOT/services/**/*.service.ts"
echo "Moleculer Runner for dev: $MOLECULER_RUNNER"
else else
# For production: direct node execution of the compiled JavaScript # For production: direct node execution of the compiled JavaScript
MOLECULER_RUNNER="moleculer-runner --config $PROJECT_ROOT/dist/moleculer.config.js $PROJECT_ROOT/dist/services/**/*.service.js" MOLECULER_RUNNER="moleculer-runner --config $PROJECT_ROOT/dist/moleculer.config.js $PROJECT_ROOT/dist/services/**/*.service.js"
echo "Moleculer Runner for prod: $MOLECULER_RUNNER"
fi fi
# Run wait-for-it, then start the application # Run wait-for-it, then start the application

34
services/jobqueue.service.ts
View File

@@ -11,7 +11,6 @@ import {
} from "../utils/uncompression.utils"; } from "../utils/uncompression.utils";
import { isNil, isUndefined } from "lodash"; import { isNil, isUndefined } from "lodash";
import { pubClient } from "../config/redis.config"; import { pubClient } from "../config/redis.config";
import IORedis from 'ioredis';
import path from "path"; import path from "path";
const { MoleculerError } = require("moleculer").Errors; const { MoleculerError } = require("moleculer").Errors;
@@ -22,9 +21,10 @@ export default class JobQueueService extends Service {
name: "jobqueue", name: "jobqueue",
hooks: {}, hooks: {},
mixins: [DbMixin("comics", Comic), BullMqMixin], mixins: [DbMixin("comics", Comic), BullMqMixin],
settings: { settings: {
bullmq: { bullmq: {
client: new IORedis(process.env.REDIS_URI, { maxRetriesPerRequest: null }), client: pubClient,
}, },
}, },
actions: { actions: {
@@ -57,20 +57,24 @@ export default class JobQueueService extends Service {
handler: async ( handler: async (
ctx: Context<{ action: string; description: string }> ctx: Context<{ action: string; description: string }>
) => { ) => {
const { action, description } = ctx.params; try {
// Enqueue the job const { action, description } = ctx.params;
const job = await this.localQueue( // Enqueue the job
ctx, const job = await this.localQueue(
action, ctx,
ctx.params, action,
{ {},
priority: 10, {
} priority: 10,
); }
console.log(`Job ${job.id} enqueued`); );
console.log(`${description}`); console.log(`Job ${job.id} enqueued`);
console.log(`${description}`);
return job.id; return job.id;
} catch (error) {
console.error("Failed to enqueue job:", error);
}
}, },
}, },

142
services/library.service.ts
View File

@@ -165,78 +165,52 @@ export default class LibraryService extends Service {
}, },
newImport: { newImport: {
rest: "POST /newImport", rest: "POST /newImport",
// params: {}, async handler(ctx) {
async handler( const { sessionId } = ctx.params;
ctx: Context<{
extractionOptions?: any;
sessionId: string;
}>
) {
try { try {
// Get params to be passed to the import jobs // Initialize Redis counters once at the start of the import
const { sessionId } = ctx.params; await pubClient.set("completedJobCount", 0);
// 1. Walk the Source folder await pubClient.set("failedJobCount", 0);
klaw(path.resolve(COMICS_DIRECTORY))
// 1.1 Filter on .cb* extensions // Convert klaw to use a promise-based approach for better flow control
.pipe( const files = await this.getComicFiles(
through2.obj(function (item, enc, next) { COMICS_DIRECTORY
let fileExtension = path.extname( );
item.path for (const file of files) {
); console.info(
if ( "Found a file at path:",
[".cbz", ".cbr", ".cb7"].includes( file.path
fileExtension );
) const comicExists = await Comic.exists({
) { "rawFileDetails.name": path.basename(
this.push(item); file.path,
} path.extname(file.path)
next(); ),
})
)
// 1.2 Pipe filtered results to the next step
// Enqueue the job in the queue
.on("data", async (item) => {
console.info(
"Found a file at path: %s",
item.path
);
let comicExists = await Comic.exists({
"rawFileDetails.name": `${path.basename(
item.path,
path.extname(item.path)
)}`,
});
if (!comicExists) {
// 2.1 Reset the job counters in Redis
await pubClient.set(
"completedJobCount",
0
);
await pubClient.set(
"failedJobCount",
0
);
// 2.2 Send the extraction job to the queue
this.broker.call("jobqueue.enqueue", {
fileObject: {
filePath: item.path,
fileSize: item.stats.size,
},
sessionId,
importType: "new",
action: "enqueue.async",
});
} else {
console.log(
"Comic already exists in the library."
);
}
})
.on("end", () => {
console.log("All files traversed.");
}); });
if (!comicExists) {
// Send the extraction job to the queue
await this.broker.call("jobqueue.enqueue", {
fileObject: {
filePath: file.path,
fileSize: file.stats.size,
},
sessionId,
importType: "new",
action: "enqueue.async",
});
} else {
console.log(
"Comic already exists in the library."
);
}
}
console.log("All files traversed.");
} catch (error) { } catch (error) {
console.log(error); console.error(
"Error during newImport processing:",
error
);
} }
}, },
}, },
@@ -821,7 +795,35 @@ export default class LibraryService extends Service {
}, },
}, },
}, },
methods: {}, methods: {
// Method to walk the directory and filter comic files
getComicFiles: (directory) => {
return new Promise((resolve, reject) => {
const files = [];
klaw(directory)
.pipe(
through2.obj(function (item, enc, next) {
const fileExtension = path.extname(
item.path
);
if (
[".cbz", ".cbr", ".cb7"].includes(
fileExtension
)
) {
this.push(item);
}
next();
})
)
.on("data", (item) => {
files.push(item);
})
.on("end", () => resolve(files))
.on("error", (err) => reject(err));
});
},
},
}); });
} }
} }

3
services/torrentjobs.service.ts
View File

@@ -10,7 +10,6 @@ import { DbMixin } from "../mixins/db.mixin";
import Comic from "../models/comic.model"; import Comic from "../models/comic.model";
import BullMqMixin from "moleculer-bullmq"; import BullMqMixin from "moleculer-bullmq";
const { MoleculerError } = require("moleculer").Errors; const { MoleculerError } = require("moleculer").Errors;
import IORedis from 'ioredis';
export default class ImageTransformation extends Service { export default class ImageTransformation extends Service {
// @ts-ignore // @ts-ignore
@@ -24,7 +23,7 @@ export default class ImageTransformation extends Service {
mixins: [DbMixin("comics", Comic), BullMqMixin], mixins: [DbMixin("comics", Comic), BullMqMixin],
settings: { settings: {
bullmq: { bullmq: {
client: new IORedis(process.env.REDIS_URI), client: process.env.REDIS_URI,
}, },
}, },
hooks: {}, hooks: {},