frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

🔧 Refactor for docker-compose

Browse Source
This commit is contained in:
Rishi Ghan
2024-05-24 14:22:59 -04:00
parent e01421f17b
commit e61ecb1143
9 changed files with 270 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
config/redis.config.ts
View File

@@ -1,14 +1,14 @@
// Import the Redis library
import Redis from "ioredis";
import IORedis from "ioredis";
// Environment variable for Redis URI
const redisURI = process.env.REDIS_URI || "redis://localhost:6379";
console.log(`process.env.REDIS_URI is ${process.env.REDIS_URI}`)
console.log(`process.env.REDIS_URI is ${process.env.REDIS_URI}`);
// Creating the publisher client
const pubClient = new Redis(redisURI);
const pubClient = new IORedis(redisURI);
// Creating the subscriber client
const subClient = new Redis(redisURI);
const subClient = new IORedis(redisURI);
// Handle connection events for the publisher
pubClient.on("connect", () => {

1
docker-compose.env
View File

@@ -6,6 +6,7 @@ SERVICEDIR=dist/services
COMICS_DIRECTORY=/Users/rishi/work/threetwo-core-service/comics
USERDATA_DIRECTORY=/Users/rishi/work/threetwo-core-service/userdata
REDIS_URI=redis://redis:6379
KAFKA_BROKER=kafka1:9092
ELASTICSEARCH_URI=http://elasticsearch:9200
MONGO_URI=mongodb://db:27017/threetwo
UNRAR_BIN_PATH=/opt/homebrew/bin/unrar

48
docker-compose.yml
View File

@@ -23,8 +23,8 @@ services:
- db
- redis
- elasticsearch
- kafka
- zookeeper
- kafka1
- zoo1
environment:
name: core-services
SERVICES: api,library,imagetransformation,opds,search,settings,jobqueue,socket,torrentjobs
@@ -35,28 +35,43 @@ services:
networks:
- proxy
zookeeper:
image: zookeeper:latest
container_name: zookeeper
zoo1:
image: confluentinc/cp-zookeeper:7.3.2
hostname: zoo1
container_name: zoo1
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_SERVERS: zoo1:2888:3888
networks:
- proxy
kafka:
image: apache/kafka:latest
container_name: kafka
kafka1:
image: confluentinc/cp-kafka:7.3.2
hostname: kafka1
container_name: kafka1
ports:
- "9092:9092"
- "29092:29092"
- "9999:9999"
environment:
KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9093,OUTSIDE://localhost:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
KAFKA_LISTENERS: INSIDE://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
volumes:
- /var/run/docker.sock:/var/run/docker.sock
KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1} :9092,DOCKER://host.docker.internal:29092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,DOCKER:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
KAFKA_BROKER_ID: 1
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state. change.logger=INFO"
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_JMX_PORT: 9999
KAFKA_JMX_HOSTNAME: ${DOCKER_HOST_IP:-127.0.0.1}
KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
depends_on:
- zookeeper
- zoo1
networks:
- proxy
@@ -72,7 +87,8 @@ services:
redis:
image: "bitnami/redis:latest"
container_name: redis
container_name: redis
hostname: redis
environment:
ALLOW_EMPTY_PASSWORD: "yes"
networks:

186
package-lock.json generated
View File

@@ -31,7 +31,6 @@
"http-response-stream": "^1.0.9",
"image-js": "^0.34.0",
"imghash": "^0.0.9",
"ioredis": "^5.4.1",
"jsdom": "^21.1.0",
"klaw": "^4.1.0",
"leven": "^3.1.0",
@@ -48,6 +47,7 @@
"nats": "^1.3.2",
"opds-extra": "^3.0.10",
"p7zip-threetwo": "^1.0.4",
"redis": "^4.6.14",
"sanitize-filename-ts": "^1.0.2",
"sharp": "^0.33.3",
"threetwo-ui-typings": "^1.0.14",
@@ -63,6 +63,7 @@
"eslint-plugin-import": "^2.20.2",
"eslint-plugin-prefer-arrow": "^1.2.2",
"install": "^0.13.0",
"ioredis": "^5.4.1",
"jest": "^29.5.0",
"jest-cli": "^29.5.0",
"moleculer-repl": "^0.7.0",
@@ -2656,6 +2657,64 @@
"resolved": "https://registry.npmjs.org/@npcz/magic/-/magic-1.3.14.tgz",
"integrity": "sha512-Jt+fjEVAVoDJh9N+nrQ/IQSC6MFLpIDag8VXxvdVGGG5mrGK2HH4X5KqC9zgzb20fqk2vBM9g2QzyczylKVvqg=="
},
"node_modules/@redis/bloom": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/client": {
"version": "1.5.16",
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.16.tgz",
"integrity": "sha512-X1a3xQ5kEMvTib5fBrHKh6Y+pXbeKXqziYuxOUo1ojQNECg4M5Etd1qqyhMap+lFUOAh8S7UYevgJHOm4A+NOg==",
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
"yallist": "4.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/@redis/client/node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"node_modules/@redis/graph": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
"integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/json": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz",
"integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/search": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz",
"integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/time-series": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz",
"integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@root/walk": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@root/walk/-/walk-1.1.0.tgz",
@@ -4623,58 +4682,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/bullmq": {
"version": "3.15.8",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.15.8.tgz",
"integrity": "sha512-k3uimHGhl5svqD7SEak+iI6c5DxeLOaOXzCufI9Ic0ST3nJr69v71TGR4cXCTXdgCff3tLec5HgoBnfyWjgn5A==",
"dependencies": {
"cron-parser": "^4.6.0",
"glob": "^8.0.3",
"ioredis": "^5.3.2",
"lodash": "^4.17.21",
"msgpackr": "^1.6.2",
"semver": "^7.3.7",
"tslib": "^2.0.0",
"uuid": "^9.0.0"
}
},
"node_modules/bullmq/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/bullmq/node_modules/glob": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
"integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^5.0.1",
"once": "^1.3.0"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/bullmq/node_modules/minimatch": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
"integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=10"
}
},
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -7102,6 +7109,14 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/generic-pool": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
"integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
"engines": {
"node": ">= 4"
}
},
"node_modules/gensync": {
"version": "1.0.0-beta.2",
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -10094,6 +10109,58 @@
"node": ">= 10.x.x"
}
},
"node_modules/moleculer-bullmq/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/moleculer-bullmq/node_modules/bullmq": {
"version": "3.15.8",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.15.8.tgz",
"integrity": "sha512-k3uimHGhl5svqD7SEak+iI6c5DxeLOaOXzCufI9Ic0ST3nJr69v71TGR4cXCTXdgCff3tLec5HgoBnfyWjgn5A==",
"dependencies": {
"cron-parser": "^4.6.0",
"glob": "^8.0.3",
"ioredis": "^5.3.2",
"lodash": "^4.17.21",
"msgpackr": "^1.6.2",
"semver": "^7.3.7",
"tslib": "^2.0.0",
"uuid": "^9.0.0"
}
},
"node_modules/moleculer-bullmq/node_modules/glob": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
"integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^5.0.1",
"once": "^1.3.0"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/moleculer-bullmq/node_modules/minimatch": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
"integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=10"
}
},
"node_modules/moleculer-db": {
"version": "0.8.24",
"resolved": "https://registry.npmjs.org/moleculer-db/-/moleculer-db-0.8.24.tgz",
@@ -14076,6 +14143,19 @@
"recursive-watch": "bin.js"
}
},
"node_modules/redis": {
"version": "4.6.14",
"resolved": "https://registry.npmjs.org/redis/-/redis-4.6.14.tgz",
"integrity": "sha512-GrNg/e33HtsQwNXL7kJT+iNFPSwE1IPmd7wzV3j4f2z0EYxZfZE7FVTmUysgAtqQQtg5NXF5SNLR9OdO/UHOfw==",
"dependencies": {
"@redis/bloom": "1.2.0",
"@redis/client": "1.5.16",
"@redis/graph": "1.1.1",
"@redis/json": "1.0.6",
"@redis/search": "1.1.6",
"@redis/time-series": "1.0.5"
}
},
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",

3
package.json
View File

@@ -27,6 +27,7 @@
"eslint-plugin-import": "^2.20.2",
"eslint-plugin-prefer-arrow": "^1.2.2",
"install": "^0.13.0",
"ioredis": "^5.4.1",
"jest": "^29.5.0",
"jest-cli": "^29.5.0",
"moleculer-repl": "^0.7.0",
@@ -61,7 +62,6 @@
"http-response-stream": "^1.0.9",
"image-js": "^0.34.0",
"imghash": "^0.0.9",
"ioredis": "^5.4.1",
"jsdom": "^21.1.0",
"klaw": "^4.1.0",
"leven": "^3.1.0",
@@ -78,6 +78,7 @@
"nats": "^1.3.2",
"opds-extra": "^3.0.10",
"p7zip-threetwo": "^1.0.4",
"redis": "^4.6.14",
"sanitize-filename-ts": "^1.0.2",
"sharp": "^0.33.3",
"threetwo-ui-typings": "^1.0.14",

8
scripts/start.sh
View File

@@ -1,21 +1,25 @@
#!/bin/bash
# Check if the first argument is 'dev', use ts-node; otherwise, use node
MODE=$1
echo "Starting script with mode: $MODE"
# Extract the host and port from MONGO_URI
HOST_PORT=$(echo $MONGO_URI | sed -e 's/mongodb:\/\///' -e 's/\/.*$//')
# Assuming the script is called from the project root
PROJECT_ROOT=$(pwd)
echo "Project root: $PROJECT_ROOT"
CONFIG_PATH="$PROJECT_ROOT/moleculer.config.ts"
echo "Configuration path: $CONFIG_PATH"
# Set the correct path for moleculer-runner based on the mode
if [ "$MODE" == "dev" ]; then
# For development: use ts-node
MOLECULER_RUNNER="ts-node $PROJECT_ROOT/node_modules/moleculer/bin/moleculer-runner.js --hot --repl --config $CONFIG_PATH $PROJECT_ROOT/services/**/*.service.ts"
echo "Moleculer Runner for dev: $MOLECULER_RUNNER"
else
# For production: direct node execution of the compiled JavaScript
MOLECULER_RUNNER="moleculer-runner --config $PROJECT_ROOT/dist/moleculer.config.js $PROJECT_ROOT/dist/services/**/*.service.js"
echo "Moleculer Runner for prod: $MOLECULER_RUNNER"
fi
# Run wait-for-it, then start the application

34
services/jobqueue.service.ts
View File

@@ -11,7 +11,6 @@ import {
} from "../utils/uncompression.utils";
import { isNil, isUndefined } from "lodash";
import { pubClient } from "../config/redis.config";
import IORedis from 'ioredis';
import path from "path";
const { MoleculerError } = require("moleculer").Errors;
@@ -22,9 +21,10 @@ export default class JobQueueService extends Service {
name: "jobqueue",
hooks: {},
mixins: [DbMixin("comics", Comic), BullMqMixin],
settings: {
bullmq: {
client: new IORedis(process.env.REDIS_URI, { maxRetriesPerRequest: null }),
client: pubClient,
},
},
actions: {
@@ -57,20 +57,24 @@ export default class JobQueueService extends Service {
handler: async (
ctx: Context<{ action: string; description: string }>
) => {
const { action, description } = ctx.params;
// Enqueue the job
const job = await this.localQueue(
ctx,
action,
ctx.params,
{
priority: 10,
}
);
console.log(`Job ${job.id} enqueued`);
console.log(`${description}`);
try {
const { action, description } = ctx.params;
// Enqueue the job
const job = await this.localQueue(
ctx,
action,
{},
{
priority: 10,
}
);
console.log(`Job ${job.id} enqueued`);
console.log(`${description}`);
return job.id;
return job.id;
} catch (error) {
console.error("Failed to enqueue job:", error);
}
},
},

142
services/library.service.ts
View File

@@ -165,78 +165,52 @@ export default class LibraryService extends Service {
},
newImport: {
rest: "POST /newImport",
// params: {},
async handler(
ctx: Context<{
extractionOptions?: any;
sessionId: string;
}>
) {
async handler(ctx) {
const { sessionId } = ctx.params;
try {
// Get params to be passed to the import jobs
const { sessionId } = ctx.params;
// 1. Walk the Source folder
klaw(path.resolve(COMICS_DIRECTORY))
// 1.1 Filter on .cb* extensions
.pipe(
through2.obj(function (item, enc, next) {
let fileExtension = path.extname(
item.path
);
if (
[".cbz", ".cbr", ".cb7"].includes(
fileExtension
)
) {
this.push(item);
}
next();
})
)
// 1.2 Pipe filtered results to the next step
// Enqueue the job in the queue
.on("data", async (item) => {
console.info(
"Found a file at path: %s",
item.path
);
let comicExists = await Comic.exists({
"rawFileDetails.name": `${path.basename(
item.path,
path.extname(item.path)
)}`,
});
if (!comicExists) {
// 2.1 Reset the job counters in Redis
await pubClient.set(
"completedJobCount",
0
);
await pubClient.set(
"failedJobCount",
0
);
// 2.2 Send the extraction job to the queue
this.broker.call("jobqueue.enqueue", {
fileObject: {
filePath: item.path,
fileSize: item.stats.size,
},
sessionId,
importType: "new",
action: "enqueue.async",
});
} else {
console.log(
"Comic already exists in the library."
);
}
})
.on("end", () => {
console.log("All files traversed.");
// Initialize Redis counters once at the start of the import
await pubClient.set("completedJobCount", 0);
await pubClient.set("failedJobCount", 0);
// Convert klaw to use a promise-based approach for better flow control
const files = await this.getComicFiles(
COMICS_DIRECTORY
);
for (const file of files) {
console.info(
"Found a file at path:",
file.path
);
const comicExists = await Comic.exists({
"rawFileDetails.name": path.basename(
file.path,
path.extname(file.path)
),
});
if (!comicExists) {
// Send the extraction job to the queue
await this.broker.call("jobqueue.enqueue", {
fileObject: {
filePath: file.path,
fileSize: file.stats.size,
},
sessionId,
importType: "new",
action: "enqueue.async",
});
} else {
console.log(
"Comic already exists in the library."
);
}
}
console.log("All files traversed.");
} catch (error) {
console.log(error);
console.error(
"Error during newImport processing:",
error
);
}
},
},
@@ -821,7 +795,35 @@ export default class LibraryService extends Service {
},
},
},
methods: {},
methods: {
// Method to walk the directory and filter comic files
getComicFiles: (directory) => {
return new Promise((resolve, reject) => {
const files = [];
klaw(directory)
.pipe(
through2.obj(function (item, enc, next) {
const fileExtension = path.extname(
item.path
);
if (
[".cbz", ".cbr", ".cb7"].includes(
fileExtension
)
) {
this.push(item);
}
next();
})
)
.on("data", (item) => {
files.push(item);
})
.on("end", () => resolve(files))
.on("error", (err) => reject(err));
});
},
},
});
}
}

3
services/torrentjobs.service.ts
View File

@@ -10,7 +10,6 @@ import { DbMixin } from "../mixins/db.mixin";
import Comic from "../models/comic.model";
import BullMqMixin from "moleculer-bullmq";
const { MoleculerError } = require("moleculer").Errors;
import IORedis from 'ioredis';
export default class ImageTransformation extends Service {
// @ts-ignore
@@ -24,7 +23,7 @@ export default class ImageTransformation extends Service {
mixins: [DbMixin("comics", Comic), BullMqMixin],
settings: {
bullmq: {
client: new IORedis(process.env.REDIS_URI),
client: process.env.REDIS_URI,
},
},
hooks: {},