From f4563c12c61f682a1b91f6828ee5fb2b1f8cfd04 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Sun, 12 May 2024 23:35:01 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20Added=20startup=20scripts=20fixi?= =?UTF-8?q?ng=20MongoDB=20timeouts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mixins/db.mixin.ts | 117 ++++++++++------------ package-lock.json | 14 +-- package.json | 6 +- scripts/start.sh | 22 +++++ scripts/wait-for-it.sh | 190 ++++++++++++++++++++++++++++++++++++ services/library.service.ts | 8 +- 6 files changed, 279 insertions(+), 78 deletions(-) create mode 100755 scripts/start.sh create mode 100755 scripts/wait-for-it.sh diff --git a/mixins/db.mixin.ts b/mixins/db.mixin.ts index f8ffedd..c999dcf 100644 --- a/mixins/db.mixin.ts +++ b/mixins/db.mixin.ts @@ -3,72 +3,59 @@ const mkdir = require("mkdirp").sync; const DbService = require("moleculer-db"); export const DbMixin = (collection, model) => { - if (process.env.MONGO_URI) { - const MongooseAdapter = require("moleculer-db-adapter-mongoose"); - console.log("Connecting to MongoDB at", process.env.MONGO_URI); - - const connectWithRetry = ( - adapter, - maxRetries = 5, - interval = 5000, - retries = 0 - ) => { - return adapter - .connect() - .then(() => console.log("MongoDB connected successfully!")) - .catch((err) => { - console.error("MongoDB connection error:", err); - if (retries < maxRetries) { - console.log( - `Retrying MongoDB connection in ${ - interval / 1000 - } seconds...` - ); - setTimeout( - () => - connectWithRetry( - adapter, - maxRetries, - interval, - retries + 1 - ), - interval - ); - } else { - console.error( - "Failed to connect to MongoDB after several attempts." - ); - } - }); - }; - - const adapter = new MongooseAdapter(process.env.MONGO_URI, { - user: process.env.MONGO_INITDB_ROOT_USERNAME, - pass: process.env.MONGO_INITDB_ROOT_PASSWORD, - keepAlive: true, - useNewUrlParser: true, - useUnifiedTopology: true, - }); - - return { - mixins: [DbService], - adapter: adapter, - model, - collection, - started() { - connectWithRetry(this.adapter); - }, - stopped() { - this.adapter - .disconnect() - .then(() => console.log("MongoDB disconnected")) - .catch((err) => - console.error("MongoDB disconnection error:", err) - ); - }, - }; - } else { + if (!process.env.MONGO_URI) { console.log("MONGO_URI not provided, initializing local storage..."); mkdir(path.resolve("./data")); + return { mixins: [DbService] }; // Handle case where no DB URI is provided } + + const MongooseAdapter = require("moleculer-db-adapter-mongoose"); + const adapter = new MongooseAdapter(process.env.MONGO_URI, { + user: process.env.MONGO_INITDB_ROOT_USERNAME, + pass: process.env.MONGO_INITDB_ROOT_PASSWORD, + keepAlive: true, + useNewUrlParser: true, + useUnifiedTopology: true, + }); + + const connectWithRetry = async ( + adapter, + maxRetries = 5, + interval = 5000 + ) => { + for (let retry = 0; retry < maxRetries; retry++) { + try { + await adapter.connect(); + console.log("MongoDB connected successfully!"); + return; + } catch (err) { + console.error("MongoDB connection error:", err); + console.log( + `Retrying MongoDB connection in ${ + interval / 1000 + } seconds...` + ); + await new Promise((resolve) => setTimeout(resolve, interval)); + } + } + console.error("Failed to connect to MongoDB after several attempts."); + }; + + return { + mixins: [DbService], + adapter, + model, + collection, + async started() { + await connectWithRetry(this.adapter); + }, + async stopped() { + try { + await this.adapter.disconnect(); + console.log("MongoDB disconnected"); + } catch (err) { + console.error("MongoDB disconnection error:", err); + } + }, + }; }; diff --git a/package-lock.json b/package-lock.json index 671a70b..d978e66 100644 --- a/package-lock.json +++ b/package-lock.json @@ -38,7 +38,7 @@ "mkdirp": "^0.5.5", "moleculer-bullmq": "^3.0.0", "moleculer-db": "^0.8.23", - "moleculer-db-adapter-mongoose": "^0.9.2", + "moleculer-db-adapter-mongoose": "^0.9.4", "moleculer-io": "^2.2.0", "moleculer-web": "^0.10.5", "mongoosastic-ts": "^6.0.3", @@ -10178,9 +10178,9 @@ } }, "node_modules/moleculer-db-adapter-mongoose": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/moleculer-db-adapter-mongoose/-/moleculer-db-adapter-mongoose-0.9.3.tgz", - "integrity": "sha512-zv77GKBZrAUCnxUESxqyq58t9HUFxB9M0CrnFkxUr87HyqzRcj8A+sAkllFuK8sMBOdgDHSA5HUi7VrchrB2NQ==", + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/moleculer-db-adapter-mongoose/-/moleculer-db-adapter-mongoose-0.9.4.tgz", + "integrity": "sha512-+ZRPfyLAVbBb6St/9RAqiZdWY3hw++fPutwqSPg8AYNL7n/eIQRnvx7oXn+ebnezbuCXg63L1siT9uzyKNzbZw==", "dependencies": { "bluebird": "^3.7.2", "lodash": "^4.17.21" @@ -13882,9 +13882,9 @@ ] }, "node_modules/r2-lcp-js": { - "version": "1.0.39", - "resolved": "https://registry.npmjs.org/r2-lcp-js/-/r2-lcp-js-1.0.39.tgz", - "integrity": "sha512-M4m1kBdMAYrHnDIGjBoxwDI+uKqbn/CEy4r2+D0Pca7OoMcHq1DQq1IF/bNnr/Ko6xyNyejaLTDgrr/pVbmEUA==", + "version": "1.0.40", + "resolved": "https://registry.npmjs.org/r2-lcp-js/-/r2-lcp-js-1.0.40.tgz", + "integrity": "sha512-KKBY8AKudLU8RQOsTY/VQX293UDWVeAgm7LxEkkz5th54l6JdJGrNdM7uFPjSy2k+ojmPu3aYZws1pfLrF/5eg==", "dependencies": { "bindings": "^1.5.0", "debug": "^4.3.4", diff --git a/package.json b/package.json index cb57c85..b558960 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "description": "Endpoints for common operations in ThreeTwo", "scripts": { "build": "tsc --build tsconfig.json", - "dev": "ts-node ./node_modules/moleculer/bin/moleculer-runner.js --hot --repl --config moleculer.config.ts services/**/*.service.ts", - "start": "moleculer-runner --config dist/moleculer.config.js", + "dev": "./scripts/start.sh dev", + "start": "npm run build && ./scripts/start.sh prod", "cli": "moleculer connect NATS", "ci": "jest --watch", "test": "jest --coverage", @@ -68,7 +68,7 @@ "mkdirp": "^0.5.5", "moleculer-bullmq": "^3.0.0", "moleculer-db": "^0.8.23", - "moleculer-db-adapter-mongoose": "^0.9.2", + "moleculer-db-adapter-mongoose": "^0.9.4", "moleculer-io": "^2.2.0", "moleculer-web": "^0.10.5", "mongoosastic-ts": "^6.0.3", diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100755 index 0000000..f89617e --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Check if the first argument is 'dev', use ts-node; otherwise, use node +MODE=$1 + +# Extract the host and port from MONGO_URI +HOST_PORT=$(echo $MONGO_URI | sed -e 's/mongodb:\/\///' -e 's/\/.*$//') + +# Assuming the script is called from the project root +PROJECT_ROOT=$(pwd) +CONFIG_PATH="$PROJECT_ROOT/moleculer.config.ts" + +# Set the correct path for moleculer-runner based on the mode +if [ "$MODE" == "dev" ]; then + # For development: use ts-node + MOLECULER_RUNNER="ts-node $PROJECT_ROOT/node_modules/moleculer/bin/moleculer-runner.js --hot --repl --config $CONFIG_PATH $PROJECT_ROOT/services/**/*.service.ts" +else + # For production: direct node execution of the compiled JavaScript + MOLECULER_RUNNER="moleculer-runner --config $PROJECT_ROOT/dist/moleculer.config.js $PROJECT_ROOT/dist/services/**/*.service.js" +fi + +# Run wait-for-it, then start the application +./scripts/wait-for-it.sh $HOST_PORT -- $MOLECULER_RUNNER diff --git a/scripts/wait-for-it.sh b/scripts/wait-for-it.sh new file mode 100755 index 0000000..c092856 --- /dev/null +++ b/scripts/wait-for-it.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} +if [[ $OSTYPE == 'darwin'* ]]; then + if ! command -v gtimeout &> /dev/null + then + echo "missing gtimeout (`brew install coreutils`)" + exit + fi + alias timeout=gtimeout +fi + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/services/library.service.ts b/services/library.service.ts index e417ae1..925a340 100644 --- a/services/library.service.ts +++ b/services/library.service.ts @@ -58,9 +58,11 @@ import klaw from "klaw"; import path from "path"; import { COMICS_DIRECTORY, USERDATA_DIRECTORY } from "../constants/directories"; -console.log(`MONGO -> ${process.env.MONGO_URI}`); -export default class ImportService extends Service { - public constructor(public broker: ServiceBroker) { +export default class LibraryService extends Service { + public constructor( + public broker: ServiceBroker, + schema: ServiceSchema<{}> = { name: "library" } + ) { super(broker); this.parseServiceSchema({ name: "library",