From fecd97c22290fc1e8246ed074feb29418f80115d Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Thu, 11 Apr 2024 10:53:32 -0400 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=97=BA=EF=B8=8F=20Added=205173=20as?= =?UTF-8?q?=20the=20UI=20port?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b2d540e..a5919c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,11 @@ version: "3.7" -x-userdata-volume: - &userdata-volume +x-userdata-volume: &userdata-volume type: bind source: ${USERDATA_DIRECTORY} target: /userdata -x-comics-volume: - &comics-volume +x-comics-volume: &comics-volume type: bind source: ${COMICS_DIRECTORY} target: /comics @@ -24,6 +22,7 @@ services: ports: - "8050:8050" - "3050:3050" + - "5173:5173" links: - core-services depends_on: From 36330540919aceaac458072bac121a981ebf83a3 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Fri, 10 May 2024 22:55:05 -0400 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=8F=97=EF=B8=8F=20Added=20kafka=20and?= =?UTF-8?q?=20zookeeper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a5919c1..8cf9ff3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,8 @@ services: - db - elasticsearch - redis + - kafka + - zookeeper networks: - proxy @@ -62,9 +64,11 @@ services: - db - redis - elasticsearch + - kafka + - zookeeper environment: name: core-services - SERVICES: api,library,importqueue,settings,search,socket,imagetransformation,opds + SERVICES: api,library,jobqueue,settings,search,socket,imagetransformation,torrentjobs,opds env_file: docker-compose.env volumes: - *comics-volume @@ -73,6 +77,31 @@ services: networks: - proxy + zookeeper: + image: zookeeper:latest + container_name: zookeeper + ports: + - "2181:2181" + networks: + - proxy + + kafka: + image: apache/kafka:latest + container_name: kafka + ports: + - "9092:9092" + environment: + KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9093,OUTSIDE://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT + KAFKA_LISTENERS: INSIDE://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + depends_on: + - zookeeper + networks: + - proxy + db: image: "bitnami/mongodb:latest" container_name: database From 68e035ceb25b3333874b12895996e0e742b8dbbd Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Fri, 17 May 2024 10:50:11 -0400 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=90=B3=20Updated=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 24 +++++++++++++++++++----- docker-compose.env | 11 ++++++----- docker-compose.yml | 4 ++-- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8ae4586..57a6728 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ This repo contains the `docker-compose` files necessary to run: 1. ThreeTwo UI 2. `threetwo-core-service` and its dependencies 3. `threetwo-metadata-service` and its dependencies +4. `threetwo-acquisition-service` and its dependencies ### Pre-requisites @@ -24,12 +25,8 @@ You need 2 folders: ``` - comics - userdata - |- covers - |- expanded - |- temporary ``` - #### ComicVine To get ComicVine to work for metadata scraping and other functions, you _must_ have a ComicVine API key. @@ -49,12 +46,21 @@ You can get one [here.](https://comicvine.gamespot.com/api/) Metadata scraping w 1. Find the hostname of your machine and set the `UNDERLYING_HOSTNAME` to that. For e.g. `UNDERLYING_HOSTNAME=foo` 2. Set the `COMICS_DIRECTORY` and `USERDATA_DIRECTORY` to the correct _absolute_ paths 3. Set the `COMICVINE_API` to your ComicVine API key + 4. To summarize, these are the only variables you need to configure: + ``` + UNDERLYING_HOSTNAME=myhost + + COMICS_DIRECTORY=/path/to/comics + USERDATA_DIRECTORY=/path/to/userdata + + COMICVINE_API_KEY=onethreetwo + ``` 4. Save the file 5. Run the stack using: `env $(cat docker-compose.env | xargs) docker-compose up` ### Ports -1. `threetwo`, the UI runs on port `8050` +1. `threetwo`, the UI runs on port `5173` 2. `threetwo-core-service` service on `3000` 3. `threetwo-metadata-service` service on `3080` @@ -65,4 +71,12 @@ General 1. Some common problems can be traced to out-of-date images, and as such can be mitigated by simply pruning orphaned images: `docker system prune -a` 2. Always check the logs of the offending service, `docker logs --follow ` +3. Once the stack is up, assuming everything went well, look for `threetwo-ui` service's logs to see something like: + + ``` + threetwo-ui | VITE v5.2.7 ready in 145 ms + threetwo-ui | + threetwo-ui | ➜ Local: http://localhost:5173/ + threetwo-ui | ➜ Network: http://172.30.0.9:5173/ +``` diff --git a/docker-compose.env b/docker-compose.env index b3f349a..8d0c19f 100644 --- a/docker-compose.env +++ b/docker-compose.env @@ -1,10 +1,11 @@ -UNDERLYING_HOSTNAME=ghost +UNDERLYING_HOSTNAME=localhost -COMICS_DIRECTORY=/mnt/unraidshare/directconnect/downloads/comics/ -USERDATA_DIRECTORY=/home/rishi/userdata +COMICS_DIRECTORY=/path/to/comics +USERDATA_DIRECTORY=/path/to/userdata -COMICVINE_API_KEY=foobar +COMICVINE_API_KEY=onethreetwo +CACHER=Memory LOGGER=true LOGLEVEL=info SERVICEDIR=dist/services @@ -15,4 +16,4 @@ MONGO_URI=mongodb://db:27017/threetwo ELASTICSEARCH_URI=http://elasticsearch:9200 REDIS_URI=redis://redis:6379 TRANSPORTER=redis://redis:6379 -CACHER=Memory + diff --git a/docker-compose.yml b/docker-compose.yml index 8cf9ff3..8f1deb9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -68,7 +68,7 @@ services: - zookeeper environment: name: core-services - SERVICES: api,library,jobqueue,settings,search,socket,imagetransformation,torrentjobs,opds + SERVICES: api,library,imagetransformation,opds,search,settings,jobqueue,socket,torrentjobs env_file: docker-compose.env volumes: - *comics-volume @@ -103,7 +103,7 @@ services: - proxy db: - image: "bitnami/mongodb:latest" + image: "mongo:latest" container_name: database networks: - proxy From c3b2bcf80e8d0d23e6d4a6661572efc6e94f44f6 Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Mon, 3 Jun 2024 17:22:44 -0400 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=94=A7=20Added=20kafka?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 151 ++++++++------------------------------------- 1 file changed, 26 insertions(+), 125 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8f1deb9..30571d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,144 +1,45 @@ version: "3.7" -x-userdata-volume: &userdata-volume - type: bind - source: ${USERDATA_DIRECTORY} - target: /userdata - -x-comics-volume: &comics-volume - type: bind - source: ${COMICS_DIRECTORY} - target: /comics - services: - threetwo: - build: - context: https://github.com/rishighan/threetwo.git - dockerfile: Dockerfile - image: frishi/threetwo - container_name: threetwo-ui - env_file: docker-compose.env - restart: unless-stopped - ports: - - "8050:8050" - - "3050:3050" - - "5173:5173" - links: - - core-services - depends_on: - - db - - elasticsearch - - redis - - kafka - - zookeeper - networks: - - proxy - - metadata-service: - build: - context: https://github.com/rishighan/threetwo-metadata-service.git - image: frishi/threetwo-metadata-service - container_name: metadata-service - ports: - - "3080:3080" - environment: - SERVICES: api,comicvine - env_file: docker-compose.env - depends_on: - - redis - volumes: - - *comics-volume - - *userdata-volume - networks: - - proxy - - core-services: - build: - context: https://github.com/rishighan/threetwo-core-service.git - image: frishi/threetwo-core-service - container_name: core-services - ports: - - "3000:3000" - - "3001:3001" - depends_on: - - db - - redis - - elasticsearch - - kafka - - zookeeper - environment: - name: core-services - SERVICES: api,library,imagetransformation,opds,search,settings,jobqueue,socket,torrentjobs - env_file: docker-compose.env - volumes: - - *comics-volume - - *userdata-volume - - networks: - - proxy zookeeper: - image: zookeeper:latest - container_name: zookeeper + image: bitnami/zookeeper:latest + restart: on-failure ports: - "2181:2181" + environment: + ZOO_MY_ID: 1 + ZOO_PORT: 2181 + ZOO_SERVERS: server.1=zookeeper:2888:3888 + ALLOW_ANONYMOUS_LOGIN: "yes" networks: - - proxy - + - proxy + kafka: - image: apache/kafka:latest - container_name: kafka + image: bitnami/kafka:latest + restart: on-failure ports: - "9092:9092" environment: - KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka:9093,OUTSIDE://localhost:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - KAFKA_LISTENERS: INSIDE://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - /var/run/docker.sock:/var/run/docker.sock + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + KAFKA_BROKER_ID: 1 + KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181 + ALLOW_PLAINTEXT_LISTENER: "yes" + KAFKA_LISTENERS: "INTERNAL://:29092,EXTERNAL://:9092" + KAFKA_ZOOKEEPER_SESSION_TIMEOUT: "6000" + KAFKA_RESTART_ATTEMPTS: "10" + KAFKA_RESTART_DELAY: "5" + ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0" depends_on: - zookeeper networks: - - proxy + - proxy - db: - image: "mongo:latest" - container_name: database - networks: - - proxy - ports: - - "27017:27017" - volumes: - - "mongodb_data:/bitnami/mongodb" - - redis: - image: "bitnami/redis:latest" - container_name: queue - environment: - ALLOW_EMPTY_PASSWORD: "yes" - networks: - - proxy - ports: - - "6379:6379" - - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2 - container_name: elasticsearch - environment: - - "discovery.type=single-node" - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - "xpack.security.enabled=true" - - "xpack.security.authc.api_key.enabled=true" - - "ELASTIC_PASSWORD=password" - ulimits: - memlock: - soft: -1 - hard: -1 - ports: - - 9200:9200 - networks: - - proxy networks: proxy: