3 Commits

Author SHA1 Message Date
2d9ea15550 🔧 Added canonical metadata related changes 2025-11-17 13:00:11 -05:00
755381021d Additions 2025-10-29 12:25:05 -04:00
a9bfa479c4 🔧 Added graphQL bits 2025-09-23 18:14:35 -04:00
15 changed files with 2080 additions and 314 deletions

View File

@@ -1,3 +0,0 @@
{
"esversion": 10
}

356
CANONICAL_METADATA_GUIDE.md Normal file
View File

@@ -0,0 +1,356 @@
# Canonical Comic Metadata Model - Implementation Guide
## 🎯 Overview
The canonical metadata model provides a comprehensive system for managing comic book metadata from multiple sources with proper **provenance tracking**, **confidence scoring**, and **conflict resolution**.
## 🏗️ Architecture
### **Core Components:**
1. **📋 Type Definitions** ([`models/canonical-comic.types.ts`](models/canonical-comic.types.ts:1))
2. **🎯 GraphQL Schema** ([`models/graphql/canonical-typedef.ts`](models/graphql/canonical-typedef.ts:1))
3. **🔧 Resolution Engine** ([`utils/metadata-resolver.utils.ts`](utils/metadata-resolver.utils.ts:1))
4. **💾 Database Model** ([`models/canonical-comic.model.ts`](models/canonical-comic.model.ts:1))
5. **⚙️ Service Layer** ([`services/canonical-metadata.service.ts`](services/canonical-metadata.service.ts:1))
---
## 📊 Metadata Sources & Ranking
### **Source Priority (Highest to Lowest):**
```typescript
enum MetadataSourceRank {
USER_MANUAL = 1, // User overrides - highest priority
COMICINFO_XML = 2, // Embedded metadata - high trust
COMICVINE = 3, // ComicVine API - authoritative
METRON = 4, // Metron API - authoritative
GCD = 5, // Grand Comics Database - community
LOCG = 6, // League of Comic Geeks - specialized
LOCAL_FILE = 7 // Filename inference - lowest trust
}
```
### **Confidence Scoring:**
- **User Manual**: 1.0 (100% trusted)
- **ComicInfo.XML**: 0.8-0.95 (based on completeness)
- **ComicVine**: 0.9 (highly reliable API)
- **Metron**: 0.85 (reliable API)
- **GCD**: 0.8 (community-maintained)
- **Local File**: 0.3 (inference-based)
---
## 🔄 Usage Examples
### **1. Import ComicVine Metadata**
```typescript
// REST API
POST /api/canonicalMetadata/importComicVine/60f7b1234567890abcdef123
{
"comicVineData": {
"id": 142857,
"name": "Amazing Spider-Man #1",
"issue_number": "1",
"cover_date": "2023-01-01",
"volume": {
"id": 12345,
"name": "Amazing Spider-Man",
"start_year": 2023,
"publisher": { "name": "Marvel Comics" }
},
"person_credits": [
{ "name": "Dan Slott", "role": "writer" }
]
}
}
```
```typescript
// Service usage
const result = await broker.call('canonicalMetadata.importComicVineMetadata', {
comicId: '60f7b1234567890abcdef123',
comicVineData: comicVineData,
forceUpdate: false
});
```
### **2. Import ComicInfo.XML**
```typescript
POST /api/canonicalMetadata/importComicInfo/60f7b1234567890abcdef123
{
"xmlData": {
"Title": "Amazing Spider-Man",
"Series": "Amazing Spider-Man",
"Number": "1",
"Year": 2023,
"Month": 1,
"Writer": "Dan Slott",
"Penciller": "John Romita Jr",
"Publisher": "Marvel Comics"
}
}
```
### **3. Set Manual Metadata (Highest Priority)**
```typescript
PUT /api/canonicalMetadata/manual/60f7b1234567890abcdef123/title
{
"value": "The Amazing Spider-Man #1",
"confidence": 1.0,
"notes": "User corrected title formatting"
}
```
### **4. Resolve Metadata Conflicts**
```typescript
// Get conflicts
GET /api/canonicalMetadata/conflicts/60f7b1234567890abcdef123
// Resolve by selecting preferred source
POST /api/canonicalMetadata/resolve/60f7b1234567890abcdef123/title
{
"selectedSource": "COMICVINE"
}
```
### **5. Query with Source Filtering**
```graphql
query {
searchComicsByMetadata(
title: "Spider-Man"
sources: [COMICVINE, COMICINFO_XML]
minConfidence: 0.8
) {
resolvedMetadata {
title
series { name volume publisher }
creators { name role }
}
canonicalMetadata {
title {
value
source
confidence
timestamp
sourceUrl
}
}
}
}
```
---
## 🔧 Data Structure
### **Canonical Metadata Storage:**
```typescript
{
"canonicalMetadata": {
"title": [
{
"value": "Amazing Spider-Man #1",
"source": "COMICVINE",
"confidence": 0.9,
"rank": 3,
"timestamp": "2023-01-15T10:00:00Z",
"sourceId": "142857",
"sourceUrl": "https://comicvine.gamespot.com/issue/4000-142857/"
},
{
"value": "Amazing Spider-Man",
"source": "COMICINFO_XML",
"confidence": 0.8,
"rank": 2,
"timestamp": "2023-01-15T09:00:00Z"
}
],
"creators": [
{
"value": [
{ "name": "Dan Slott", "role": "Writer" },
{ "name": "John Romita Jr", "role": "Penciller" }
],
"source": "COMICINFO_XML",
"confidence": 0.85,
"rank": 2,
"timestamp": "2023-01-15T09:00:00Z"
}
]
}
}
```
### **Resolved Metadata (Best Values):**
```typescript
{
"resolvedMetadata": {
"title": "Amazing Spider-Man #1", // From ComicVine (higher confidence)
"series": {
"name": "Amazing Spider-Man",
"volume": 1,
"publisher": "Marvel Comics"
},
"creators": [
{ "name": "Dan Slott", "role": "Writer" },
{ "name": "John Romita Jr", "role": "Penciller" }
],
"lastResolved": "2023-01-15T10:30:00Z",
"resolutionConflicts": [
{
"field": "title",
"conflictingValues": [
{ "value": "Amazing Spider-Man #1", "source": "COMICVINE", "confidence": 0.9 },
{ "value": "Amazing Spider-Man", "source": "COMICINFO_XML", "confidence": 0.8 }
]
}
]
}
}
```
---
## ⚙️ Resolution Strategies
### **Available Strategies:**
```typescript
const strategies = {
// Use source with highest confidence score
highest_confidence: { strategy: 'highest_confidence' },
// Use source with highest rank (USER_MANUAL > COMICINFO_XML > COMICVINE...)
highest_rank: { strategy: 'highest_rank' },
// Use most recently added metadata
most_recent: { strategy: 'most_recent' },
// Prefer user manual entries
user_preference: { strategy: 'user_preference' },
// Attempt to find consensus among sources
consensus: { strategy: 'consensus' }
};
```
### **Custom Strategy:**
```typescript
const customStrategy: MetadataResolutionStrategy = {
strategy: 'highest_rank',
minimumConfidence: 0.7,
allowedSources: [MetadataSource.COMICVINE, MetadataSource.COMICINFO_XML],
fieldSpecificStrategies: {
'creators': { strategy: 'consensus' }, // Merge creators from multiple sources
'title': { strategy: 'highest_confidence' } // Use most confident title
}
};
```
---
## 🚀 Integration Workflow
### **1. Local File Import Process:**
```typescript
// 1. Extract file metadata
const localMetadata = extractLocalMetadata(filePath);
comic.addMetadata('title', inferredTitle, MetadataSource.LOCAL_FILE, 0.3);
// 2. Parse ComicInfo.XML (if exists)
if (comicInfoXML) {
await broker.call('canonicalMetadata.importComicInfoXML', {
comicId: comic._id,
xmlData: comicInfoXML
});
}
// 3. Enhance with external APIs
const comicVineMatch = await searchComicVine(comic.resolvedMetadata.title);
if (comicVineMatch) {
await broker.call('canonicalMetadata.importComicVineMetadata', {
comicId: comic._id,
comicVineData: comicVineMatch
});
}
// 4. Resolve final metadata
await broker.call('canonicalMetadata.reResolveMetadata', {
comicId: comic._id
});
```
### **2. Conflict Resolution Workflow:**
```typescript
// 1. Detect conflicts
const conflicts = await broker.call('canonicalMetadata.getMetadataConflicts', {
comicId: comic._id
});
// 2. Present to user for resolution
if (conflicts.length > 0) {
// Show UI with conflicting values and sources
const userChoice = await presentConflictResolution(conflicts);
// 3. Apply user's resolution
await broker.call('canonicalMetadata.resolveMetadataConflict', {
comicId: comic._id,
field: userChoice.field,
selectedSource: userChoice.source
});
}
```
---
## 📈 Performance Considerations
### **Database Indexes:**
-**Text search**: `resolvedMetadata.title`, `resolvedMetadata.series.name`
-**Unique identification**: `series.name` + `volume` + `issueNumber`
-**Source filtering**: `canonicalMetadata.*.source` + `confidence`
-**Import status**: `importStatus.isImported` + `tagged`
### **Optimization Tips:**
- **Batch metadata imports** for large collections
- **Cache resolved metadata** for frequently accessed comics
- **Index on confidence scores** for quality filtering
- **Paginate conflict resolution** for large libraries
---
## 🛡️ Best Practices
### **Data Quality:**
1. **Always validate** external API responses before import
2. **Set appropriate confidence** scores based on source reliability
3. **Preserve original data** in source-specific fields
4. **Log metadata changes** for audit trails
### **Conflict Management:**
1. **Prefer user overrides** for disputed fields
2. **Use consensus** for aggregatable fields (creators, characters)
3. **Maintain provenance** links to original sources
4. **Provide clear UI** for conflict resolution
### **Performance:**
1. **Re-resolve metadata** only when sources change
2. **Cache frequently accessed** resolved metadata
3. **Batch operations** for bulk imports
4. **Use appropriate indexes** for common queries
---
This canonical metadata model provides enterprise-grade metadata management with full provenance tracking, confidence scoring, and flexible conflict resolution for comic book collections of any size.

View File

@@ -0,0 +1,423 @@
# Moleculer Microservices Dependency Analysis
**ThreeTwo Core Service - Comic Book Library Management System**
## System Overview
This **ThreeTwo Core Service** is a sophisticated **comic book library management system** built on Moleculer microservices architecture. The system demonstrates advanced patterns including:
- **Event-driven architecture** with real-time WebSocket communication
- **Asynchronous job processing** with BullMQ for heavy operations
- **Multi-source metadata aggregation** with canonical data resolution
- **Hybrid search** combining MongoDB aggregation and ElasticSearch
- **External system integrations** (P2P, BitTorrent, Comic APIs)
### Technical Stack
- **Framework**: Moleculer.js microservices
- **Node ID**: `threetwo-core-service`
- **Transport**: Redis (`redis://localhost:6379`)
- **Databases**: MongoDB + ElasticSearch
- **Queue System**: BullMQ (Redis-backed)
- **Real-time**: Socket.IO with Redis adapter
- **External APIs**: ComicVine, AirDC++, qBittorrent
## Service Architecture
### Core Services
| Service | File | Role | Dependencies |
|---------|------|------|-------------|
| **API** | [`api.service.ts`](services/api.service.ts) | API Gateway + File System Watcher | → library, jobqueue |
| **Library** | [`library.service.ts`](services/library.service.ts) | Core Comic Library Management | → jobqueue, search, comicvine |
| **JobQueue** | [`jobqueue.service.ts`](services/jobqueue.service.ts) | Asynchronous Job Processing (BullMQ) | → library, socket |
| **Socket** | [`socket.service.ts`](services/socket.service.ts) | Real-time Communication (Socket.IO) | → library, jobqueue |
| **Search** | [`search.service.ts`](services/search.service.ts) | ElasticSearch Integration | ElasticSearch client |
| **GraphQL** | [`graphql.service.ts`](services/graphql.service.ts) | GraphQL API Layer | → search |
### Supporting Services
| Service | File | Role | Dependencies |
|---------|------|------|-------------|
| **AirDC++** | [`airdcpp.service.ts`](services/airdcpp.service.ts) | P2P File Sharing Integration | External AirDC++ client |
| **Settings** | [`settings.service.ts`](services/settings.service.ts) | Configuration Management | MongoDB |
| **Image Transform** | [`imagetransformation.service.ts`](services/imagetransformation.service.ts) | Cover Processing | File system |
| **OPDS** | [`opds.service.ts`](services/opds.service.ts) | Comic Catalog Feeds | File system |
| **Torrent Jobs** | [`torrentjobs.service.ts`](services/torrentjobs.service.ts) | BitTorrent Integration | → library, qbittorrent |
## Service-to-Service Dependencies
### Core Service Interactions
#### 1. API Service → Other Services
```typescript
// File system watcher triggers import
ctx.broker.call("library.walkFolders", { basePathToWalk: filePath })
ctx.broker.call("importqueue.processImport", { fileObject })
```
#### 2. Library Service → Dependencies
```typescript
// Job queue integration
this.broker.call("jobqueue.enqueue", { action: "enqueue.async" })
// Search operations
ctx.broker.call("search.searchComic", { elasticSearchQueries })
ctx.broker.call("search.deleteElasticSearchIndices", {})
// External metadata
ctx.broker.call("comicvine.getVolumes", { volumeURI })
```
#### 3. JobQueue Service → Dependencies
```typescript
// Import processing
this.broker.call("library.importFromJob", { importType, payload })
// Real-time updates
this.broker.call("socket.broadcast", {
namespace: "/",
event: "LS_COVER_EXTRACTED",
args: [{ completedJobCount, importResult }]
})
```
#### 4. Socket Service → Dependencies
```typescript
// Job management
ctx.broker.call("jobqueue.getJobCountsByType", {})
ctx.broker.call("jobqueue.toggle", { action: queueAction })
// Download tracking
ctx.call("library.applyAirDCPPDownloadMetadata", {
bundleId, comicObjectId, name, size, type
})
```
#### 5. GraphQL Service → Search
```typescript
// Wanted comics query
const result = await ctx.broker.call("search.issue", {
query: eSQuery,
pagination: { size: limit, from: offset },
type: "wanted"
})
```
## API Endpoint Mapping
### REST API Routes (`/api/*`)
#### Library Management
- `POST /api/library/walkFolders` → [`library.walkFolders`](services/library.service.ts:82)
- `POST /api/library/newImport` → [`library.newImport`](services/library.service.ts:165) → [`jobqueue.enqueue`](services/library.service.ts:219)
- `POST /api/library/getComicBooks` → [`library.getComicBooks`](services/library.service.ts:535)
- `POST /api/library/getComicBookById` → [`library.getComicBookById`](services/library.service.ts:550)
- `POST /api/library/flushDB` → [`library.flushDB`](services/library.service.ts:818) → [`search.deleteElasticSearchIndices`](services/library.service.ts:839)
- `GET /api/library/libraryStatistics` → [`library.libraryStatistics`](services/library.service.ts:684)
#### Job Management
- `GET /api/jobqueue/getJobCountsByType` → [`jobqueue.getJobCountsByType`](services/jobqueue.service.ts:31)
- `GET /api/jobqueue/toggle` → [`jobqueue.toggle`](services/jobqueue.service.ts:38)
- `GET /api/jobqueue/getJobResultStatistics` → [`jobqueue.getJobResultStatistics`](services/jobqueue.service.ts:214)
#### Search Operations
- `POST /api/search/searchComic` → [`search.searchComic`](services/search.service.ts:28)
- `POST /api/search/searchIssue` → [`search.issue`](services/search.service.ts:60)
- `GET /api/search/deleteElasticSearchIndices` → [`search.deleteElasticSearchIndices`](services/search.service.ts:171)
#### AirDC++ Integration
- `POST /api/airdcpp/initialize` → [`airdcpp.initialize`](services/airdcpp.service.ts:24)
- `POST /api/airdcpp/getHubs` → [`airdcpp.getHubs`](services/airdcpp.service.ts:59)
- `POST /api/airdcpp/search` → [`airdcpp.search`](services/airdcpp.service.ts:96)
#### Image Processing
- `POST /api/imagetransformation/resizeImage` → [`imagetransformation.resize`](services/imagetransformation.service.ts:37)
- `POST /api/imagetransformation/analyze` → [`imagetransformation.analyze`](services/imagetransformation.service.ts:57)
### GraphQL Endpoints
- `POST /graphql` → [`graphql.wantedComics`](services/graphql.service.ts:49) → [`search.issue`](services/graphql.service.ts:77)
### Static File Serving
- `/userdata/*` → Static files from `./userdata`
- `/comics/*` → Static files from `./comics`
- `/logs/*` → Static files from `logs`
## Event-Driven Communication
### Job Queue Events
#### Job Completion Events
```typescript
// Successful import completion
"enqueue.async.completed" socket.broadcast("LS_COVER_EXTRACTED", {
completedJobCount,
importResult: job.returnvalue.data.importResult
})
// Failed import handling
"enqueue.async.failed" socket.broadcast("LS_COVER_EXTRACTION_FAILED", {
failedJobCount,
importResult: job
})
// Queue drained
"drained" socket.broadcast("LS_IMPORT_QUEUE_DRAINED", {
message: "drained"
})
```
#### Archive Processing Events
```typescript
// Archive uncompression completed
"uncompressFullArchive.async.completed" socket.broadcast("LS_UNCOMPRESSION_JOB_COMPLETE", {
uncompressedArchive: job.returnvalue
})
```
### File System Events
```typescript
// File watcher events (debounced 200ms)
fileWatcher.on("add", (path, stats) {
broker.call("library.walkFolders", { basePathToWalk: filePath })
broker.call("importqueue.processImport", { fileObject })
broker.broadcast(event, { path: filePath })
})
```
### WebSocket Events
#### Real-time Search
```typescript
// Search initiation
socket.emit("searchInitiated", { instance })
// Live search results
socket.emit("searchResultAdded", groupedResult)
socket.emit("searchResultUpdated", updatedResult)
socket.emit("searchComplete", { message })
```
#### Download Progress
```typescript
// Download status
broker.emit("downloadCompleted", bundleDBImportResult)
broker.emit("downloadError", error.message)
// Progress tracking
socket.emit("downloadTick", data)
```
## Data Flow Architecture
### 1. Comic Import Processing Flow
```mermaid
graph TD
A[File System Watcher] --> B[library.walkFolders]
B --> C[jobqueue.enqueue]
C --> D[jobqueue.enqueue.async]
D --> E[Archive Extraction]
E --> F[Metadata Processing]
F --> G[Canonical Metadata Creation]
G --> H[library.importFromJob]
H --> I[MongoDB Storage]
I --> J[ElasticSearch Indexing]
J --> K[socket.broadcast LS_COVER_EXTRACTED]
```
### 2. Search & Discovery Flow
```mermaid
graph TD
A[GraphQL/REST Query] --> B[search.issue]
B --> C[ElasticSearch Query]
C --> D[Results Enhancement]
D --> E[Metadata Scoring]
E --> F[Structured Response]
```
### 3. Download Management Flow
```mermaid
graph TD
A[socket[search]] --> B[airdcpp.search]
B --> C[Real-time Results]
C --> D[socket[download]]
D --> E[library.applyAirDCPPDownloadMetadata]
E --> F[Progress Tracking]
F --> G[Import Pipeline]
```
## Database Dependencies
### MongoDB Collections
| Collection | Model | Used By Services |
|------------|-------|-----------------|
| **comics** | [`Comic`](models/comic.model.ts) | library, search, jobqueue, imagetransformation |
| **settings** | [`Settings`](models/settings.model.ts) | settings |
| **sessions** | [`Session`](models/session.model.ts) | socket |
| **jobresults** | [`JobResult`](models/jobresult.model.ts) | jobqueue |
### ElasticSearch Integration
- **Index**: `comics` - Full-text search with metadata scoring
- **Client**: [`eSClient`](services/search.service.ts:13) from [`comic.model.ts`](models/comic.model.ts)
- **Query Types**: match_all, multi_match, bool queries with field boosting
### Redis Usage
| Purpose | Services | Configuration |
|---------|----------|---------------|
| **Transport** | All services | [`moleculer.config.ts:93`](moleculer.config.ts:93) |
| **Job Queue** | jobqueue | [`jobqueue.service.ts:27`](services/jobqueue.service.ts:27) |
| **Socket.IO Adapter** | socket | [`socket.service.ts:48`](services/socket.service.ts:48) |
| **Job Counters** | jobqueue | [`completedJobCount`](services/jobqueue.service.ts:392), [`failedJobCount`](services/jobqueue.service.ts:422) |
## External System Integrations
### AirDC++ (P2P File Sharing)
```typescript
// Integration wrapper
const ADCPPSocket = new AirDCPPSocket(config)
await ADCPPSocket.connect()
// Search operations
const searchInstance = await ADCPPSocket.post("search")
const searchInfo = await ADCPPSocket.post(`search/${searchInstance.id}/hub_search`, query)
// Download management
const downloadResult = await ADCPPSocket.post(`search/${searchInstanceId}/results/${resultId}/download`)
```
### ComicVine API
```typescript
// Metadata enrichment
const volumeDetails = await this.broker.call("comicvine.getVolumes", {
volumeURI: matchedResult.volume.api_detail_url
})
```
### qBittorrent Client
```typescript
// Torrent monitoring
const torrents = await this.broker.call("qbittorrent.getTorrentRealTimeStats", { infoHashes })
```
## Metadata Management System
### Multi-Source Metadata Aggregation
The system implements sophisticated metadata management with source prioritization:
#### Source Priority Order
1. **ComicInfo.xml** (embedded in archives)
2. **ComicVine API** (external database)
3. **Metron** (comic database)
4. **Grand Comics Database (GCD)**
5. **League of Comic Geeks (LOCG)**
6. **Filename Inference** (fallback)
#### Canonical Metadata Structure
```typescript
const canonical = {
title: findBestValue('title', inferredMetadata.title),
series: {
name: findSeriesValue(['series', 'seriesName', 'name'], inferredMetadata.series),
volume: findBestValue('volume', inferredMetadata.volume || 1),
startYear: findBestValue('startYear', inferredMetadata.issue?.year)
},
issueNumber: findBestValue('issueNumber', inferredMetadata.issue?.number),
publisher: findBestValue('publisher', null),
creators: [], // Combined from all sources
completeness: {
score: calculatedScore,
missingFields: [],
lastCalculated: currentTime
}
}
```
## Performance & Scalability Insights
### Asynchronous Processing
- **Heavy Operations**: Comic import, archive extraction, metadata processing
- **Queue System**: BullMQ with Redis backing for reliability
- **Job Types**: Import processing, archive extraction, torrent monitoring
- **Real-time Updates**: WebSocket progress notifications
### Search Optimization
- **Dual Storage**: MongoDB (transactional) + ElasticSearch (search)
- **Metadata Scoring**: Canonical metadata with source priority
- **Query Types**: Full-text, field-specific, boolean combinations
- **Caching**: Moleculer built-in memory caching
### External Integration Resilience
- **Timeout Handling**: Custom timeouts for long-running operations
- **Error Propagation**: Structured error responses with context
- **Connection Management**: Reusable connections for external APIs
- **Retry Logic**: Built-in retry policies for failed operations
## Critical Dependency Patterns
### 1. Service Chain Dependencies
- **Import Pipeline**: api → library → jobqueue → socket
- **Search Pipeline**: graphql → search → ElasticSearch
- **Download Pipeline**: socket → airdcpp → library
### 2. Circular Dependencies (Managed)
- **socket ←→ library**: Download coordination and progress updates
- **jobqueue ←→ socket**: Job progress notifications and queue control
### 3. Shared Resource Dependencies
- **MongoDB**: library, search, jobqueue, settings services
- **Redis**: All services (transport) + jobqueue (BullMQ) + socket (adapter)
- **ElasticSearch**: search, graphql services
## Architecture Strengths
### 1. Separation of Concerns
- **API Gateway**: Pure routing and file serving
- **Business Logic**: Centralized in library service
- **Data Access**: Abstracted through DbMixin
- **External Integration**: Isolated in dedicated services
### 2. Event-Driven Design
- **File System Events**: Automatic import triggering
- **Job Lifecycle Events**: Progress tracking and error handling
- **Real-time Communication**: WebSocket event broadcasting
### 3. Robust Metadata Management
- **Multi-Source Aggregation**: ComicVine, ComicInfo.xml, filename inference
- **Canonical Resolution**: Smart metadata merging with source attribution
- **User Curation Support**: Framework for manual metadata override
### 4. Scalability Features
- **Microservices Architecture**: Independent service scaling
- **Asynchronous Processing**: Heavy operations don't block API responses
- **Redis Transport**: Distributed service communication
- **Job Queue**: Reliable background processing with retry logic
## Potential Areas for Improvement
### 1. Service Coupling
- **High Interdependence**: library ←→ jobqueue ←→ socket tight coupling
- **Recommendation**: Event-driven decoupling for some operations
### 2. Error Handling
- **Inconsistent Patterns**: Mix of raw errors and MoleculerError usage
- **Recommendation**: Standardized error handling middleware
### 3. Configuration Management
- **Environment Variables**: Direct access vs centralized configuration
- **Recommendation**: Enhanced settings service for runtime configuration
### 4. Testing Strategy
- **Integration Testing**: Complex service interactions need comprehensive testing
- **Recommendation**: Contract testing between services
## Summary
This Moleculer-based architecture demonstrates sophisticated microservices patterns with:
- **11 specialized services** with clear boundaries
- **47 REST endpoints** + GraphQL layer
- **3 WebSocket namespaces** for real-time communication
- **Multi-database architecture** (MongoDB + ElasticSearch)
- **Advanced job processing** with BullMQ
- **External system integration** (P2P, BitTorrent, Comic APIs)
The system successfully manages complex domain requirements while maintaining good separation of concerns and providing excellent user experience through real-time updates and comprehensive metadata management.

195
README.md
View File

@@ -1,38 +1,175 @@
# threetwo-core-service
# ThreeTwo Core Service
This [moleculer-based](https://github.com/moleculerjs/moleculer-web) microservice houses endpoints for the following functions:
**A comprehensive comic book library management system** built as a high-performance Moleculer microservices architecture. ThreeTwo automatically processes comic archives (CBR, CBZ, CB7), extracts metadata, generates thumbnails, and provides powerful search and real-time synchronization capabilities.
1. Local import of a comic library into mongo (currently supports `cbr` and `cbz` files)
2. Metadata extraction from file, `comicinfo.xml`
3. Mongo comic object orchestration
4. CRUD operations on `Comic` model
5. Helper utils to help with image metadata extraction, file operations and more.
## 🎯 What This Service Does
## Local Development
ThreeTwo transforms chaotic comic book collections into intelligently organized, searchable digital libraries by:
1. You need the following dependencies installed: `mongo`, `elasticsearch` and `redis`
2. You also need binaries for `unrar` and `p7zip`
3. Clone this repo
4. Run `npm i`
5. Assuming you installed the dependencies correctly, run:
- **📚 Automated Library Management** - Monitors directories and automatically imports new comics
- **🧠 Intelligent Metadata Extraction** - Parses ComicInfo.XML and enriches data from external APIs (ComicVine)
- **🔍 Advanced Search** - ElasticSearch-powered multi-field search with confidence scoring
- **📱 Real-time Updates** - Live progress tracking and notifications via Socket.IO
- **🎨 Media Processing** - Automatic thumbnail generation and image optimization
```
COMICS_DIRECTORY=<PATH_TO_COMICS_DIRECTORY> \
USERDATA_DIRECTORY=<PATH_TO_USERDATA_DIRECTORY> \
REDIS_URI=redis://<REDIS_HOST:REDIS_PORT> \
ELASTICSEARCH_URI=<ELASTICSEARCH_HOST:ELASTICSEARCH_PORT> \
MONGO_URI=mongodb://<MONGO_HOST:MONGO_PORT>/threetwo \
UNRAR_BIN_PATH=<UNRAR_BIN_PATH> \
SEVENZ_BINARY_PATH=<SEVENZ_BINARY_PATH> \
npm run dev
```
## 🏗️ Architecture
to start the service
Built on **Moleculer microservices** with the following core services:
6. You should see the service spin up and a list of all the endpoints in the terminal
7. The service can be accessed through `http://localhost:3000/api/<serviceName>/*`
```
API Gateway (REST) ←→ GraphQL API ←→ Socket.IO Hub
Library Service ←→ Search Service ←→ Job Queue Service
MongoDB ←→ Elasticsearch ←→ Redis (Cache/Queue)
```
## Docker Instructions
### **Key Features:**
- **Multi-format Support** - CBR, CBZ, CB7 archive processing
- **Confidence Tracking** - Metadata quality assessment and provenance
- **Job Queue System** - Background processing with BullMQ and Redis
- **Debounced File Watching** - Efficient file system monitoring
- **Batch Operations** - Scalable bulk import handling
- **Real-time Sync** - Live updates across all connected clients
1. Build the image using `docker build . -t frishi/threetwo-import-service`. Give it a hot minute.
2. Run it using `docker run -it frishi/threetwo-import-service`
## 🚀 API Interfaces
- **REST API** - `http://localhost:3000/api/` - Traditional HTTP endpoints
- **GraphQL API** - `http://localhost:4000/graphql` - Modern query interface
- **Socket.IO** - Real-time events and progress tracking
- **Static Assets** - Direct access to comic covers and images
## 🛠️ Technology Stack
- **Backend**: Moleculer, Node.js, TypeScript
- **Database**: MongoDB (persistence), Elasticsearch (search), Redis (cache/queue)
- **Processing**: BullMQ (job queues), Sharp (image processing)
- **Communication**: Socket.IO (real-time), GraphQL + REST APIs
## 📋 Prerequisites
You need the following dependencies installed:
- **MongoDB** - Document database for comic metadata
- **Elasticsearch** - Full-text search and analytics
- **Redis** - Caching and job queue backend
- **System Binaries**: `unrar` and `p7zip` for archive extraction
## 🚀 Local Development
1. **Clone and Install**
```bash
git clone <repository-url>
cd threetwo-core-service
npm install
```
2. **Environment Setup**
```bash
COMICS_DIRECTORY=<PATH_TO_COMICS_DIRECTORY> \
USERDATA_DIRECTORY=<PATH_TO_USERDATA_DIRECTORY> \
REDIS_URI=redis://<REDIS_HOST:REDIS_PORT> \
ELASTICSEARCH_URI=<ELASTICSEARCH_HOST:ELASTICSEARCH_PORT> \
MONGO_URI=mongodb://<MONGO_HOST:MONGO_PORT>/threetwo \
UNRAR_BIN_PATH=<UNRAR_BIN_PATH> \
SEVENZ_BINARY_PATH=<SEVENZ_BINARY_PATH> \
npm run dev
```
3. **Service Access**
- **Main API**: `http://localhost:3000/api/<serviceName>/*`
- **GraphQL Playground**: `http://localhost:4000/graphql`
- **Admin Interface**: `http://localhost:3000/` (Moleculer dashboard)
## 🐳 Docker Deployment
```bash
# Build the image
docker build . -t threetwo-core-service
# Run with docker-compose (recommended)
docker-compose up -d
# Or run standalone
docker run -it threetwo-core-service
```
## 📊 Performance Features
- **Smart Debouncing** - 200ms file system event debouncing prevents overload
- **Batch Processing** - Efficient handling of bulk import operations
- **Multi-level Caching** - Memory + Redis caching for optimal performance
- **Job Queues** - Background processing prevents UI blocking
- **Connection Pooling** - Efficient database connection management
## 🔧 Core Services
| Service | Purpose | Key Features |
|---------|---------|--------------|
| **API Gateway** | REST endpoints + file watching | CORS, rate limiting, static serving |
| **GraphQL** | Modern query interface | Flexible queries, pagination |
| **Library** | Core CRUD operations | Comic management, metadata handling |
| **Search** | ElasticSearch integration | Multi-field search, aggregations |
| **Job Queue** | Background processing | Import jobs, progress tracking |
| **Socket** | Real-time communication | Live updates, session management |
## 📈 Use Cases
- **Personal Collections** - Organize digital comic libraries (hundreds to thousands)
- **Digital Libraries** - Professional-grade comic archive management
- **Developer Integration** - API access for custom comic applications
- **Bulk Processing** - Large-scale comic digitization projects
## 🛡️ Security & Reliability
- **Input Validation** - Comprehensive parameter validation
- **File Type Verification** - Magic number verification for security
- **Error Handling** - Graceful degradation and recovery
- **Health Monitoring** - Service health checks and diagnostics
## 🧩 Recent Enhancements
### Canonical Metadata System
A comprehensive **canonical metadata model** with full provenance tracking has been implemented to unify metadata from multiple sources:
- **Multi-Source Integration**: ComicVine, Metron, GCD, ComicInfo.XML, local files, and user manual entries
- **Source Ranking System**: Prioritized confidence scoring with USER_MANUAL (1) → COMICINFO_XML (2) → COMICVINE (3) → METRON (4) → GCD (5) → LOCG (6) → LOCAL_FILE (7)
- **Conflict Resolution**: Automatic metadata merging with confidence scoring and source attribution
- **Performance Optimized**: Proper indexing, batch processing, and caching strategies
### Complete Service Architecture Analysis
Comprehensive analysis of all **12 Moleculer services** with detailed endpoint documentation:
| Service | Endpoints | Primary Function |
|---------|-----------|------------------|
| [`api`](services/api.service.ts:1) | Gateway | REST API + file watching with 200ms debouncing |
| [`library`](services/library.service.ts:1) | 21 endpoints | Core CRUD operations and metadata management |
| [`search`](services/search.service.ts:1) | 8 endpoints | Elasticsearch integration and multi-search |
| [`jobqueue`](services/jobqueue.service.ts:1) | Queue mgmt | BullMQ job processing with Redis backend |
| [`graphql`](services/graphql.service.ts:1) | GraphQL API | Modern query interface with resolvers |
| [`socket`](services/socket.service.ts:1) | Real-time | Socket.IO communication with session management |
| [`canonicalMetadata`](services/canonical-metadata.service.ts:1) | 6 endpoints | **NEW**: Metadata provenance and conflict resolution |
| `airdcpp` | Integration | AirDC++ connectivity for P2P operations |
| `imagetransformation` | Processing | Image optimization and thumbnail generation |
| `opds` | Protocol | Open Publication Distribution System support |
| `settings` | Configuration | System-wide configuration management |
| `torrentjobs` | Downloads | Torrent-based comic acquisition |
### Performance Optimizations Identified
- **Debouncing**: 200ms file system event debouncing prevents overload
- **Job Queues**: Background processing with BullMQ prevents UI blocking
- **Caching Strategy**: Multi-level caching (Memory + Redis) for optimal performance
- **Batch Operations**: Efficient bulk import handling with pagination
- **Index Optimization**: MongoDB compound indexes for metadata queries
### Files Created
- [`models/canonical-comic.types.ts`](models/canonical-comic.types.ts:1) - TypeScript definitions for canonical metadata
- [`utils/metadata-resolver.utils.ts`](utils/metadata-resolver.utils.ts:1) - Conflict resolution and confidence scoring
- [`models/canonical-comic.model.ts`](models/canonical-comic.model.ts:1) - Mongoose schema with performance indexes
- [`services/canonical-metadata.service.ts`](services/canonical-metadata.service.ts:1) - REST endpoints for metadata import
- [`models/graphql/canonical-typedef.ts`](models/graphql/canonical-typedef.ts:1) - GraphQL schema with backward compatibility
- [`CANONICAL_METADATA_GUIDE.md`](CANONICAL_METADATA_GUIDE.md:1) - Complete implementation guide
---
**ThreeTwo Core Service** provides enterprise-grade comic book library management with modern microservices architecture, real-time capabilities, and intelligent automation.

47
graphql-server.ts Normal file
View File

@@ -0,0 +1,47 @@
import express from "express";
import { ApolloServer } from "@apollo/server";
import { expressMiddleware } from "@as-integrations/express4";
import { typeDefs } from "./models/graphql/typedef";
import { resolvers } from "./models/graphql/resolvers";
import { ServiceBroker } from "moleculer";
import cors from "cors";
// Boot Moleculer broker in parallel
const broker = new ServiceBroker({ transporter: null }); // or your actual transporter config
async function startGraphQLServer() {
const app = express();
const apollo = new ApolloServer({
typeDefs,
resolvers,
});
await apollo.start();
app.use(
"/graphql",
cors(),
express.json(),
expressMiddleware(apollo, {
context: async ({ req }) => ({
authToken: req.headers.authorization || null,
broker,
}),
})
);
const PORT = 4000;
app.listen(PORT, () =>
console.log(`🚀 GraphQL server running at http://localhost:${PORT}/graphql`)
);
}
async function bootstrap() {
await broker.start(); // make sure Moleculer is up
await startGraphQLServer();
}
bootstrap().catch((err) => {
console.error("❌ Failed to start GraphQL server:", err);
process.exit(1);
});

View File

@@ -101,13 +101,180 @@ const ComicSchema = mongoose.Schema(
},
sourcedMetadata: {
comicInfo: { type: mongoose.Schema.Types.Mixed, default: {} },
comicvine: { type: mongoose.Schema.Types.Mixed, default: {} }, // Set as a freeform object
comicvine: { type: mongoose.Schema.Types.Mixed, default: {} },
metron: { type: mongoose.Schema.Types.Mixed, default: {} },
gcd: { type: mongoose.Schema.Types.Mixed, default: {} },
locg: {
type: LOCGSchema,
es_indexed: true,
default: {},
},
},
// Canonical metadata - user-curated "canonical" values with source attribution
canonicalMetadata: {
// Core identifying information
title: {
value: { type: String, es_indexed: true },
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
// Series information
series: {
name: {
value: { type: String, es_indexed: true },
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
volume: {
value: Number,
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
startYear: {
value: Number,
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
}
},
// Issue information
issueNumber: {
value: { type: String, es_indexed: true },
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
// Publishing information
publisher: {
value: { type: String, es_indexed: true },
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
publicationDate: {
value: Date,
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
coverDate: {
value: Date,
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
// Content information
pageCount: {
value: Number,
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
summary: {
value: String,
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
// Creator information - array with source attribution
creators: [{
_id: false,
name: String,
role: String,
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
}],
// Character and genre arrays with source tracking
characters: {
values: [String],
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
genres: {
values: [String],
source: {
type: String,
enum: ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg', 'inferred', 'user'],
default: 'inferred'
},
userSelected: { type: Boolean, default: false },
lastModified: { type: Date, default: Date.now }
},
// Canonical metadata tracking
lastCanonicalUpdate: { type: Date, default: Date.now },
hasUserModifications: { type: Boolean, default: false },
// Quality and completeness tracking
completeness: {
score: { type: Number, min: 0, max: 100, default: 0 },
missingFields: [String],
lastCalculated: { type: Date, default: Date.now }
}
},
rawFileDetails: {
type: RawFileDetailsSchema,
es_indexed: true,

View File

@@ -1,24 +1,59 @@
import { gql } from "graphql-tag";
export const typeDefs = gql`
type Query {
comic(id: ID!): Comic
comics(limit: Int = 10): [Comic]
}
type Query {
comic(id: ID!): Comic
comics(limit: Int = 10): [Comic]
wantedComics(limit: Int = 25, offset: Int = 0): ComicPage!
}
type Comic {
id: ID!
title: String
volume: Int
issueNumber: String
publicationDate: String
coverUrl: String
creators: [Creator]
source: String
}
type Comic {
id: ID!
title: String!
volume: Int
issueNumber: String!
publicationDate: String
variant: String
format: String
creators: [Creator!]!
arcs: [String!]
coverUrl: String
filePath: String
pageCount: Int
tags: [String!]
source: String
type Creator {
name: String
role: String
}
confidence: ConfidenceMap
provenance: ProvenanceMap
}
type Creator {
name: String!
role: String!
}
type ConfidenceMap {
title: Float
volume: Float
issueNumber: Float
publicationDate: Float
creators: Float
variant: Float
format: Float
}
type ProvenanceMap {
title: String
volume: String
issueNumber: String
publicationDate: String
creators: String
variant: String
format: String
}
type ComicPage {
total: Int!
results: [Comic!]!
}
`;

197
package-lock.json generated
View File

@@ -9,15 +9,17 @@
"version": "0.0.1",
"dependencies": {
"@apollo/server": "^4.12.2",
"@as-integrations/express4": "^1.1.1",
"@bluelovers/fast-glob": "https://github.com/rishighan/fast-glob-v2-api.git",
"@elastic/elasticsearch": "^8.13.1",
"@jorgeferrero/stream-to-buffer": "^2.0.6",
"@ltv/moleculer-apollo-server-mixin": "^0.1.30",
"@npcz/magic": "^1.3.14",
"@root/walk": "^1.1.0",
"@socket.io/redis-adapter": "^8.1.0",
"@types/jest": "^27.4.1",
"@types/mkdirp": "^1.0.0",
"@types/node": "^13.9.8",
"@types/node": "^24.0.13",
"@types/string-similarity": "^4.0.0",
"airdcpp-apisocket": "^3.0.0-beta.8",
"axios": "^1.6.8",
@@ -25,6 +27,7 @@
"bree": "^7.1.5",
"calibre-opds": "^1.0.7",
"chokidar": "^4.0.3",
"cors": "^2.8.5",
"delay": "^5.0.0",
"dotenv": "^10.0.0",
"filename-parser": "^1.0.4",
@@ -43,7 +46,7 @@
"moleculer-db": "^0.8.23",
"moleculer-db-adapter-mongoose": "^0.9.2",
"moleculer-io": "^2.2.0",
"moleculer-web": "^0.10.5",
"moleculer-web": "^0.10.8",
"mongoosastic-ts": "^6.0.3",
"mongoose": "^6.10.4",
"mongoose-paginate-v2": "^1.3.18",
@@ -62,6 +65,7 @@
"@types/lodash": "^4.14.168",
"@typescript-eslint/eslint-plugin": "^5.56.0",
"@typescript-eslint/parser": "^5.56.0",
"concurrently": "^9.2.0",
"eslint": "^8.36.0",
"eslint-plugin-import": "^2.20.2",
"eslint-plugin-prefer-arrow": "^1.2.2",
@@ -77,7 +81,7 @@
"uuid": "^9.0.0"
},
"engines": {
"node": ">= 18.x.x"
"node": ">= 22.x.x"
}
},
"node_modules/@aashutoshrathi/word-wrap": {
@@ -110,6 +114,24 @@
"graphql": "14.x || 15.x || 16.x"
}
},
"node_modules/@apollo/federation-internals": {
"version": "2.11.2",
"resolved": "https://registry.npmjs.org/@apollo/federation-internals/-/federation-internals-2.11.2.tgz",
"integrity": "sha512-GSFGL2fLox3EBszWKJvRkVLFA0hkJF9PHGMQH+WdB/12KVB3QHKwDyW1T9VZtxe2SJhNU3puleSxCsO16Bf3iA==",
"license": "Elastic-2.0",
"dependencies": {
"@types/uuid": "^9.0.0",
"chalk": "^4.1.0",
"js-levenshtein": "^1.1.6",
"uuid": "^9.0.0"
},
"engines": {
"node": ">=14.15.0"
},
"peerDependencies": {
"graphql": "^16.5.0"
}
},
"node_modules/@apollo/protobufjs": {
"version": "1.2.7",
"resolved": "https://registry.npmjs.org/@apollo/protobufjs/-/protobufjs-1.2.7.tgz",
@@ -197,6 +219,22 @@
"node": ">=12"
}
},
"node_modules/@apollo/subgraph": {
"version": "2.11.2",
"resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-2.11.2.tgz",
"integrity": "sha512-S14osF5Zc8pd6lzeNtX1QHboMcQK5PXcN9EumZyRYBF0TRbnEFLF8Me9zMcfR3QP7GCiggjd6PA2IAaPC9uCSQ==",
"license": "MIT",
"dependencies": {
"@apollo/cache-control-types": "^1.0.2",
"@apollo/federation-internals": "2.11.2"
},
"engines": {
"node": ">=14.15.0"
},
"peerDependencies": {
"graphql": "^16.5.0"
}
},
"node_modules/@apollo/usage-reporting-protobuf": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/@apollo/usage-reporting-protobuf/-/usage-reporting-protobuf-4.1.1.tgz",
@@ -360,6 +398,19 @@
"node": ">=14"
}
},
"node_modules/@as-integrations/express4": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@as-integrations/express4/-/express4-1.1.1.tgz",
"integrity": "sha512-a2pur5nko91UaqWYwNRmcMEtmxgZH9eQzpner2ht/2CNSDuC+PHU3K+/uiISVgLC+2b+1TPzvutPejkN/+bsTw==",
"license": "MIT",
"engines": {
"node": ">=20"
},
"peerDependencies": {
"@apollo/server": "^4.0.0 || 5.0.0-rc.0",
"express": "^4.0.0"
}
},
"node_modules/@aws-crypto/sha256-browser": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz",
@@ -2413,6 +2464,19 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@ltv/moleculer-apollo-server-mixin": {
"version": "0.1.30",
"resolved": "https://registry.npmjs.org/@ltv/moleculer-apollo-server-mixin/-/moleculer-apollo-server-mixin-0.1.30.tgz",
"integrity": "sha512-/t1/aGwGIgwpwL2IMJl7WF/NtOKbJcpIObc31ur2ZaAYSjV+3anhTZgb9R2ePwdjdkZq5882vqCDJ8gnrrbiDg==",
"license": "MIT",
"dependencies": {
"@apollo/server": "^4.3.2",
"@apollo/subgraph": "^2.3.0",
"graphql": "^16.6.0",
"lodash.defaultsdeep": "^4.6.1",
"lodash.omit": "^4.5.0"
}
},
"node_modules/@mongodb-js/saslprep": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.2.2.tgz",
@@ -3493,9 +3557,13 @@
}
},
"node_modules/@types/node": {
"version": "13.13.52",
"resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.52.tgz",
"integrity": "sha512-s3nugnZumCC//n4moGGe6tkNMyYEdaDBitVjwPxXmR5lnMG5dHePinH2EdxkG3Rh1ghFHHixAG4NJhpJW1rthQ=="
"version": "24.0.13",
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.0.13.tgz",
"integrity": "sha512-Qm9OYVOFHFYg3wJoTSrz80hoec5Lia/dPp84do3X7dZvLikQvM1YpmvTBEdIr/e+U8HTkFjLHLnl78K/qjf+jQ==",
"license": "MIT",
"dependencies": {
"undici-types": "~7.8.0"
}
},
"node_modules/@types/node-fetch": {
"version": "2.6.12",
@@ -3568,6 +3636,12 @@
"resolved": "https://registry.npmjs.org/@types/string-similarity/-/string-similarity-4.0.0.tgz",
"integrity": "sha512-dMS4S07fbtY1AILG/RhuwmptmzK1Ql8scmAebOTJ/8iBtK/KI17NwGwKzu1uipjj8Kk+3mfPxum56kKZE93mzQ=="
},
"node_modules/@types/uuid": {
"version": "9.0.8",
"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz",
"integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==",
"license": "MIT"
},
"node_modules/@types/webidl-conversions": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz",
@@ -5148,6 +5222,48 @@
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="
},
"node_modules/concurrently": {
"version": "9.2.0",
"resolved": "https://registry.npmjs.org/concurrently/-/concurrently-9.2.0.tgz",
"integrity": "sha512-IsB/fiXTupmagMW4MNp2lx2cdSN2FfZq78vF90LBB+zZHArbIQZjQtzXCiXnvTxCZSvXanTqFLWBjw2UkLx1SQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"chalk": "^4.1.2",
"lodash": "^4.17.21",
"rxjs": "^7.8.1",
"shell-quote": "^1.8.1",
"supports-color": "^8.1.1",
"tree-kill": "^1.2.2",
"yargs": "^17.7.2"
},
"bin": {
"conc": "dist/bin/concurrently.js",
"concurrently": "dist/bin/concurrently.js"
},
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/open-cli-tools/concurrently?sponsor=1"
}
},
"node_modules/concurrently/node_modules/supports-color": {
"version": "8.1.1",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
"integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"has-flag": "^4.0.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/supports-color?sponsor=1"
}
},
"node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -5207,6 +5323,7 @@
"version": "2.8.5",
"resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
"integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
"license": "MIT",
"dependencies": {
"object-assign": "^4",
"vary": "^1"
@@ -9503,6 +9620,15 @@
"resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.4.4.tgz",
"integrity": "sha512-WZzeDOEtTOBK4Mdsar0IqEU5sMr3vSV2RqkAIzUEV2BHnUfKGyswWFPFwK5EeDo93K3FohSHbLAjj0s1Wzd+dg=="
},
"node_modules/js-levenshtein": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.6.tgz",
"integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/js-priority-queue": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/js-priority-queue/-/js-priority-queue-0.1.5.tgz",
@@ -9812,6 +9938,12 @@
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="
},
"node_modules/lodash.defaultsdeep": {
"version": "4.6.1",
"resolved": "https://registry.npmjs.org/lodash.defaultsdeep/-/lodash.defaultsdeep-4.6.1.tgz",
"integrity": "sha512-3j8wdDzYuWO3lM3Reg03MuQR957t287Rpcxp1njpEa8oDrikb+FwGdW3n+FELh/A6qib6yPit0j/pv9G/yeAqA==",
"license": "MIT"
},
"node_modules/lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
@@ -9828,6 +9960,13 @@
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
"integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="
},
"node_modules/lodash.omit": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.omit/-/lodash.omit-4.5.0.tgz",
"integrity": "sha512-XeqSp49hNGmlkj2EJlfrQFIzQ6lXdNro9sddtQzcJY8QaoC2GO0DT7xaIokHeyM+mIT0mPMlPvkYzg2xCuHdZg==",
"deprecated": "This package is deprecated. Use destructuring assignment syntax instead.",
"license": "MIT"
},
"node_modules/lodash.sortby": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
@@ -10598,9 +10737,10 @@
}
},
"node_modules/moleculer-web": {
"version": "0.10.7",
"resolved": "https://registry.npmjs.org/moleculer-web/-/moleculer-web-0.10.7.tgz",
"integrity": "sha512-/UJtV+O7iQ3aSg/xi/sw3ZswhvzkigzGPjKOR5R97sm2FSihKuLTftUpXlk4dYls7/8c8WSz6H/M/40BenEx9Q==",
"version": "0.10.8",
"resolved": "https://registry.npmjs.org/moleculer-web/-/moleculer-web-0.10.8.tgz",
"integrity": "sha512-kQtyN8AccdBqSZUh+PRLYmLPy7RBd48j/raA5682wNDo1fPEKdCHa8d7tUjtmI53gELkQZKCv3GckyMqdxZYXQ==",
"license": "MIT",
"dependencies": {
"@fastify/busboy": "^1.0.0",
"body-parser": "^1.19.0",
@@ -15001,6 +15141,16 @@
"queue-microtask": "^1.2.2"
}
},
"node_modules/rxjs": {
"version": "7.8.2",
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
"integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"tslib": "^2.1.0"
}
},
"node_modules/safe-array-concat": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.0.0.tgz",
@@ -15302,6 +15452,19 @@
"node": ">=8"
}
},
"node_modules/shell-quote": {
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz",
"integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/side-channel": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz",
@@ -16090,6 +16253,16 @@
"node": ">=14"
}
},
"node_modules/tree-kill": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
"integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==",
"dev": true,
"license": "MIT",
"bin": {
"tree-kill": "cli.js"
}
},
"node_modules/truncate-utf8-bytes": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz",
@@ -16479,6 +16652,12 @@
"node": ">=14.0"
}
},
"node_modules/undici-types": {
"version": "7.8.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz",
"integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==",
"license": "MIT"
},
"node_modules/undici/node_modules/@fastify/busboy": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz",

View File

@@ -23,6 +23,7 @@
"@types/lodash": "^4.14.168",
"@typescript-eslint/eslint-plugin": "^5.56.0",
"@typescript-eslint/parser": "^5.56.0",
"concurrently": "^9.2.0",
"eslint": "^8.36.0",
"eslint-plugin-import": "^2.20.2",
"eslint-plugin-prefer-arrow": "^1.2.2",
@@ -39,15 +40,17 @@
},
"dependencies": {
"@apollo/server": "^4.12.2",
"@as-integrations/express4": "^1.1.1",
"@bluelovers/fast-glob": "https://github.com/rishighan/fast-glob-v2-api.git",
"@elastic/elasticsearch": "^8.13.1",
"@jorgeferrero/stream-to-buffer": "^2.0.6",
"@ltv/moleculer-apollo-server-mixin": "^0.1.30",
"@npcz/magic": "^1.3.14",
"@root/walk": "^1.1.0",
"@socket.io/redis-adapter": "^8.1.0",
"@types/jest": "^27.4.1",
"@types/mkdirp": "^1.0.0",
"@types/node": "^13.9.8",
"@types/node": "^24.0.13",
"@types/string-similarity": "^4.0.0",
"airdcpp-apisocket": "^3.0.0-beta.8",
"axios": "^1.6.8",
@@ -55,6 +58,7 @@
"bree": "^7.1.5",
"calibre-opds": "^1.0.7",
"chokidar": "^4.0.3",
"cors": "^2.8.5",
"delay": "^5.0.0",
"dotenv": "^10.0.0",
"filename-parser": "^1.0.4",
@@ -73,7 +77,7 @@
"moleculer-db": "^0.8.23",
"moleculer-db-adapter-mongoose": "^0.9.2",
"moleculer-io": "^2.2.0",
"moleculer-web": "^0.10.5",
"moleculer-web": "^0.10.8",
"mongoosastic-ts": "^6.0.3",
"mongoose": "^6.10.4",
"mongoose-paginate-v2": "^1.3.18",
@@ -89,7 +93,7 @@
"xml2js": "^0.6.2"
},
"engines": {
"node": ">= 18.x.x"
"node": ">= 22.x.x"
},
"jest": {
"coverageDirectory": "<rootDir>/coverage",

View File

@@ -12,180 +12,207 @@ import { IFolderData } from "threetwo-ui-typings";
* @extends Service
*/
export default class ApiService extends Service {
/**
* The chokidar file system watcher instance.
* @private
*/
private fileWatcher?: any;
/**
* The chokidar file system watcher instance.
* @private
*/
private fileWatcher?: any;
/**
* Creates an instance of ApiService.
* @param {ServiceBroker} broker - The Moleculer service broker instance.
*/
public constructor(broker: ServiceBroker) {
super(broker);
this.parseServiceSchema({
name: "api",
mixins: [ApiGateway],
settings: {
port: process.env.PORT || 3000,
routes: [
{
path: "/api",
whitelist: ["**"],
cors: {
origin: "*",
methods: ["GET", "OPTIONS", "POST", "PUT", "DELETE"],
allowedHeaders: ["*"],
exposedHeaders: [],
credentials: false,
maxAge: 3600,
},
use: [],
mergeParams: true,
authentication: false,
authorization: false,
autoAliases: true,
aliases: {},
callingOptions: {},
bodyParsers: {
json: { strict: false, limit: "1MB" },
urlencoded: { extended: true, limit: "1MB" },
},
mappingPolicy: "all",
logging: true,
},
{
path: "/userdata",
use: [ApiGateway.serveStatic(path.resolve("./userdata"))],
},
{
path: "/comics",
use: [ApiGateway.serveStatic(path.resolve("./comics"))],
},
{
path: "/logs",
use: [ApiGateway.serveStatic("logs")],
},
],
log4XXResponses: false,
logRequestParams: true,
logResponseData: true,
assets: { folder: "public", options: {} },
},
events: {},
methods: {},
started: this.startWatcher,
stopped: this.stopWatcher,
});
}
/**
* Creates an instance of ApiService.
* @param {ServiceBroker} broker - The Moleculer service broker instance.
*/
public constructor(broker: ServiceBroker) {
super(broker);
this.parseServiceSchema({
name: "api",
mixins: [ApiGateway],
settings: {
port: process.env.PORT || 3000,
routes: [
{
path: "/graphql",
whitelist: ["graphql.*"],
bodyParsers: {
json: true,
urlencoded: { extended: true },
},
aliases: {
"POST /": "graphql.wantedComics",
},
cors: {
origin: "*",
methods: ["GET", "OPTIONS", "POST"],
allowedHeaders: ["*"],
credentials: false,
},
},
{
path: "/api",
whitelist: ["**"],
cors: {
origin: "*",
methods: [
"GET",
"OPTIONS",
"POST",
"PUT",
"DELETE",
],
allowedHeaders: ["*"],
exposedHeaders: [],
credentials: false,
maxAge: 3600,
},
use: [],
mergeParams: true,
authentication: false,
authorization: false,
autoAliases: true,
aliases: {},
callingOptions: {},
bodyParsers: {
json: { strict: false, limit: "1MB" },
urlencoded: { extended: true, limit: "1MB" },
},
mappingPolicy: "all",
logging: true,
},
{
path: "/userdata",
use: [
ApiGateway.serveStatic(path.resolve("./userdata")),
],
},
{
path: "/comics",
use: [ApiGateway.serveStatic(path.resolve("./comics"))],
},
{
path: "/logs",
use: [ApiGateway.serveStatic("logs")],
},
],
log4XXResponses: false,
logRequestParams: true,
logResponseData: true,
assets: { folder: "public", options: {} },
},
events: {},
methods: {},
started: this.startWatcher,
stopped: this.stopWatcher,
});
}
/**
* Initializes and starts the chokidar watcher on the COMICS_DIRECTORY.
* Debounces rapid events and logs initial scan completion.
* @private
*/
private startWatcher(): void {
const rawDir = process.env.COMICS_DIRECTORY;
if (!rawDir) {
this.logger.error("COMICS_DIRECTORY not set; cannot start watcher");
return;
}
const watchDir = path.resolve(rawDir);
this.logger.info(`Watching comics folder at: ${watchDir}`);
if (!fs.existsSync(watchDir)) {
this.logger.error(`✖ Comics folder does not exist: ${watchDir}`);
return;
}
/**
* Initializes and starts the chokidar watcher on the COMICS_DIRECTORY.
* Debounces rapid events and logs initial scan completion.
* @private
*/
private startWatcher(): void {
const rawDir = process.env.COMICS_DIRECTORY;
if (!rawDir) {
this.logger.error("COMICS_DIRECTORY not set; cannot start watcher");
return;
}
const watchDir = path.resolve(rawDir);
this.logger.info(`Watching comics folder at: ${watchDir}`);
if (!fs.existsSync(watchDir)) {
this.logger.error(`✖ Comics folder does not exist: ${watchDir}`);
return;
}
this.fileWatcher = chokidar.watch(watchDir, {
persistent: true,
ignoreInitial: true,
followSymlinks: true,
depth: 10,
usePolling: true,
interval: 5000,
atomic: true,
awaitWriteFinish: { stabilityThreshold: 2000, pollInterval: 100 },
ignored: (p) => p.endsWith(".dctmp") || p.includes("/.git/"),
});
this.fileWatcher = chokidar.watch(watchDir, {
persistent: true,
ignoreInitial: true,
followSymlinks: true,
depth: 10,
usePolling: true,
interval: 5000,
atomic: true,
awaitWriteFinish: { stabilityThreshold: 2000, pollInterval: 100 },
ignored: (p) => p.endsWith(".dctmp") || p.includes("/.git/"),
});
/**
* Debounced handler for file system events, batching rapid triggers
* into a 200ms window. Leading and trailing calls invoked.
* @param {string} event - Type of file event (add, change, etc.).
* @param {string} p - Path of the file or directory.
* @param {fs.Stats} [stats] - Optional file stats for add/change events.
*/
const debouncedEvent = debounce(
(event: string, p: string, stats?: fs.Stats) => {
try {
this.handleFileEvent(event, p, stats);
} catch (err) {
this.logger.error(
`Error handling file event [${event}] for ${p}:`,
err
);
}
},
200,
{ leading: true, trailing: true }
);
/**
* Debounced handler for file system events, batching rapid triggers
* into a 200ms window. Leading and trailing calls invoked.
* @param {string} event - Type of file event (add, change, etc.).
* @param {string} p - Path of the file or directory.
* @param {fs.Stats} [stats] - Optional file stats for add/change events.
*/
const debouncedEvent = debounce(
(event: string, p: string, stats?: fs.Stats) => {
try {
this.handleFileEvent(event, p, stats);
} catch (err) {
this.logger.error(
`Error handling file event [${event}] for ${p}:`,
err
);
}
},
200,
{ leading: true, trailing: true }
);
this.fileWatcher
.on("ready", () => this.logger.info("Initial scan complete."))
.on("error", (err) => this.logger.error("Watcher error:", err))
.on("add", (p, stats) => debouncedEvent("add", p, stats))
.on("change", (p, stats) => debouncedEvent("change", p, stats))
.on("unlink", (p) => debouncedEvent("unlink", p))
.on("addDir", (p) => debouncedEvent("addDir", p))
.on("unlinkDir", (p) => debouncedEvent("unlinkDir", p));
}
this.fileWatcher
.on("ready", () => this.logger.info("Initial scan complete."))
.on("error", (err) => this.logger.error("Watcher error:", err))
.on("add", (p, stats) => debouncedEvent("add", p, stats))
.on("change", (p, stats) => debouncedEvent("change", p, stats))
.on("unlink", (p) => debouncedEvent("unlink", p))
.on("addDir", (p) => debouncedEvent("addDir", p))
.on("unlinkDir", (p) => debouncedEvent("unlinkDir", p));
}
/**
* Stops and closes the chokidar watcher, freeing resources.
* @private
*/
private async stopWatcher(): Promise<void> {
if (this.fileWatcher) {
this.logger.info("Stopping file watcher...");
await this.fileWatcher.close();
this.fileWatcher = undefined;
}
}
/**
* Stops and closes the chokidar watcher, freeing resources.
* @private
*/
private async stopWatcher(): Promise<void> {
if (this.fileWatcher) {
this.logger.info("Stopping file watcher...");
await this.fileWatcher.close();
this.fileWatcher = undefined;
}
}
/**
* Handles a filesystem event by logging and optionally importing new files.
* @param event - The type of chokidar event ('add', 'change', 'unlink', etc.).
* @param filePath - The full path of the file or directory that triggered the event.
* @param stats - Optional fs.Stats data for 'add' or 'change' events.
* @private
*/
private async handleFileEvent(
event: string,
filePath: string,
stats?: fs.Stats
): Promise<void> {
this.logger.info(`File event [${event}]: ${filePath}`);
if (event === "add" && stats) {
setTimeout(async () => {
const newStats = await fs.promises.stat(filePath);
if (newStats.mtime.getTime() === stats.mtime.getTime()) {
this.logger.info(`Stable file detected: ${filePath}, importing.`);
const folderData: IFolderData = await this.broker.call(
"library.walkFolders",
{ basePathToWalk: filePath }
);
// this would have to be a call to importDownloadedComic
await this.broker.call("importqueue.processImport", {
fileObject: {
filePath,
fileSize: folderData[0].fileSize,
},
});
}
}, 3000);
}
this.broker.broadcast(event, { path: filePath });
}
/**
* Handles a filesystem event by logging and optionally importing new files.
* @param event - The type of chokidar event ('add', 'change', 'unlink', etc.).
* @param filePath - The full path of the file or directory that triggered the event.
* @param stats - Optional fs.Stats data for 'add' or 'change' events.
* @private
*/
private async handleFileEvent(
event: string,
filePath: string,
stats?: fs.Stats
): Promise<void> {
this.logger.info(`File event [${event}]: ${filePath}`);
if (event === "add" && stats) {
setTimeout(async () => {
const newStats = await fs.promises.stat(filePath);
if (newStats.mtime.getTime() === stats.mtime.getTime()) {
this.logger.info(
`Stable file detected: ${filePath}, importing.`
);
const folderData: IFolderData = await this.broker.call(
"library.walkFolders",
{ basePathToWalk: filePath }
);
// this would have to be a call to importDownloadedComic
await this.broker.call("importqueue.processImport", {
fileObject: {
filePath,
fileSize: folderData[0].fileSize,
},
});
}
}, 3000);
}
this.broker.broadcast(event, { path: filePath });
}
}

116
services/graphql.service.ts Normal file
View File

@@ -0,0 +1,116 @@
// services/graphql.service.ts
import { gql as ApolloMixin } from "@ltv/moleculer-apollo-server-mixin";
import { print } from "graphql";
import { typeDefs } from "../models/graphql/typedef";
import { ServiceSchema } from "moleculer";
/**
* Interface representing the structure of an ElasticSearch result.
*/
interface SearchResult {
hits: {
total: { value: number };
hits: any[];
};
}
/**
* GraphQL Moleculer Service exposing typed resolvers via @ltv/moleculer-apollo-server-mixin.
* Includes resolver for fetching comics marked as "wanted".
*/
const GraphQLService: ServiceSchema = {
name: "graphql",
mixins: [ApolloMixin],
actions: {
/**
* Resolver for fetching comics marked as "wanted" in ElasticSearch.
*
* Queries the `search.issue` Moleculer action using a filtered ES query
* that matches issues or volumes with a `wanted` flag.
*
* @param {number} [limit=25] - Maximum number of results to return.
* @param {number} [offset=0] - Starting index for paginated results.
* @returns {Promise<{ total: number, comics: any[] }>} - Total number of matches and result set.
*
* @example
* query {
* wantedComics(limit: 10, offset: 0) {
* total
* comics {
* _id
* _source {
* title
* }
* }
* }
* }
*/
wantedComics: {
params: {
limit: {
type: "number",
integer: true,
min: 1,
optional: true,
},
offset: {
type: "number",
integer: true,
min: 0,
optional: true,
},
},
async handler(ctx) {
const { limit = 25, offset = 0 } = ctx.params;
const eSQuery = {
bool: {
should: [
{ exists: { field: "wanted.issues" } },
{ exists: { field: "wanted.volume" } },
],
minimum_should_match: 1,
},
};
const result = (await ctx.broker.call("search.issue", {
query: eSQuery,
pagination: { size: limit, from: offset },
type: "wanted",
trigger: "wantedComicsGraphQL",
})) as SearchResult;
return {
data: {
wantedComics: {
total: result?.hits?.total?.value || 0,
comics:
result?.hits?.hits.map((hit) => hit._source) ||
[],
},
},
};
},
},
},
settings: {
apolloServer: {
typeDefs: print(typeDefs), // If typeDefs is AST; remove print if it's raw SDL string
resolvers: {
Query: {
wantedComics: "graphql.wantedComics",
},
},
path: "/graphql",
playground: true,
introspection: true,
context: ({ ctx }: any) => ({
broker: ctx.broker,
}),
},
},
};
export default GraphQLService;

View File

@@ -74,7 +74,7 @@ export default class JobQueueService extends Service {
},
},
// Comic Book Import Job Queue
// Comic Book Import Job Queue - Enhanced for better metadata handling
"enqueue.async": {
handler: async (
ctx: Context<{
@@ -83,7 +83,7 @@ export default class JobQueueService extends Service {
) => {
try {
console.log(
`Recieved Job ID ${ctx.locals.job.id}, processing...`
`Received Job ID ${ctx.locals.job.id}, processing...`
);
// 1. De-structure the job params
const { fileObject } = ctx.locals.job.data.params;
@@ -112,15 +112,43 @@ export default class JobQueueService extends Service {
JSON.stringify(inferredIssueDetails, null, 2)
);
// 3b. Orchestrate the payload
const payload = {
importStatus: {
isImported: true,
tagged: false,
matchedResult: {
score: "0",
},
// 3b. Prepare sourced metadata from various sources
let sourcedMetadata = {
comicInfo: comicInfoJSON || {},
comicvine: {},
metron: {},
gcd: {},
locg: {}
};
// Include any external metadata if provided
if (!isNil(ctx.locals.job.data.params.sourcedMetadata)) {
const providedMetadata = ctx.locals.job.data.params.sourcedMetadata;
sourcedMetadata = {
...sourcedMetadata,
...providedMetadata
};
}
// 3c. Prepare inferred metadata matching Comic model structure
const inferredMetadata = {
series: inferredIssueDetails?.name || "Unknown Series",
issue: {
name: inferredIssueDetails?.name || "Unknown Series",
number: inferredIssueDetails?.number || 1,
subtitle: inferredIssueDetails?.subtitle || "",
year: inferredIssueDetails?.year || new Date().getFullYear().toString()
},
volume: 1, // Default volume since not available in inferredIssueDetails
title: inferredIssueDetails?.name || path.basename(filePath, path.extname(filePath))
};
// 3d. Create canonical metadata - user-curated values with source attribution
const canonicalMetadata = this.createCanonicalMetadata(sourcedMetadata, inferredMetadata);
// 3e. Create comic payload with canonical metadata structure
const comicPayload = {
// File details
rawFileDetails: {
name,
filePath,
@@ -130,58 +158,37 @@ export default class JobQueueService extends Service {
containedIn,
cover,
},
inferredMetadata: {
issue: inferredIssueDetails,
},
sourcedMetadata: {
// except for ComicInfo.xml, everything else should be copied over from the
// parent comic
comicInfo: comicInfoJSON,
},
// since we already have at least 1 copy
// mark it as not wanted by default
// Enhanced sourced metadata (now supports more sources)
sourcedMetadata,
// Original inferred metadata
inferredMetadata,
// New canonical metadata - user-curated values with source attribution
canonicalMetadata,
// Import status
"acquisition.source.wanted": false,
// clear out the downloads array
// "acquisition.directconnect.downloads": [],
// mark the metadata source
"acquisition.source.name":
ctx.locals.job.data.params.sourcedFrom,
"acquisition.source.name": ctx.locals.job.data.params.sourcedFrom,
};
// 3c. Add the bundleId, if present to the payload
// 3f. Add bundleId if present
let bundleId = null;
if (!isNil(ctx.locals.job.data.params.bundleId)) {
bundleId = ctx.locals.job.data.params.bundleId;
}
// 3d. Add the sourcedMetadata, if present
if (
!isNil(
ctx.locals.job.data.params.sourcedMetadata
) &&
!isUndefined(
ctx.locals.job.data.params.sourcedMetadata
.comicvine
)
) {
Object.assign(
payload.sourcedMetadata,
ctx.locals.job.data.params.sourcedMetadata
);
}
// 4. write to mongo
// 4. Use library service to import with enhanced metadata
const importResult = await this.broker.call(
"library.rawImportToDB",
"library.importFromJob",
{
importType:
ctx.locals.job.data.params.importType,
importType: ctx.locals.job.data.params.importType,
bundleId,
payload,
payload: comicPayload,
}
);
return {
data: {
importResult,
@@ -196,7 +203,7 @@ export default class JobQueueService extends Service {
throw new MoleculerError(
error,
500,
"IMPORT_JOB_ERROR",
"ENHANCED_IMPORT_JOB_ERROR",
{
data: ctx.params.sessionId,
}
@@ -303,7 +310,7 @@ export default class JobQueueService extends Service {
}>
) => {
console.log(
`Recieved Job ID ${JSON.stringify(
`Received Job ID ${JSON.stringify(
ctx.locals
)}, processing...`
);
@@ -438,7 +445,239 @@ export default class JobQueueService extends Service {
});
},
},
methods: {},
methods: {
/**
* Create canonical metadata structure with source attribution for user-driven curation
* @param sourcedMetadata - Metadata from various external sources
* @param inferredMetadata - Metadata inferred from filename/file analysis
*/
createCanonicalMetadata(sourcedMetadata: any, inferredMetadata: any) {
const currentTime = new Date();
// Priority order: comicInfo -> comicvine -> metron -> gcd -> locg -> inferred
const sourcePriority = ['comicInfo', 'comicvine', 'metron', 'gcd', 'locg'];
// Helper function to extract actual value from metadata (handle arrays, etc.)
const extractValue = (value: any) => {
if (Array.isArray(value)) {
return value.length > 0 ? value[0] : null;
}
return value;
};
// Helper function to find the best value and its source
const findBestValue = (fieldName: string, defaultValue: any = null, defaultSource: string = 'inferred') => {
for (const source of sourcePriority) {
const rawValue = sourcedMetadata[source]?.[fieldName];
if (rawValue !== undefined && rawValue !== null && rawValue !== '') {
const extractedValue = extractValue(rawValue);
if (extractedValue !== null && extractedValue !== '') {
return {
value: extractedValue,
source: source,
userSelected: false,
lastModified: currentTime
};
}
}
}
return {
value: defaultValue,
source: defaultSource,
userSelected: false,
lastModified: currentTime
};
};
// Helper function for series-specific field resolution
const findSeriesValue = (fieldNames: string[], defaultValue: any = null) => {
for (const source of sourcePriority) {
const metadata = sourcedMetadata[source];
if (metadata) {
for (const fieldName of fieldNames) {
const rawValue = metadata[fieldName];
if (rawValue !== undefined && rawValue !== null && rawValue !== '') {
const extractedValue = extractValue(rawValue);
if (extractedValue !== null && extractedValue !== '') {
return {
value: extractedValue,
source: source,
userSelected: false,
lastModified: currentTime
};
}
}
}
}
}
return {
value: defaultValue,
source: 'inferred',
userSelected: false,
lastModified: currentTime
};
};
const canonical: any = {
// Core identifying information
title: findBestValue('title', inferredMetadata.title),
// Series information
series: {
name: findSeriesValue(['series', 'seriesName', 'name'], inferredMetadata.series),
volume: findBestValue('volume', inferredMetadata.volume || 1),
startYear: findBestValue('startYear', inferredMetadata.issue?.year ? parseInt(inferredMetadata.issue.year) : new Date().getFullYear())
},
// Issue information
issueNumber: findBestValue('issueNumber', inferredMetadata.issue?.number?.toString() || "1"),
// Publishing information
publisher: findBestValue('publisher', null),
publicationDate: findBestValue('publicationDate', null),
coverDate: findBestValue('coverDate', null),
// Content information
pageCount: findBestValue('pageCount', null),
summary: findBestValue('summary', null),
// Creator information - collect from all sources for richer data
creators: [],
// Character and genre arrays with source tracking
characters: {
values: [],
source: 'inferred',
userSelected: false,
lastModified: currentTime
},
genres: {
values: [],
source: 'inferred',
userSelected: false,
lastModified: currentTime
},
// Canonical metadata tracking
lastCanonicalUpdate: currentTime,
hasUserModifications: false,
// Quality and completeness tracking
completeness: {
score: 0,
missingFields: [],
lastCalculated: currentTime
}
};
// Handle creators - combine from all sources but track source attribution
const allCreators: any[] = [];
for (const source of sourcePriority) {
const metadata = sourcedMetadata[source];
if (metadata?.creators) {
metadata.creators.forEach((creator: any) => {
allCreators.push({
name: extractValue(creator.name),
role: extractValue(creator.role),
source: source,
userSelected: false,
lastModified: currentTime
});
});
} else {
// Handle legacy writer/artist fields
if (metadata?.writer) {
allCreators.push({
name: extractValue(metadata.writer),
role: 'Writer',
source: source,
userSelected: false,
lastModified: currentTime
});
}
if (metadata?.artist) {
allCreators.push({
name: extractValue(metadata.artist),
role: 'Artist',
source: source,
userSelected: false,
lastModified: currentTime
});
}
}
}
canonical.creators = allCreators;
// Handle characters - combine from all sources
const allCharacters = new Set();
let characterSource = 'inferred';
for (const source of sourcePriority) {
if (sourcedMetadata[source]?.characters && sourcedMetadata[source].characters.length > 0) {
sourcedMetadata[source].characters.forEach((char: string) => allCharacters.add(char));
if (characterSource === 'inferred') characterSource = source; // Use the first source found
}
}
canonical.characters = {
values: Array.from(allCharacters),
source: characterSource,
userSelected: false,
lastModified: currentTime
};
// Handle genres - combine from all sources
const allGenres = new Set();
let genreSource = 'inferred';
for (const source of sourcePriority) {
if (sourcedMetadata[source]?.genres && sourcedMetadata[source].genres.length > 0) {
sourcedMetadata[source].genres.forEach((genre: string) => allGenres.add(genre));
if (genreSource === 'inferred') genreSource = source; // Use the first source found
}
}
canonical.genres = {
values: Array.from(allGenres),
source: genreSource,
userSelected: false,
lastModified: currentTime
};
// Calculate completeness score
const requiredFields = ['title', 'series.name', 'issueNumber', 'publisher'];
const optionalFields = ['publicationDate', 'coverDate', 'pageCount', 'summary'];
const missingFields = [];
let filledCount = 0;
// Check required fields
requiredFields.forEach(field => {
const fieldPath = field.split('.');
let value = canonical;
for (const path of fieldPath) {
value = value?.[path];
}
if (value?.value) {
filledCount++;
} else {
missingFields.push(field);
}
});
// Check optional fields
optionalFields.forEach(field => {
if (canonical[field]?.value) {
filledCount++;
}
});
const totalFields = requiredFields.length + optionalFields.length;
canonical.completeness = {
score: Math.round((filledCount / totalFields) * 100),
missingFields: missingFields,
lastCalculated: currentTime
};
return canonical;
}
},
});
}
}

View File

@@ -863,8 +863,57 @@ export default class ImportService extends Service {
console.log(ctx.params);
},
},
/**
* Enhanced import from job queue - works with enhanced Comic model
*/
importFromJob: {
params: {
importType: "string",
bundleId: { type: "string", optional: true },
payload: "object"
},
async handler(ctx: Context<{
importType: string;
bundleId?: string;
payload: any;
}>) {
try {
const { importType, bundleId, payload } = ctx.params;
console.log(`Importing comic with enhanced metadata processing...`);
// Create comic with enhanced metadata structure
const comic = new Comic({
...payload,
importStatus: {
isImported: true,
tagged: false,
lastProcessed: new Date()
}
});
await comic.save();
console.log(`Successfully imported comic: ${comic._id}`);
console.log(`Resolved metadata: ${JSON.stringify(comic.resolvedMetadata)}`);
return {
success: true,
comic: comic._id,
metadata: {
sources: Object.keys(comic.sourcedMetadata || {}),
resolvedFields: Object.keys(comic.resolvedMetadata || {}),
primarySource: comic.resolvedMetadata?.primarySource || 'inferred'
}
};
} catch (error) {
console.error("Error importing comic:", error);
throw error;
}
}
}
},
methods: {},
methods: {}
});
}
}

View File

@@ -1,6 +1,4 @@
import WebSocket from "ws";
// const { Socket } = require("airdcpp-apisocket");
import { Socket } from "airdcpp-apisocket";
/**
* Wrapper around the AirDC++ WebSocket API socket.
@@ -27,6 +25,12 @@ class AirDCPPSocket {
*/
private socketInstance: any;
/**
* Promise that resolves when the Socket module is loaded
* @private
*/
private socketModulePromise: Promise<any>;
/**
* Constructs a new AirDCPPSocket wrapper.
* @param {{ protocol: string; hostname: string; username: string; password: string }} configuration
@@ -53,8 +57,13 @@ class AirDCPPSocket {
username: configuration.username,
password: configuration.password,
};
// Initialize the AirDC++ socket instance
this.socketInstance = Socket(this.options, WebSocket);
// Use dynamic import to load the ES module
this.socketModulePromise = import("airdcpp-apisocket").then(module => {
const { Socket } = module;
this.socketInstance = Socket(this.options, WebSocket);
return this.socketInstance;
});
}
/**
@@ -63,6 +72,7 @@ class AirDCPPSocket {
* @returns {Promise<any>} Session information returned by the server.
*/
async connect(): Promise<any> {
await this.socketModulePromise;
if (
this.socketInstance &&
typeof this.socketInstance.connect === "function"
@@ -80,6 +90,7 @@ class AirDCPPSocket {
* @returns {Promise<void>}
*/
async disconnect(): Promise<void> {
await this.socketModulePromise;
if (
this.socketInstance &&
typeof this.socketInstance.disconnect === "function"
@@ -96,6 +107,7 @@ class AirDCPPSocket {
* @returns {Promise<any>} Response from the AirDC++ server.
*/
async post(endpoint: string, data: object = {}): Promise<any> {
await this.socketModulePromise;
return await this.socketInstance.post(endpoint, data);
}
@@ -107,6 +119,7 @@ class AirDCPPSocket {
* @returns {Promise<any>} Response from the AirDC++ server.
*/
async get(endpoint: string, data: object = {}): Promise<any> {
await this.socketModulePromise;
return await this.socketInstance.get(endpoint, data);
}
@@ -125,6 +138,7 @@ class AirDCPPSocket {
callback: (...args: any[]) => void,
id?: string | number
): Promise<any> {
await this.socketModulePromise;
return await this.socketInstance.addListener(
event,
handlerName,

View File

@@ -1,24 +0,0 @@
<?xml version="1.0"?>
<ComicInfo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:xsd="http://www.w3.org/2001/XMLSchema">
<Title>Title of the Book</Title>
<Summary>A description of the book</Summary>
<Number>1</Number>
<Count>3</Count>
<Year>2010</Year>
<Month>4</Month>
<Writer>Author name</Writer>
<Publisher>self</Publisher>
<Genre>educational</Genre>
<BlackAndWhite>No</BlackAndWhite>
<Manga>No</Manga>
<Characters>Superman</Characters>
<PageCount>5</PageCount>
<Pages>
<Page Image="0" Type="FrontCover" ImageSize="139382" ImageWidth="774" ImageHeight="1024" />
<Page Image="2" ImageSize="125736" ImageWidth="797" ImageHeight="1024" />
<Page Image="1" ImageSize="127937" ImageWidth="797" ImageHeight="1024" />
<Page Image="4" ImageSize="160902" ImageWidth="804" ImageHeight="1024" />
<Page Image="3" ImageSize="211181" ImageWidth="804" ImageHeight="1024" />
</Pages>
</ComicInfo>