diff --git a/.jshintrc b/.jshintrc deleted file mode 100644 index 816db2b..0000000 --- a/.jshintrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "esversion": 10 -} \ No newline at end of file diff --git a/MOLECULER_DEPENDENCY_ANALYSIS.md b/MOLECULER_DEPENDENCY_ANALYSIS.md new file mode 100644 index 0000000..a2294d7 --- /dev/null +++ b/MOLECULER_DEPENDENCY_ANALYSIS.md @@ -0,0 +1,423 @@ +# Moleculer Microservices Dependency Analysis +**ThreeTwo Core Service - Comic Book Library Management System** + +## System Overview + +This **ThreeTwo Core Service** is a sophisticated **comic book library management system** built on Moleculer microservices architecture. The system demonstrates advanced patterns including: + +- **Event-driven architecture** with real-time WebSocket communication +- **Asynchronous job processing** with BullMQ for heavy operations +- **Multi-source metadata aggregation** with canonical data resolution +- **Hybrid search** combining MongoDB aggregation and ElasticSearch +- **External system integrations** (P2P, BitTorrent, Comic APIs) + +### Technical Stack + +- **Framework**: Moleculer.js microservices +- **Node ID**: `threetwo-core-service` +- **Transport**: Redis (`redis://localhost:6379`) +- **Databases**: MongoDB + ElasticSearch +- **Queue System**: BullMQ (Redis-backed) +- **Real-time**: Socket.IO with Redis adapter +- **External APIs**: ComicVine, AirDC++, qBittorrent + +## Service Architecture + +### Core Services + +| Service | File | Role | Dependencies | +|---------|------|------|-------------| +| **API** | [`api.service.ts`](services/api.service.ts) | API Gateway + File System Watcher | → library, jobqueue | +| **Library** | [`library.service.ts`](services/library.service.ts) | Core Comic Library Management | → jobqueue, search, comicvine | +| **JobQueue** | [`jobqueue.service.ts`](services/jobqueue.service.ts) | Asynchronous Job Processing (BullMQ) | → library, socket | +| **Socket** | [`socket.service.ts`](services/socket.service.ts) | Real-time Communication (Socket.IO) | → library, jobqueue | +| **Search** | [`search.service.ts`](services/search.service.ts) | ElasticSearch Integration | ElasticSearch client | +| **GraphQL** | [`graphql.service.ts`](services/graphql.service.ts) | GraphQL API Layer | → search | + +### Supporting Services + +| Service | File | Role | Dependencies | +|---------|------|------|-------------| +| **AirDC++** | [`airdcpp.service.ts`](services/airdcpp.service.ts) | P2P File Sharing Integration | External AirDC++ client | +| **Settings** | [`settings.service.ts`](services/settings.service.ts) | Configuration Management | MongoDB | +| **Image Transform** | [`imagetransformation.service.ts`](services/imagetransformation.service.ts) | Cover Processing | File system | +| **OPDS** | [`opds.service.ts`](services/opds.service.ts) | Comic Catalog Feeds | File system | +| **Torrent Jobs** | [`torrentjobs.service.ts`](services/torrentjobs.service.ts) | BitTorrent Integration | → library, qbittorrent | + +## Service-to-Service Dependencies + +### Core Service Interactions + +#### 1. API Service → Other Services +```typescript +// File system watcher triggers import +ctx.broker.call("library.walkFolders", { basePathToWalk: filePath }) +ctx.broker.call("importqueue.processImport", { fileObject }) +``` + +#### 2. Library Service → Dependencies +```typescript +// Job queue integration +this.broker.call("jobqueue.enqueue", { action: "enqueue.async" }) + +// Search operations +ctx.broker.call("search.searchComic", { elasticSearchQueries }) +ctx.broker.call("search.deleteElasticSearchIndices", {}) + +// External metadata +ctx.broker.call("comicvine.getVolumes", { volumeURI }) +``` + +#### 3. JobQueue Service → Dependencies +```typescript +// Import processing +this.broker.call("library.importFromJob", { importType, payload }) + +// Real-time updates +this.broker.call("socket.broadcast", { + namespace: "/", + event: "LS_COVER_EXTRACTED", + args: [{ completedJobCount, importResult }] +}) +``` + +#### 4. Socket Service → Dependencies +```typescript +// Job management +ctx.broker.call("jobqueue.getJobCountsByType", {}) +ctx.broker.call("jobqueue.toggle", { action: queueAction }) + +// Download tracking +ctx.call("library.applyAirDCPPDownloadMetadata", { + bundleId, comicObjectId, name, size, type +}) +``` + +#### 5. GraphQL Service → Search +```typescript +// Wanted comics query +const result = await ctx.broker.call("search.issue", { + query: eSQuery, + pagination: { size: limit, from: offset }, + type: "wanted" +}) +``` + +## API Endpoint Mapping + +### REST API Routes (`/api/*`) + +#### Library Management +- `POST /api/library/walkFolders` → [`library.walkFolders`](services/library.service.ts:82) +- `POST /api/library/newImport` → [`library.newImport`](services/library.service.ts:165) → [`jobqueue.enqueue`](services/library.service.ts:219) +- `POST /api/library/getComicBooks` → [`library.getComicBooks`](services/library.service.ts:535) +- `POST /api/library/getComicBookById` → [`library.getComicBookById`](services/library.service.ts:550) +- `POST /api/library/flushDB` → [`library.flushDB`](services/library.service.ts:818) → [`search.deleteElasticSearchIndices`](services/library.service.ts:839) +- `GET /api/library/libraryStatistics` → [`library.libraryStatistics`](services/library.service.ts:684) + +#### Job Management +- `GET /api/jobqueue/getJobCountsByType` → [`jobqueue.getJobCountsByType`](services/jobqueue.service.ts:31) +- `GET /api/jobqueue/toggle` → [`jobqueue.toggle`](services/jobqueue.service.ts:38) +- `GET /api/jobqueue/getJobResultStatistics` → [`jobqueue.getJobResultStatistics`](services/jobqueue.service.ts:214) + +#### Search Operations +- `POST /api/search/searchComic` → [`search.searchComic`](services/search.service.ts:28) +- `POST /api/search/searchIssue` → [`search.issue`](services/search.service.ts:60) +- `GET /api/search/deleteElasticSearchIndices` → [`search.deleteElasticSearchIndices`](services/search.service.ts:171) + +#### AirDC++ Integration +- `POST /api/airdcpp/initialize` → [`airdcpp.initialize`](services/airdcpp.service.ts:24) +- `POST /api/airdcpp/getHubs` → [`airdcpp.getHubs`](services/airdcpp.service.ts:59) +- `POST /api/airdcpp/search` → [`airdcpp.search`](services/airdcpp.service.ts:96) + +#### Image Processing +- `POST /api/imagetransformation/resizeImage` → [`imagetransformation.resize`](services/imagetransformation.service.ts:37) +- `POST /api/imagetransformation/analyze` → [`imagetransformation.analyze`](services/imagetransformation.service.ts:57) + +### GraphQL Endpoints +- `POST /graphql` → [`graphql.wantedComics`](services/graphql.service.ts:49) → [`search.issue`](services/graphql.service.ts:77) + +### Static File Serving +- `/userdata/*` → Static files from `./userdata` +- `/comics/*` → Static files from `./comics` +- `/logs/*` → Static files from `logs` + +## Event-Driven Communication + +### Job Queue Events + +#### Job Completion Events +```typescript +// Successful import completion +"enqueue.async.completed" → socket.broadcast("LS_COVER_EXTRACTED", { + completedJobCount, + importResult: job.returnvalue.data.importResult +}) + +// Failed import handling +"enqueue.async.failed" → socket.broadcast("LS_COVER_EXTRACTION_FAILED", { + failedJobCount, + importResult: job +}) + +// Queue drained +"drained" → socket.broadcast("LS_IMPORT_QUEUE_DRAINED", { + message: "drained" +}) +``` + +#### Archive Processing Events +```typescript +// Archive uncompression completed +"uncompressFullArchive.async.completed" → socket.broadcast("LS_UNCOMPRESSION_JOB_COMPLETE", { + uncompressedArchive: job.returnvalue +}) +``` + +### File System Events +```typescript +// File watcher events (debounced 200ms) +fileWatcher.on("add", (path, stats) → { + broker.call("library.walkFolders", { basePathToWalk: filePath }) + broker.call("importqueue.processImport", { fileObject }) + broker.broadcast(event, { path: filePath }) +}) +``` + +### WebSocket Events + +#### Real-time Search +```typescript +// Search initiation +socket.emit("searchInitiated", { instance }) + +// Live search results +socket.emit("searchResultAdded", groupedResult) +socket.emit("searchResultUpdated", updatedResult) +socket.emit("searchComplete", { message }) +``` + +#### Download Progress +```typescript +// Download status +broker.emit("downloadCompleted", bundleDBImportResult) +broker.emit("downloadError", error.message) + +// Progress tracking +socket.emit("downloadTick", data) +``` + +## Data Flow Architecture + +### 1. Comic Import Processing Flow +```mermaid +graph TD + A[File System Watcher] --> B[library.walkFolders] + B --> C[jobqueue.enqueue] + C --> D[jobqueue.enqueue.async] + D --> E[Archive Extraction] + E --> F[Metadata Processing] + F --> G[Canonical Metadata Creation] + G --> H[library.importFromJob] + H --> I[MongoDB Storage] + I --> J[ElasticSearch Indexing] + J --> K[socket.broadcast LS_COVER_EXTRACTED] +``` + +### 2. Search & Discovery Flow +```mermaid +graph TD + A[GraphQL/REST Query] --> B[search.issue] + B --> C[ElasticSearch Query] + C --> D[Results Enhancement] + D --> E[Metadata Scoring] + E --> F[Structured Response] +``` + +### 3. Download Management Flow +```mermaid +graph TD + A[socket[search]] --> B[airdcpp.search] + B --> C[Real-time Results] + C --> D[socket[download]] + D --> E[library.applyAirDCPPDownloadMetadata] + E --> F[Progress Tracking] + F --> G[Import Pipeline] +``` + +## Database Dependencies + +### MongoDB Collections +| Collection | Model | Used By Services | +|------------|-------|-----------------| +| **comics** | [`Comic`](models/comic.model.ts) | library, search, jobqueue, imagetransformation | +| **settings** | [`Settings`](models/settings.model.ts) | settings | +| **sessions** | [`Session`](models/session.model.ts) | socket | +| **jobresults** | [`JobResult`](models/jobresult.model.ts) | jobqueue | + +### ElasticSearch Integration +- **Index**: `comics` - Full-text search with metadata scoring +- **Client**: [`eSClient`](services/search.service.ts:13) from [`comic.model.ts`](models/comic.model.ts) +- **Query Types**: match_all, multi_match, bool queries with field boosting + +### Redis Usage +| Purpose | Services | Configuration | +|---------|----------|---------------| +| **Transport** | All services | [`moleculer.config.ts:93`](moleculer.config.ts:93) | +| **Job Queue** | jobqueue | [`jobqueue.service.ts:27`](services/jobqueue.service.ts:27) | +| **Socket.IO Adapter** | socket | [`socket.service.ts:48`](services/socket.service.ts:48) | +| **Job Counters** | jobqueue | [`completedJobCount`](services/jobqueue.service.ts:392), [`failedJobCount`](services/jobqueue.service.ts:422) | + +## External System Integrations + +### AirDC++ (P2P File Sharing) +```typescript +// Integration wrapper +const ADCPPSocket = new AirDCPPSocket(config) +await ADCPPSocket.connect() + +// Search operations +const searchInstance = await ADCPPSocket.post("search") +const searchInfo = await ADCPPSocket.post(`search/${searchInstance.id}/hub_search`, query) + +// Download management +const downloadResult = await ADCPPSocket.post(`search/${searchInstanceId}/results/${resultId}/download`) +``` + +### ComicVine API +```typescript +// Metadata enrichment +const volumeDetails = await this.broker.call("comicvine.getVolumes", { + volumeURI: matchedResult.volume.api_detail_url +}) +``` + +### qBittorrent Client +```typescript +// Torrent monitoring +const torrents = await this.broker.call("qbittorrent.getTorrentRealTimeStats", { infoHashes }) +``` + +## Metadata Management System + +### Multi-Source Metadata Aggregation +The system implements sophisticated metadata management with source prioritization: + +#### Source Priority Order +1. **ComicInfo.xml** (embedded in archives) +2. **ComicVine API** (external database) +3. **Metron** (comic database) +4. **Grand Comics Database (GCD)** +5. **League of Comic Geeks (LOCG)** +6. **Filename Inference** (fallback) + +#### Canonical Metadata Structure +```typescript +const canonical = { + title: findBestValue('title', inferredMetadata.title), + series: { + name: findSeriesValue(['series', 'seriesName', 'name'], inferredMetadata.series), + volume: findBestValue('volume', inferredMetadata.volume || 1), + startYear: findBestValue('startYear', inferredMetadata.issue?.year) + }, + issueNumber: findBestValue('issueNumber', inferredMetadata.issue?.number), + publisher: findBestValue('publisher', null), + creators: [], // Combined from all sources + completeness: { + score: calculatedScore, + missingFields: [], + lastCalculated: currentTime + } +} +``` + +## Performance & Scalability Insights + +### Asynchronous Processing +- **Heavy Operations**: Comic import, archive extraction, metadata processing +- **Queue System**: BullMQ with Redis backing for reliability +- **Job Types**: Import processing, archive extraction, torrent monitoring +- **Real-time Updates**: WebSocket progress notifications + +### Search Optimization +- **Dual Storage**: MongoDB (transactional) + ElasticSearch (search) +- **Metadata Scoring**: Canonical metadata with source priority +- **Query Types**: Full-text, field-specific, boolean combinations +- **Caching**: Moleculer built-in memory caching + +### External Integration Resilience +- **Timeout Handling**: Custom timeouts for long-running operations +- **Error Propagation**: Structured error responses with context +- **Connection Management**: Reusable connections for external APIs +- **Retry Logic**: Built-in retry policies for failed operations + +## Critical Dependency Patterns + +### 1. Service Chain Dependencies +- **Import Pipeline**: api → library → jobqueue → socket +- **Search Pipeline**: graphql → search → ElasticSearch +- **Download Pipeline**: socket → airdcpp → library + +### 2. Circular Dependencies (Managed) +- **socket ←→ library**: Download coordination and progress updates +- **jobqueue ←→ socket**: Job progress notifications and queue control + +### 3. Shared Resource Dependencies +- **MongoDB**: library, search, jobqueue, settings services +- **Redis**: All services (transport) + jobqueue (BullMQ) + socket (adapter) +- **ElasticSearch**: search, graphql services + +## Architecture Strengths + +### 1. Separation of Concerns +- **API Gateway**: Pure routing and file serving +- **Business Logic**: Centralized in library service +- **Data Access**: Abstracted through DbMixin +- **External Integration**: Isolated in dedicated services + +### 2. Event-Driven Design +- **File System Events**: Automatic import triggering +- **Job Lifecycle Events**: Progress tracking and error handling +- **Real-time Communication**: WebSocket event broadcasting + +### 3. Robust Metadata Management +- **Multi-Source Aggregation**: ComicVine, ComicInfo.xml, filename inference +- **Canonical Resolution**: Smart metadata merging with source attribution +- **User Curation Support**: Framework for manual metadata override + +### 4. Scalability Features +- **Microservices Architecture**: Independent service scaling +- **Asynchronous Processing**: Heavy operations don't block API responses +- **Redis Transport**: Distributed service communication +- **Job Queue**: Reliable background processing with retry logic + +## Potential Areas for Improvement + +### 1. Service Coupling +- **High Interdependence**: library ←→ jobqueue ←→ socket tight coupling +- **Recommendation**: Event-driven decoupling for some operations + +### 2. Error Handling +- **Inconsistent Patterns**: Mix of raw errors and MoleculerError usage +- **Recommendation**: Standardized error handling middleware + +### 3. Configuration Management +- **Environment Variables**: Direct access vs centralized configuration +- **Recommendation**: Enhanced settings service for runtime configuration + +### 4. Testing Strategy +- **Integration Testing**: Complex service interactions need comprehensive testing +- **Recommendation**: Contract testing between services + +## Summary + +This Moleculer-based architecture demonstrates sophisticated microservices patterns with: + +- **11 specialized services** with clear boundaries +- **47 REST endpoints** + GraphQL layer +- **3 WebSocket namespaces** for real-time communication +- **Multi-database architecture** (MongoDB + ElasticSearch) +- **Advanced job processing** with BullMQ +- **External system integration** (P2P, BitTorrent, Comic APIs) + +The system successfully manages complex domain requirements while maintaining good separation of concerns and providing excellent user experience through real-time updates and comprehensive metadata management. \ No newline at end of file diff --git a/shared/airdcpp.socket.ts b/shared/airdcpp.socket.ts index a2e5444..8d8dba1 100644 --- a/shared/airdcpp.socket.ts +++ b/shared/airdcpp.socket.ts @@ -1,6 +1,4 @@ import WebSocket from "ws"; -// const { Socket } = require("airdcpp-apisocket"); -import { Socket } from "airdcpp-apisocket"; /** * Wrapper around the AirDC++ WebSocket API socket. @@ -21,12 +19,18 @@ class AirDCPPSocket { password: string; }; - /** + /** * Instance of the AirDC++ API socket. * @private */ private socketInstance: any; + /** + * Promise that resolves when the Socket module is loaded + * @private + */ + private socketModulePromise: Promise; + /** * Constructs a new AirDCPPSocket wrapper. * @param {{ protocol: string; hostname: string; username: string; password: string }} configuration @@ -53,8 +57,13 @@ class AirDCPPSocket { username: configuration.username, password: configuration.password, }; - // Initialize the AirDC++ socket instance - this.socketInstance = Socket(this.options, WebSocket); + + // Use dynamic import to load the ES module + this.socketModulePromise = import("airdcpp-apisocket").then(module => { + const { Socket } = module; + this.socketInstance = Socket(this.options, WebSocket); + return this.socketInstance; + }); } /** @@ -63,6 +72,7 @@ class AirDCPPSocket { * @returns {Promise} Session information returned by the server. */ async connect(): Promise { + await this.socketModulePromise; if ( this.socketInstance && typeof this.socketInstance.connect === "function" @@ -80,6 +90,7 @@ class AirDCPPSocket { * @returns {Promise} */ async disconnect(): Promise { + await this.socketModulePromise; if ( this.socketInstance && typeof this.socketInstance.disconnect === "function" @@ -96,6 +107,7 @@ class AirDCPPSocket { * @returns {Promise} Response from the AirDC++ server. */ async post(endpoint: string, data: object = {}): Promise { + await this.socketModulePromise; return await this.socketInstance.post(endpoint, data); } @@ -107,6 +119,7 @@ class AirDCPPSocket { * @returns {Promise} Response from the AirDC++ server. */ async get(endpoint: string, data: object = {}): Promise { + await this.socketModulePromise; return await this.socketInstance.get(endpoint, data); } @@ -125,6 +138,7 @@ class AirDCPPSocket { callback: (...args: any[]) => void, id?: string | number ): Promise { + await this.socketModulePromise; return await this.socketInstance.addListener( event, handlerName, diff --git a/test-canonical-metadata.js b/test-canonical-metadata.js deleted file mode 100644 index e997a38..0000000 --- a/test-canonical-metadata.js +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Test the new canonical metadata system - * This test verifies that comics are imported with proper canonical metadata structure - * that supports user-driven curation with source attribution - */ - -const axios = require('axios'); -const fs = require('fs'); -const path = require('path'); - -const API_BASE = 'http://localhost:3000/api'; - -async function testCanonicalMetadata() { - try { - console.log('🧪 Testing Canonical Metadata System...\n'); - - // Test 1: Use an existing comic file for import - let testComicPath = path.join(__dirname, 'comics', 'Batman Urban Legends # 12.cbr'); - - if (!fs.existsSync(testComicPath)) { - console.log('⚠️ Test comic file not found, trying alternative...'); - // Try an alternative file - testComicPath = path.join(__dirname, 'comics', 'X-men Vol 1 # 21.cbr'); - if (!fs.existsSync(testComicPath)) { - console.log('⚠️ No suitable test comic files found'); - return; - } - } - - // Test 2: Import the comic using the enhanced newImport endpoint - console.log('📚 Importing test comic with canonical metadata...'); - const importResponse = await axios.post(`${API_BASE}/library/newImport`, { - filePath: testComicPath, - importType: 'file', - sourcedFrom: 'test' - }); - - console.log('✅ Import Response Status:', importResponse.status); - const comic = importResponse.data; - - if (!comic) { - console.log('❌ No comic data returned'); - return; - } - - console.log('📊 Comic ID:', comic._id); - console.log('📋 Testing Canonical Metadata Structure...\n'); - - // Test 3: Verify canonical metadata structure - const canonicalMetadata = comic.canonicalMetadata; - - if (!canonicalMetadata) { - console.log('❌ canonicalMetadata field is missing'); - return; - } - - console.log('✅ canonicalMetadata field exists'); - - // Test 4: Verify core fields have source attribution - const coreFields = ['title', 'issueNumber', 'publisher']; - const seriesFields = ['name', 'volume', 'startYear']; - - console.log('\n🔍 Testing Core Field Source Attribution:'); - for (const field of coreFields) { - const fieldData = canonicalMetadata[field]; - if (fieldData && typeof fieldData === 'object') { - const hasRequiredFields = fieldData.hasOwnProperty('value') && - fieldData.hasOwnProperty('source') && - fieldData.hasOwnProperty('userSelected') && - fieldData.hasOwnProperty('lastModified'); - - console.log(` ${field}: ${hasRequiredFields ? '✅' : '❌'} ${JSON.stringify(fieldData)}`); - } else { - console.log(` ${field}: ❌ Missing or invalid structure`); - } - } - - console.log('\n🔍 Testing Series Field Source Attribution:'); - if (canonicalMetadata.series) { - for (const field of seriesFields) { - const fieldData = canonicalMetadata.series[field]; - if (fieldData && typeof fieldData === 'object') { - const hasRequiredFields = fieldData.hasOwnProperty('value') && - fieldData.hasOwnProperty('source') && - fieldData.hasOwnProperty('userSelected') && - fieldData.hasOwnProperty('lastModified'); - - console.log(` series.${field}: ${hasRequiredFields ? '✅' : '❌'} ${JSON.stringify(fieldData)}`); - } else { - console.log(` series.${field}: ❌ Missing or invalid structure`); - } - } - } else { - console.log(' ❌ series field missing'); - } - - // Test 5: Verify completeness tracking - console.log('\n📊 Testing Completeness Tracking:'); - if (canonicalMetadata.completeness) { - const comp = canonicalMetadata.completeness; - console.log(` Score: ${comp.score !== undefined ? '✅' : '❌'} ${comp.score}%`); - console.log(` Missing Fields: ${Array.isArray(comp.missingFields) ? '✅' : '❌'} ${JSON.stringify(comp.missingFields)}`); - console.log(` Last Calculated: ${comp.lastCalculated ? '✅' : '❌'} ${comp.lastCalculated}`); - } else { - console.log(' ❌ completeness field missing'); - } - - // Test 6: Verify tracking fields - console.log('\n📅 Testing Tracking Fields:'); - console.log(` lastCanonicalUpdate: ${canonicalMetadata.lastCanonicalUpdate ? '✅' : '❌'} ${canonicalMetadata.lastCanonicalUpdate}`); - console.log(` hasUserModifications: ${canonicalMetadata.hasUserModifications !== undefined ? '✅' : '❌'} ${canonicalMetadata.hasUserModifications}`); - - // Test 7: Verify creators structure (if present) - console.log('\n👥 Testing Creators Structure:'); - if (canonicalMetadata.creators && Array.isArray(canonicalMetadata.creators)) { - console.log(` Creators array: ✅ Found ${canonicalMetadata.creators.length} creators`); - - if (canonicalMetadata.creators.length > 0) { - const firstCreator = canonicalMetadata.creators[0]; - const hasCreatorFields = firstCreator.hasOwnProperty('name') && - firstCreator.hasOwnProperty('role') && - firstCreator.hasOwnProperty('source') && - firstCreator.hasOwnProperty('userSelected') && - firstCreator.hasOwnProperty('lastModified'); - - console.log(` Creator source attribution: ${hasCreatorFields ? '✅' : '❌'} ${JSON.stringify(firstCreator)}`); - } - } else { - console.log(' Creators array: ✅ Empty or not applicable'); - } - - // Test 8: Verify characters and genres structure - console.log('\n🎭 Testing Characters and Genres Structure:'); - ['characters', 'genres'].forEach(arrayField => { - const field = canonicalMetadata[arrayField]; - if (field && typeof field === 'object') { - const hasRequiredFields = field.hasOwnProperty('values') && - Array.isArray(field.values) && - field.hasOwnProperty('source') && - field.hasOwnProperty('userSelected') && - field.hasOwnProperty('lastModified'); - - console.log(` ${arrayField}: ${hasRequiredFields ? '✅' : '❌'} ${field.values.length} items from ${field.source}`); - } else { - console.log(` ${arrayField}: ❌ Missing or invalid structure`); - } - }); - - // Test 9: Test backward compatibility with sourcedMetadata - console.log('\n🔄 Testing Backward Compatibility:'); - console.log(` sourcedMetadata: ${comic.sourcedMetadata ? '✅' : '❌'} Still preserved`); - console.log(` inferredMetadata: ${comic.inferredMetadata ? '✅' : '❌'} Still preserved`); - - console.log('\n🎉 Canonical Metadata Test Complete!'); - console.log('📋 Summary:'); - console.log(' ✅ Canonical metadata structure implemented'); - console.log(' ✅ Source attribution working'); - console.log(' ✅ User selection tracking ready'); - console.log(' ✅ Completeness scoring functional'); - console.log(' ✅ Backward compatibility maintained'); - - console.log('\n🚀 Ready for User-Driven Curation UI Implementation!'); - - } catch (error) { - console.error('❌ Test failed:', error.message); - if (error.response) { - console.error('📋 Response data:', JSON.stringify(error.response.data, null, 2)); - } - console.error('🔍 Full error:', error); - } -} - -// Run the test -testCanonicalMetadata().then(() => { - console.log('\n✨ Test execution completed'); -}).catch(error => { - console.error('💥 Test execution failed:', error); -}); \ No newline at end of file diff --git a/test-directory-scan.js b/test-directory-scan.js deleted file mode 100644 index 37a4ff6..0000000 --- a/test-directory-scan.js +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Test directory scanning with enhanced metadata processing - */ - -const axios = require('axios'); -const fs = require('fs'); -const path = require('path'); - -const API_BASE = 'http://localhost:3000/api'; -const COMICS_DIRECTORY = process.env.COMICS_DIRECTORY || '/Users/rishi/work/threetwo-core-service/comics'; - -async function testDirectoryScan() { - console.log("🧪 Testing Directory Scan with Enhanced Metadata Processing"); - console.log(`📁 Comics directory: ${COMICS_DIRECTORY}`); - - try { - // Test 1: Check if comics directory exists and create test structure if needed - console.log("\n📝 Test 1: Checking comics directory structure"); - - if (!fs.existsSync(COMICS_DIRECTORY)) { - fs.mkdirSync(COMICS_DIRECTORY, { recursive: true }); - console.log("✅ Created comics directory"); - } - - // Create a test comic file if none exist (just for testing) - const testFiles = fs.readdirSync(COMICS_DIRECTORY).filter(file => - ['.cbz', '.cbr', '.cb7'].includes(path.extname(file)) - ); - - if (testFiles.length === 0) { - console.log("ℹ️ No comic files found in directory"); - console.log(" You can add .cbz, .cbr, or .cb7 files to test the scanning"); - } else { - console.log(`✅ Found ${testFiles.length} comic files:`, testFiles.slice(0, 3)); - } - - // Test 2: Check library service health - console.log("\n📝 Test 2: Checking library service health"); - const healthResponse = await axios.get(`${API_BASE}/library/getHealthInformation`); - console.log("✅ Library service is healthy"); - - // Test 3: Test directory scanning endpoint - console.log("\n📝 Test 3: Testing directory scan with enhanced metadata"); - - const sessionId = `test-session-${Date.now()}`; - const scanResponse = await axios.post(`${API_BASE}/library/newImport`, { - sessionId: sessionId, - extractionOptions: {} - }); - - console.log("✅ Directory scan initiated successfully"); - console.log("📊 Session ID:", sessionId); - - // Test 4: Check job queue status - console.log("\n📝 Test 4: Checking job queue statistics"); - - // Wait a moment for jobs to be enqueued - await new Promise(resolve => setTimeout(resolve, 2000)); - - try { - const jobStatsResponse = await axios.get(`${API_BASE}/jobqueue/getJobResultStatistics`); - console.log("✅ Job statistics retrieved:", jobStatsResponse.data.length, "sessions"); - } catch (error) { - console.log("ℹ️ Job statistics not available (may be empty)"); - } - - // Test 5: Check recent comics to see if any were imported - console.log("\n📝 Test 5: Checking for recently imported comics"); - - const recentComicsResponse = await axios.post(`${API_BASE}/library/getComicBooks`, { - paginationOptions: { - limit: 5, - sort: { createdAt: -1 } - }, - predicate: {} - }); - - const recentComics = recentComicsResponse.data.docs || []; - console.log(`✅ Found ${recentComics.length} recent comics`); - - if (recentComics.length > 0) { - const latestComic = recentComics[0]; - console.log("📋 Latest comic details:"); - console.log(" • File path:", latestComic.rawFileDetails?.filePath); - console.log(" • Sourced metadata sources:", Object.keys(latestComic.sourcedMetadata || {})); - console.log(" • Has resolved metadata:", !!latestComic.resolvedMetadata); - console.log(" • Primary source:", latestComic.resolvedMetadata?.primarySource); - - if (latestComic.resolvedMetadata) { - console.log(" • Resolved title:", latestComic.resolvedMetadata.title); - console.log(" • Resolved series:", latestComic.resolvedMetadata.series?.name); - } - } - - console.log("\n🎉 Directory scan integration test completed!"); - console.log("\n📊 Summary:"); - console.log("• Directory scanning endpoint works with enhanced metadata system"); - console.log("• Jobs are properly enqueued through enhanced job queue"); - console.log("• Multiple metadata sources are processed during import"); - console.log("• Enhanced Comic model stores resolved metadata from all sources"); - console.log("• System maintains backward compatibility while adding new capabilities"); - - if (testFiles.length === 0) { - console.log("\n💡 To see full import workflow:"); - console.log("1. Add some .cbz, .cbr, or .cb7 files to:", COMICS_DIRECTORY); - console.log("2. Run this test again to see enhanced metadata processing in action"); - } - - } catch (error) { - if (error.response) { - console.error("❌ API Error:", error.response.status, error.response.statusText); - if (error.response.data) { - console.error(" Details:", error.response.data); - } - } else { - console.error("❌ Test failed:", error.message); - } - } -} - -// Run the test -testDirectoryScan().catch(console.error); \ No newline at end of file diff --git a/test-real-canonical.js b/test-real-canonical.js deleted file mode 100644 index 5b77bf1..0000000 --- a/test-real-canonical.js +++ /dev/null @@ -1,59 +0,0 @@ -const mongoose = require('mongoose'); -const Comic = require('./models/comic.model.js'); - -async function testRealCanonicalMetadata() { - try { - await mongoose.connect('mongodb://localhost:27017/threetwo'); - console.log('🔍 Testing canonical metadata with real comics from database...\n'); - - // Find a recently imported comic - const comic = await Comic.findOne({}).sort({createdAt: -1}).limit(1); - - if (!comic) { - console.log('❌ No comics found in database'); - return; - } - - console.log('📚 Found comic:', comic.inferredMetadata?.name || 'Unknown'); - console.log('📅 Created:', comic.createdAt); - console.log(''); - - // Check if canonical metadata exists - if (comic.canonicalMetadata) { - console.log('✅ Canonical metadata structure exists!'); - console.log('📊 Completeness score:', comic.canonicalMetadata.completenessScore); - console.log('📝 Has user modifications:', comic.canonicalMetadata.hasUserModifications); - console.log(''); - - // Show some sample canonical fields - if (comic.canonicalMetadata.title) { - console.log('🏷️ Title:', comic.canonicalMetadata.title.value); - console.log(' Source:', comic.canonicalMetadata.title.source); - console.log(' User selected:', comic.canonicalMetadata.title.userSelected); - } - - if (comic.canonicalMetadata.publisher) { - console.log('🏢 Publisher:', comic.canonicalMetadata.publisher.value); - console.log(' Source:', comic.canonicalMetadata.publisher.source); - } - - if (comic.canonicalMetadata.series && comic.canonicalMetadata.series.name) { - console.log('📖 Series:', comic.canonicalMetadata.series.name.value); - console.log(' Source:', comic.canonicalMetadata.series.name.source); - } - - console.log(''); - console.log('🎯 Canonical metadata system is working with real comics!'); - } else { - console.log('❌ No canonical metadata found'); - console.log('📋 Available fields:', Object.keys(comic.toObject())); - } - - } catch (error) { - console.error('❌ Error:', error.message); - } finally { - await mongoose.disconnect(); - } -} - -testRealCanonicalMetadata(); \ No newline at end of file diff --git a/utils/comicinfo.xml b/utils/comicinfo.xml deleted file mode 100644 index c2ca4ff..0000000 --- a/utils/comicinfo.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - Title of the Book - A description of the book - 1 - 3 - 2010 - 4 - Author name - self - educational - No - No - Superman - 5 - - - - - - - -