frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

🔨 Fixes for import statuses

Browse Source
This commit is contained in:
Rishi Ghan
2026-03-05 21:29:38 -05:00
parent 8138e0fe4f
commit c7d3d46bcf
8 changed files with 277 additions and 372 deletions
Download Patch File Download Diff File Expand all files Collapse all files

289
services/api.service.ts
View File

@@ -6,18 +6,6 @@ import ApiGateway from "moleculer-web";
import debounce from "lodash/debounce";
import { IFolderData } from "threetwo-ui-typings";
/**
* Import statistics cache for real-time updates
*/
interface ImportStatisticsCache {
totalLocalFiles: number;
alreadyImported: number;
newFiles: number;
percentageImported: string;
lastUpdated: Date;
pendingFiles: Set<string>; // Files in stabilization period
}
/**
* ApiService exposes REST endpoints and watches the comics directory for changes.
* It uses chokidar to monitor filesystem events and broadcasts them via the Moleculer broker.
@@ -30,18 +18,6 @@ export default class ApiService extends Service {
*/
private fileWatcher?: any;
/**
* Import statistics cache for real-time updates
* @private
*/
private statsCache: ImportStatisticsCache | null = null;
/**
* Debounced function to broadcast statistics updates
* @private
*/
private broadcastStatsUpdate?: ReturnType<typeof debounce>;
/**
* Creates an instance of ApiService.
* @param {ServiceBroker} broker - The Moleculer service broker instance.
@@ -134,61 +110,6 @@ export default class ApiService extends Service {
assets: { folder: "public", options: {} },
},
events: {
/**
* Listen for import session completion to refresh statistics
*/
"IMPORT_SESSION_COMPLETED": {
async handler(ctx: Context<{
sessionId: string;
type: string;
success: boolean;
stats: any;
}>) {
const { sessionId, type, success } = ctx.params;
this.logger.info(
`[Stats Cache] Import session completed: ${sessionId} (${type}, success: ${success})`
);
// Invalidate and refresh statistics cache
await this.actions.invalidateStatsCache();
},
},
/**
* Listen for import progress to update cache incrementally
*/
"IMPORT_PROGRESS": {
async handler(ctx: Context<{
sessionId: string;
stats: any;
}>) {
// Update cache with current progress
if (this.statsCache) {
const { stats } = ctx.params;
// Update alreadyImported count based on files succeeded
if (stats.filesSucceeded) {
this.statsCache.alreadyImported += 1;
this.statsCache.newFiles = Math.max(0, this.statsCache.newFiles - 1);
// Recalculate percentage
if (this.statsCache.totalLocalFiles > 0) {
const percentage = (
(this.statsCache.alreadyImported / this.statsCache.totalLocalFiles) * 100
).toFixed(2);
this.statsCache.percentageImported = `${percentage}%`;
}
this.statsCache.lastUpdated = new Date();
// Trigger debounced broadcast
if (this.broadcastStatsUpdate) {
this.broadcastStatsUpdate();
}
}
}
},
},
/**
* Listen for watcher disable events
*/
@@ -230,192 +151,8 @@ export default class ApiService extends Service {
},
},
},
actions: {
/**
* Get cached import statistics (fast, no filesystem scan)
* @returns Cached statistics or null if not initialized
*/
getCachedImportStatistics: {
rest: "GET /cachedImportStatistics",
async handler() {
// If cache not initialized, try to initialize it now
if (!this.statsCache) {
this.logger.info("[Stats Cache] Cache not initialized, initializing now...");
try {
await this.initializeStatsCache();
} catch (error) {
this.logger.error("[Stats Cache] Failed to initialize:", error);
return {
success: false,
message: "Failed to initialize statistics cache",
stats: null,
lastUpdated: null,
};
}
}
// Check again after initialization attempt
if (!this.statsCache) {
return {
success: false,
message: "Statistics cache not initialized yet",
stats: null,
lastUpdated: null,
};
}
return {
success: true,
stats: {
totalLocalFiles: this.statsCache.totalLocalFiles,
alreadyImported: this.statsCache.alreadyImported,
newFiles: this.statsCache.newFiles,
percentageImported: this.statsCache.percentageImported,
pendingFiles: this.statsCache.pendingFiles.size,
},
lastUpdated: this.statsCache.lastUpdated.toISOString(),
};
},
},
/**
* Invalidate statistics cache (force refresh on next request)
*/
invalidateStatsCache: {
async handler() {
this.logger.info("[Stats Cache] Invalidating cache...");
await this.initializeStatsCache();
return { success: true, message: "Cache invalidated and refreshed" };
},
},
},
methods: {
/**
* Initialize statistics cache by fetching current import statistics
* @private
*/
initializeStatsCache: async function() {
try {
this.logger.info("[Stats Cache] Initializing import statistics cache...");
const stats = await this.broker.call("library.getImportStatistics", {});
if (stats && stats.success) {
this.statsCache = {
totalLocalFiles: stats.stats.totalLocalFiles,
alreadyImported: stats.stats.alreadyImported,
newFiles: stats.stats.newFiles,
percentageImported: stats.stats.percentageImported,
lastUpdated: new Date(),
pendingFiles: new Set<string>(),
};
this.logger.info("[Stats Cache] Cache initialized successfully");
}
} catch (error) {
this.logger.error("[Stats Cache] Failed to initialize cache:", error);
}
},
/**
* Update statistics cache when files are added or removed
* @param event - File event type ('add' or 'unlink')
* @param filePath - Path to the file
* @private
*/
updateStatsCache: function(event: string, filePath: string) {
if (!this.statsCache) return;
const fileExtension = path.extname(filePath);
const isComicFile = [".cbz", ".cbr", ".cb7"].includes(fileExtension);
if (!isComicFile) return;
if (event === "add") {
// Add to pending files (in stabilization period)
this.statsCache.pendingFiles.add(filePath);
this.statsCache.totalLocalFiles++;
this.statsCache.newFiles++;
} else if (event === "unlink") {
// Remove from pending if it was there
this.statsCache.pendingFiles.delete(filePath);
this.statsCache.totalLocalFiles--;
// Could be either new or already imported, but we'll decrement newFiles for safety
if (this.statsCache.newFiles > 0) {
this.statsCache.newFiles--;
}
}
// Recalculate percentage
if (this.statsCache.totalLocalFiles > 0) {
const percentage = ((this.statsCache.alreadyImported / this.statsCache.totalLocalFiles) * 100).toFixed(2);
this.statsCache.percentageImported = `${percentage}%`;
} else {
this.statsCache.percentageImported = "0.00%";
}
this.statsCache.lastUpdated = new Date();
// Trigger debounced broadcast
if (this.broadcastStatsUpdate) {
this.broadcastStatsUpdate();
}
},
/**
* Broadcast statistics update via Socket.IO
* @private
*/
broadcastStats: async function() {
if (!this.statsCache) return;
try {
await this.broker.call("socket.broadcast", {
namespace: "/",
event: "IMPORT_STATISTICS_UPDATED",
args: [{
stats: {
totalLocalFiles: this.statsCache.totalLocalFiles,
alreadyImported: this.statsCache.alreadyImported,
newFiles: this.statsCache.newFiles,
percentageImported: this.statsCache.percentageImported,
pendingFiles: this.statsCache.pendingFiles.size,
},
lastUpdated: this.statsCache.lastUpdated.toISOString(),
}],
});
this.logger.debug("[Stats Cache] Broadcasted statistics update");
} catch (error) {
this.logger.error("[Stats Cache] Failed to broadcast statistics:", error);
}
},
/**
* Mark a file as imported (moved from pending to imported)
* @param filePath - Path to the imported file
* @private
*/
markFileAsImported: function(filePath: string) {
if (!this.statsCache) return;
this.statsCache.pendingFiles.delete(filePath);
this.statsCache.alreadyImported++;
if (this.statsCache.newFiles > 0) {
this.statsCache.newFiles--;
}
// Recalculate percentage
if (this.statsCache.totalLocalFiles > 0) {
const percentage = ((this.statsCache.alreadyImported / this.statsCache.totalLocalFiles) * 100).toFixed(2);
this.statsCache.percentageImported = `${percentage}%`;
}
this.statsCache.lastUpdated = new Date();
// Trigger debounced broadcast
if (this.broadcastStatsUpdate) {
this.broadcastStatsUpdate();
}
},
},
actions: {},
methods: {},
started: this.startWatcher,
stopped: this.stopWatcher,
});
@@ -439,20 +176,6 @@ export default class ApiService extends Service {
return;
}
// Initialize debounced broadcast function (2 second debounce for statistics updates)
this.broadcastStatsUpdate = debounce(
() => {
this.broadcastStats();
},
2000,
{ leading: false, trailing: true }
);
// Initialize statistics cache (async, but don't block watcher startup)
this.initializeStatsCache().catch(err => {
this.logger.error("[Stats Cache] Failed to initialize on startup:", err);
});
this.fileWatcher = chokidar.watch(watchDir, {
persistent: true,
ignoreInitial: true,
@@ -534,11 +257,6 @@ export default class ApiService extends Service {
}
}
// Update statistics cache for add/unlink events
if (event === "add" || event === "unlink") {
this.updateStatsCache(event, filePath);
}
if (event === "add" && stats) {
setTimeout(async () => {
try {
@@ -620,9 +338,6 @@ export default class ApiService extends Service {
this.logger.info(`Successfully imported: ${filePath}`);
// Mark file as imported in statistics cache
this.markFileAsImported(filePath);
// Complete watcher session
await this.broker.call("importstate.completeSession", {
sessionId,

31
services/importstate.service.ts
View File

@@ -237,7 +237,26 @@ export default class ImportStateService extends Service {
*/
getActiveSession: {
async handler() {
return this.getActiveSession();
const session = this.getActiveSession();
if (session) {
// Format session for GraphQL response
return {
sessionId: session.sessionId,
type: session.type,
status: session.status,
startedAt: session.startedAt.toISOString(),
completedAt: session.completedAt?.toISOString() || null,
stats: {
totalFiles: session.stats.totalFiles,
filesQueued: session.stats.filesQueued,
filesProcessed: session.stats.filesProcessed,
filesSucceeded: session.stats.filesSucceeded,
filesFailed: session.stats.filesFailed,
},
directoryPath: session.directoryPath || null,
};
}
return null;
},
},
@@ -339,9 +358,13 @@ export default class ImportStateService extends Service {
started: async () => {
this.logger.info("[Import State] Service started");
// Clean up any stale sessions from Redis on startup
const keys = await pubClient.keys("import:session:*");
this.logger.info(`[Import State] Found ${keys.length} session keys in Redis`);
// Auto-complete stuck sessions every 5 minutes
setInterval(() => {
for (const [id, session] of this.activeSessions.entries()) {
const age = Date.now() - session.startedAt.getTime();
if (age > 30 * 60 * 1000 && session.stats.filesProcessed === 0) this.actions.completeSession({ sessionId: id, success: false });
}
}, 5 * 60 * 1000);
},
});
}

14
services/jobqueue.service.ts
View File

@@ -379,6 +379,13 @@ export default class JobQueueService extends Service {
},
],
});
// Emit final library statistics when queue is drained
try {
await this.broker.call("socket.broadcastLibraryStatistics", {});
} catch (err) {
console.error("Failed to emit library statistics after queue drained:", err);
}
},
async "enqueue.async.completed"(ctx: Context<{ id: Number }>) {
// 1. Fetch the job result using the job Id
@@ -410,6 +417,13 @@ export default class JobQueueService extends Service {
});
console.log(`Job ID ${ctx.params.id} completed.`);
// 6. Emit updated library statistics after each import
try {
await this.broker.call("socket.broadcastLibraryStatistics", {});
} catch (err) {
console.error("Failed to emit library statistics after import:", err);
}
},
async "enqueue.async.failed"(ctx) {

32
services/library.service.ts
View File

@@ -253,6 +253,15 @@ export default class ImportService extends Service {
sessionId,
status: "active",
});
// Emit library statistics after scanning
try {
await this.broker.call("socket.broadcastLibraryStatistics", {
directoryPath: resolvedPath,
});
} catch (err) {
console.error("Failed to emit library statistics:", err);
}
});
} catch (error) {
console.log(error);
@@ -466,12 +475,30 @@ export default class ImportService extends Service {
sessionId,
status: "active",
});
// Emit library statistics after queueing
try {
await this.broker.call("socket.broadcastLibraryStatistics", {
directoryPath: resolvedPath,
});
} catch (err) {
console.error("Failed to emit library statistics:", err);
}
} else {
// No files to import, complete immediately
await this.broker.call("importstate.completeSession", {
sessionId,
success: true,
});
// Emit library statistics even when no new files
try {
await this.broker.call("socket.broadcastLibraryStatistics", {
directoryPath: resolvedPath,
});
} catch (err) {
console.error("Failed to emit library statistics:", err);
}
}
// Emit completion event (queueing complete, not import complete)
@@ -1210,6 +1237,11 @@ export default class ImportService extends Service {
"search.deleteElasticSearchIndices",
{}
);
// Invalidate statistics cache after flushing database
console.info("Invalidating statistics cache after flushDB...");
await ctx.broker.call("api.invalidateStatsCache");
return {
data,
coversFolderDeleteResult,

100
services/socket.service.ts
View File

@@ -326,6 +326,106 @@ export default class SocketService extends Service {
},
},
},
events: {
// File watcher events - forward to Socket.IO clients
async "add"(ctx: Context<{ path: string }>) {
console.log(`[File Watcher] File added: ${ctx.params.path}`);
await this.broker.call("socket.broadcast", {
namespace: "/",
event: "LS_FILE_ADDED",
args: [
{
path: ctx.params.path,
timestamp: new Date().toISOString(),
},
],
});
// Emit updated statistics after file addition
try {
await this.broker.call("socket.broadcastLibraryStatistics", {});
} catch (err) {
console.error("Failed to emit library statistics after file add:", err);
}
},
async "unlink"(ctx: Context<{ path: string }>) {
console.log(`[File Watcher] File removed: ${ctx.params.path}`);
await this.broker.call("socket.broadcast", {
namespace: "/",
event: "LS_FILE_REMOVED",
args: [
{
path: ctx.params.path,
timestamp: new Date().toISOString(),
},
],
});
// Emit updated statistics after file removal
try {
await this.broker.call("socket.broadcastLibraryStatistics", {});
} catch (err) {
console.error("Failed to emit library statistics after file remove:", err);
}
},
async "addDir"(ctx: Context<{ path: string }>) {
console.log(`[File Watcher] Directory added: ${ctx.params.path}`);
await this.broker.call("socket.broadcast", {
namespace: "/",
event: "LS_DIRECTORY_ADDED",
args: [
{
path: ctx.params.path,
timestamp: new Date().toISOString(),
},
],
});
// Emit updated statistics after directory addition
try {
await this.broker.call("socket.broadcastLibraryStatistics", {});
} catch (err) {
console.error("Failed to emit library statistics after directory add:", err);
}
},
async "unlinkDir"(ctx: Context<{ path: string }>) {
console.log(`[File Watcher] Directory removed: ${ctx.params.path}`);
await this.broker.call("socket.broadcast", {
namespace: "/",
event: "LS_DIRECTORY_REMOVED",
args: [
{
path: ctx.params.path,
timestamp: new Date().toISOString(),
},
],
});
// Emit updated statistics after directory removal
try {
await this.broker.call("socket.broadcastLibraryStatistics", {});
} catch (err) {
console.error("Failed to emit library statistics after directory remove:", err);
}
},
async "change"(ctx: Context<{ path: string }>) {
console.log(`[File Watcher] File changed: ${ctx.params.path}`);
await this.broker.call("socket.broadcast", {
namespace: "/",
event: "LS_FILE_CHANGED",
args: [
{
path: ctx.params.path,
timestamp: new Date().toISOString(),
},
],
});
},
},
methods: {
sleep: (ms: number): Promise<NodeJS.Timeout> => {
return new Promise((resolve) => setTimeout(resolve, ms));