📈 Added real time import stats and stats cache

services/api.service.ts

@@ -6,6 +6,18 @@ import ApiGateway from "moleculer-web";
 import debounce from "lodash/debounce";
 import { IFolderData } from "threetwo-ui-typings";
 
+/**
+ * Import statistics cache for real-time updates
+ */
+interface ImportStatisticsCache {
+  totalLocalFiles: number;
+  alreadyImported: number;
+  newFiles: number;
+  percentageImported: string;
+  lastUpdated: Date;
+  pendingFiles: Set<string>; // Files in stabilization period
+}
+
 /**
  * ApiService exposes REST endpoints and watches the comics directory for changes.
  * It uses chokidar to monitor filesystem events and broadcasts them via the Moleculer broker.
@@ -18,6 +30,18 @@ export default class ApiService extends Service {
    */
   private fileWatcher?: any;
 
+  /**
+   * Import statistics cache for real-time updates
+   * @private
+   */
+  private statsCache: ImportStatisticsCache | null = null;
+
+  /**
+   * Debounced function to broadcast statistics updates
+   * @private
+   */
+  private broadcastStatsUpdate?: ReturnType<typeof debounce>;
+
   /**
    * Creates an instance of ApiService.
    * @param {ServiceBroker} broker - The Moleculer service broker instance.
@@ -109,7 +133,192 @@ export default class ApiService extends Service {
         assets: { folder: "public", options: {} },
       },
       events: {},
-      methods: {},
+      actions: {
+        /**
+         * Get cached import statistics (fast, no filesystem scan)
+         * @returns Cached statistics or null if not initialized
+         */
+        getCachedImportStatistics: {
+          rest: "GET /cachedImportStatistics",
+          async handler() {
+            // If cache not initialized, try to initialize it now
+            if (!this.statsCache) {
+              this.logger.info("[Stats Cache] Cache not initialized, initializing now...");
+              try {
+                await this.initializeStatsCache();
+              } catch (error) {
+                this.logger.error("[Stats Cache] Failed to initialize:", error);
+                return {
+                  success: false,
+                  message: "Failed to initialize statistics cache",
+                  stats: null,
+                  lastUpdated: null,
+                };
+              }
+            }
+
+            // Check again after initialization attempt
+            if (!this.statsCache) {
+              return {
+                success: false,
+                message: "Statistics cache not initialized yet",
+                stats: null,
+                lastUpdated: null,
+              };
+            }
+
+            return {
+              success: true,
+              stats: {
+                totalLocalFiles: this.statsCache.totalLocalFiles,
+                alreadyImported: this.statsCache.alreadyImported,
+                newFiles: this.statsCache.newFiles,
+                percentageImported: this.statsCache.percentageImported,
+                pendingFiles: this.statsCache.pendingFiles.size,
+              },
+              lastUpdated: this.statsCache.lastUpdated.toISOString(),
+            };
+          },
+        },
+
+        /**
+         * Invalidate statistics cache (force refresh on next request)
+         */
+        invalidateStatsCache: {
+          async handler() {
+            this.logger.info("[Stats Cache] Invalidating cache...");
+            await this.initializeStatsCache();
+            return { success: true, message: "Cache invalidated and refreshed" };
+          },
+        },
+      },
+      methods: {
+        /**
+         * Initialize statistics cache by fetching current import statistics
+         * @private
+         */
+        initializeStatsCache: async function() {
+          try {
+            this.logger.info("[Stats Cache] Initializing import statistics cache...");
+            const stats = await this.broker.call("library.getImportStatistics", {});
+            
+            if (stats && stats.success) {
+              this.statsCache = {
+                totalLocalFiles: stats.stats.totalLocalFiles,
+                alreadyImported: stats.stats.alreadyImported,
+                newFiles: stats.stats.newFiles,
+                percentageImported: stats.stats.percentageImported,
+                lastUpdated: new Date(),
+                pendingFiles: new Set<string>(),
+              };
+              this.logger.info("[Stats Cache] Cache initialized successfully");
+            }
+          } catch (error) {
+            this.logger.error("[Stats Cache] Failed to initialize cache:", error);
+          }
+        },
+
+        /**
+         * Update statistics cache when files are added or removed
+         * @param event - File event type ('add' or 'unlink')
+         * @param filePath - Path to the file
+         * @private
+         */
+        updateStatsCache: function(event: string, filePath: string) {
+          if (!this.statsCache) return;
+
+          const fileExtension = path.extname(filePath);
+          const isComicFile = [".cbz", ".cbr", ".cb7"].includes(fileExtension);
+          
+          if (!isComicFile) return;
+
+          if (event === "add") {
+            // Add to pending files (in stabilization period)
+            this.statsCache.pendingFiles.add(filePath);
+            this.statsCache.totalLocalFiles++;
+            this.statsCache.newFiles++;
+          } else if (event === "unlink") {
+            // Remove from pending if it was there
+            this.statsCache.pendingFiles.delete(filePath);
+            this.statsCache.totalLocalFiles--;
+            // Could be either new or already imported, but we'll decrement newFiles for safety
+            if (this.statsCache.newFiles > 0) {
+              this.statsCache.newFiles--;
+            }
+          }
+
+          // Recalculate percentage
+          if (this.statsCache.totalLocalFiles > 0) {
+            const percentage = ((this.statsCache.alreadyImported / this.statsCache.totalLocalFiles) * 100).toFixed(2);
+            this.statsCache.percentageImported = `${percentage}%`;
+          } else {
+            this.statsCache.percentageImported = "0.00%";
+          }
+
+          this.statsCache.lastUpdated = new Date();
+
+          // Trigger debounced broadcast
+          if (this.broadcastStatsUpdate) {
+            this.broadcastStatsUpdate();
+          }
+        },
+
+        /**
+         * Broadcast statistics update via Socket.IO
+         * @private
+         */
+        broadcastStats: async function() {
+          if (!this.statsCache) return;
+
+          try {
+            await this.broker.call("socket.broadcast", {
+              namespace: "/",
+              event: "IMPORT_STATISTICS_UPDATED",
+              args: [{
+                stats: {
+                  totalLocalFiles: this.statsCache.totalLocalFiles,
+                  alreadyImported: this.statsCache.alreadyImported,
+                  newFiles: this.statsCache.newFiles,
+                  percentageImported: this.statsCache.percentageImported,
+                  pendingFiles: this.statsCache.pendingFiles.size,
+                },
+                lastUpdated: this.statsCache.lastUpdated.toISOString(),
+              }],
+            });
+            this.logger.debug("[Stats Cache] Broadcasted statistics update");
+          } catch (error) {
+            this.logger.error("[Stats Cache] Failed to broadcast statistics:", error);
+          }
+        },
+
+        /**
+         * Mark a file as imported (moved from pending to imported)
+         * @param filePath - Path to the imported file
+         * @private
+         */
+        markFileAsImported: function(filePath: string) {
+          if (!this.statsCache) return;
+
+          this.statsCache.pendingFiles.delete(filePath);
+          this.statsCache.alreadyImported++;
+          if (this.statsCache.newFiles > 0) {
+            this.statsCache.newFiles--;
+          }
+
+          // Recalculate percentage
+          if (this.statsCache.totalLocalFiles > 0) {
+            const percentage = ((this.statsCache.alreadyImported / this.statsCache.totalLocalFiles) * 100).toFixed(2);
+            this.statsCache.percentageImported = `${percentage}%`;
+          }
+
+          this.statsCache.lastUpdated = new Date();
+
+          // Trigger debounced broadcast
+          if (this.broadcastStatsUpdate) {
+            this.broadcastStatsUpdate();
+          }
+        },
+      },
       started: this.startWatcher,
       stopped: this.stopWatcher,
     });
@@ -120,7 +329,7 @@ export default class ApiService extends Service {
    * Debounces rapid events and logs initial scan completion.
    * @private
    */
-  private startWatcher(): void {
+  private async startWatcher(): Promise<void> {
     const rawDir = process.env.COMICS_DIRECTORY;
     if (!rawDir) {
       this.logger.error("COMICS_DIRECTORY not set; cannot start watcher");
@@ -133,6 +342,20 @@ export default class ApiService extends Service {
       return;
     }
 
+    // Initialize debounced broadcast function (2 second debounce for statistics updates)
+    this.broadcastStatsUpdate = debounce(
+      () => {
+        this.broadcastStats();
+      },
+      2000,
+      { leading: false, trailing: true }
+    );
+
+    // Initialize statistics cache (async, but don't block watcher startup)
+    this.initializeStatsCache().catch(err => {
+      this.logger.error("[Stats Cache] Failed to initialize on startup:", err);
+    });
+
     this.fileWatcher = chokidar.watch(watchDir, {
       persistent: true,
       ignoreInitial: true,
@@ -202,13 +425,19 @@ export default class ApiService extends Service {
     stats?: fs.Stats
   ): Promise<void> {
     this.logger.info(`File event [${event}]: ${filePath}`);
+    
+    // Update statistics cache for add/unlink events
+    if (event === "add" || event === "unlink") {
+      this.updateStatsCache(event, filePath);
+    }
+    
     if (event === "add" && stats) {
       setTimeout(async () => {
-        const newStats = await fs.promises.stat(filePath);
-        if (newStats.mtime.getTime() === stats.mtime.getTime()) {
-          this.logger.info(`Stable file detected: ${filePath}, importing.`);
-          
-          try {
+        try {
+          const newStats = await fs.promises.stat(filePath);
+          if (newStats.mtime.getTime() === stats.mtime.getTime()) {
+            this.logger.info(`Stable file detected: ${filePath}, importing.`);
+            
             const folderData: IFolderData[] = await this.broker.call(
               "library.walkFolders",
               { basePathToWalk: filePath }
@@ -266,13 +495,18 @@ export default class ApiService extends Service {
               });
               
               this.logger.info(`Successfully queued import for: ${filePath}`);
+              
+              // Mark file as imported in statistics cache
+              this.markFileAsImported(filePath);
             }
-          } catch (error) {
-            this.logger.error(`Error importing file ${filePath}:`, error);
           }
+        } catch (error) {
+          this.logger.error(`Error importing file ${filePath}:`, error);
         }
       }, 3000);
     }
+    
+    // Broadcast file system event
     this.broker.broadcast(event, { path: filePath });
   }
 }