frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

🔧 Refactoring core-service to autowatch and import comics

Browse Source
This commit is contained in:
Rishi Ghan
2022-01-26 00:40:30 -08:00
parent 44940b51aa
commit 019493a6f3
7 changed files with 108 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
mixins/db.mixin.ts
View File

@@ -3,7 +3,7 @@ const mkdir = require("mkdirp").sync;
const DbService = require("moleculer-db"); const DbService = require("moleculer-db");
const MongoAdapter = require("moleculer-db-adapter-mongoose"); const MongoAdapter = require("moleculer-db-adapter-mongoose");
export const DbMixin = (collection, model) => { export const DbMixin = (collection, model) => {
if(process.env.MONGO_URI) { if(process.env.MONGO_URI) {
return { return {
mixins: [DbService], mixins: [DbService],
@@ -12,6 +12,7 @@ export const DbMixin = (collection, model) => {
pass: process.env.MONGO_INITDB_ROOT_PASSWORD, pass: process.env.MONGO_INITDB_ROOT_PASSWORD,
keepAlive: true, keepAlive: true,
useUnifiedTopology: true, useUnifiedTopology: true,
family: 4,
}), }),
model, model,
collection, collection,

2
package.json
View File

@@ -43,7 +43,7 @@
"@types/string-similarity": "^4.0.0", "@types/string-similarity": "^4.0.0",
"7zip-bin": "^5.1.1", "7zip-bin": "^5.1.1",
"7zip-min": "^1.4.0", "7zip-min": "^1.4.0",
"chokidar": "^3.5.2", "chokidar": "^3.5.3",
"dotenv": "^10.0.0", "dotenv": "^10.0.0",
"fs-extra": "^10.0.0", "fs-extra": "^10.0.0",
"imghash": "^0.0.9", "imghash": "^0.0.9",

171
services/api.service.ts
View File

@@ -90,7 +90,6 @@ export default class ApiService extends Service {
methods: {}, methods: {},
started(): any { started(): any {
// Add a connect listener // Add a connect listener
io.on("connection", (client) => { io.on("connection", (client) => {
console.log("Client connected via websocket!"); console.log("Client connected via websocket!");
@@ -112,94 +111,102 @@ export default class ApiService extends Service {
client.on("disconnect", () => { client.on("disconnect", () => {
console.log("Client disconnected"); console.log("Client disconnected");
}); });
});
// Filewatcher // Filewatcher
const fileWatcher = chokidar.watch(path.resolve("./comics"), { const fileWatcher = chokidar.watch(
ignored: /(^|[\/\\])\../, // ignore dotfiles path.resolve("./comics"),
persistent: true, {
ignoreInitial: true, ignored: /(^|[\/\\])\../, // ignore dotfiles
atomic: true, persistent: true,
awaitWriteFinish: { usePolling: true,
stabilityThreshold: 2000, ignoreInitial: true,
pollInterval: 100, atomic: true,
}, depth: 1,
}); awaitWriteFinish: {
const fileCopyDelaySeconds = 10; stabilityThreshold: 2000,
const checkFileCopyComplete = (path, previousPath) => { pollInterval: 100,
fs.stat(path, async (err, stat) => { },
if (err) {
throw err;
} }
if ( );
stat.mtime.getTime() === const fileCopyDelaySeconds = 10;
previousPath.mtime.getTime() const checkFileCopyComplete = (path, previousPath) => {
) { fs.stat(path, async (err, stat) => {
console.log("File detected, starting import...");
// this walking business needs to go, SACURATAYYY, SACURATAYYY!! This dude needs to go.
const walkedFolders: IFolderData =
await broker.call("import.walkFolders", {
basePathToWalk: path,
});
const extractionOptions: IExtractionOptions = {
extractTarget: "cover",
targetExtractionFolder: "./userdata/covers",
extractionMode: "single",
};
await this.broker.call(
"import.processAndImportToDB",
{ walkedFolders, extractionOptions }
);
} else {
setTimeout(
checkFileCopyComplete,
fileCopyDelaySeconds * 1000,
path,
stat
);
}
});
};
fileWatcher
.on("add", async (path, stats) => {
console.log("Watcher detected new files.");
console.log(
`File ${path} has been added with stats: ${JSON.stringify(
stats
)}`
);
console.log("File copy started...");
fs.stat(path, function (err, stat) {
if (err) { if (err) {
console.log(
"Error watching file for copy completion. ERR: " +
err.message
);
console.log(
"Error file not processed. PATH: " + path
);
throw err; throw err;
} }
setTimeout( if (
checkFileCopyComplete, stat.mtime.getTime() ===
fileCopyDelaySeconds * 1000, previousPath.mtime.getTime()
path, ) {
stat console.log(
); "File detected, starting import..."
);
// this walking business needs to go, SACURATAYYY, SACURATAYYY!! This dude needs to go.
const walkedFolder: IFolderData =
await broker.call("import.walkFolders", {
basePathToWalk: path,
});
console.log(walkedFolder);
await this.broker.call(
"import.processAndImportToDB",
{ walkedFolder }
);
} else {
setTimeout(
checkFileCopyComplete,
fileCopyDelaySeconds * 1000,
path,
stat
);
}
}); });
}) };
.on("change", (path, stats) => fileWatcher
console.log( .on("add", async (path, stats) => {
`File ${path} has been changed. Stats: ${stats}` console.log("Watcher detected new files.");
console.log(
`File ${path} has been added with stats: ${JSON.stringify(
stats
)}`
);
console.log("File copy started...");
fs.stat(path, function (err, stat) {
if (err) {
console.log(
"Error watching file for copy completion. ERR: " +
err.message
);
console.log(
"Error file not processed. PATH: " +
path
);
throw err;
}
setTimeout(
checkFileCopyComplete,
fileCopyDelaySeconds * 1000,
path,
stat
);
client.emit("action", {
type: "LS_COMIC_ADDED",
result: path,
});
});
})
.on("change", (path, stats) =>
console.log(
`File ${path} has been changed. Stats: ${stats}`
)
) )
) .on("unlink", (path) =>
.on("unlink", (path) => console.log(`File ${path} has been removed`)
console.log(`File ${path} has been removed`) )
) .on("addDir", (path) =>
.on("addDir", (path) => console.log(`Directory ${path} has been added`)
console.log(`Directory ${path} has been added`) );
); });
}, },
}); });
} }

17
services/import.service.ts
View File

@@ -27,6 +27,7 @@ import klaw from "klaw";
import path from "path"; import path from "path";
import { COMICS_DIRECTORY, USERDATA_DIRECTORY } from "../constants/directories"; import { COMICS_DIRECTORY, USERDATA_DIRECTORY } from "../constants/directories";
console.log(process.env.MONGO_URI);
export default class ImportService extends Service { export default class ImportService extends Service {
public constructor(public broker: ServiceBroker) { public constructor(public broker: ServiceBroker) {
super(broker); super(broker);
@@ -128,8 +129,7 @@ export default class ImportService extends Service {
params: {}, params: {},
async handler( async handler(
ctx: Context<{ ctx: Context<{
extractionOptions: any; walkedFolder: {
walkedFolders: {
name: string; name: string;
path: string; path: string;
extension: string; extension: string;
@@ -141,10 +141,9 @@ export default class ImportService extends Service {
}> }>
) { ) {
try { try {
const { extractionOptions, walkedFolders } = const { walkedFolder } = ctx.params;
ctx.params;
let comicExists = await Comic.exists({ let comicExists = await Comic.exists({
"rawFileDetails.name": `${walkedFolders.name}`, "rawFileDetails.name": `${walkedFolder.name}`,
}); });
// rough flow of import process // rough flow of import process
// 1. Walk folder // 1. Walk folder
@@ -157,7 +156,7 @@ export default class ImportService extends Service {
| IExtractedComicBookCoverFile | IExtractedComicBookCoverFile
| IExtractComicBookCoverErrorResponse | IExtractComicBookCoverErrorResponse
| IExtractedComicBookCoverFile[] = await extractCoverFromFile2( | IExtractedComicBookCoverFile[] = await extractCoverFromFile2(
extractionOptions walkedFolder[0]
); );
// 2. Add to mongo // 2. Add to mongo
@@ -185,7 +184,7 @@ export default class ImportService extends Service {
}; };
} else { } else {
console.info( console.info(
`Comic: \"${walkedFolders.name}\" already exists in the database` `Comic: \"${walkedFolder.name}\" already exists in the database`
); );
} }
} catch (error) { } catch (error) {
@@ -459,7 +458,9 @@ export default class ImportService extends Service {
}); });
resp.on("end", () => { resp.on("end", () => {
console.log(`${apiDetailURL} returned data.`) console.log(
`${apiDetailURL} returned data.`
);
const volumeInformation = const volumeInformation =
JSON.parse(data); JSON.parse(data);
resolve(volumeInformation.results); resolve(volumeInformation.results);

1
services/libraryqueue.service.ts
View File

@@ -14,6 +14,7 @@ import { extractCoverFromFile2 } from "../utils/uncompression.utils";
import { io } from "./api.service"; import { io } from "./api.service";
const REDIS_URI = process.env.REDIS_URI || `redis://0.0.0.0:6379`; const REDIS_URI = process.env.REDIS_URI || `redis://0.0.0.0:6379`;
console.log(`REDIS -> ${REDIS_URI}`);
export default class LibraryQueueService extends Service { export default class LibraryQueueService extends Service {
public constructor(public broker: ServiceBroker) { public constructor(public broker: ServiceBroker) {
super(broker); super(broker);

4
utils/file.utils.ts
View File

@@ -17,7 +17,7 @@ export const walkFolder = async (folder: string, formats: string[]): Promise<IFo
const result: IFolderData[] = []; const result: IFolderData[] = [];
let walkResult: IFolderData = { let walkResult: IFolderData = {
name: "", name: "",
path: "", filePath: "",
extension: "", extension: "",
containedIn: "", containedIn: "",
isFile: false, isFile: false,
@@ -34,7 +34,7 @@ export const walkFolder = async (folder: string, formats: string[]): Promise<IFo
if ([...formats].includes(path.extname(dirent.name))) { if ([...formats].includes(path.extname(dirent.name))) {
walkResult = { walkResult = {
name: path.basename(dirent.name, path.extname(dirent.name)), name: path.basename(dirent.name, path.extname(dirent.name)),
path: path.dirname(pathname), filePath: path.resolve(pathname),
extension: path.extname(dirent.name), extension: path.extname(dirent.name),
fileSize: fs.statSync(path.resolve(pathname)).size, fileSize: fs.statSync(path.resolve(pathname)).size,
containedIn: path.dirname(pathname), containedIn: path.dirname(pathname),

6
utils/uncompression.utils.ts
View File

@@ -55,7 +55,9 @@ export const extractCoverFromFile2 = async (
fileObject: any fileObject: any
): Promise<any> => { ): Promise<any> => {
try { try {
const { filePath, size} = fileObject; console.log("ASDASD!@#!#!@#!@#");
console.log(fileObject);
const { filePath, fileSize} = fileObject;
const calibre = new Calibre(); const calibre = new Calibre();
console.info(`Initiating extraction process for path ${filePath}`); console.info(`Initiating extraction process for path ${filePath}`);
@@ -111,7 +113,7 @@ export const extractCoverFromFile2 = async (
return { return {
name: fileNameWithoutExtension, name: fileNameWithoutExtension,
path: filePath, path: filePath,
fileSize: size, fileSize,
extension: path.extname(filePath), extension: path.extname(filePath),
cover: { cover: {
filePath: path.relative(process.cwd(),renditionPath), filePath: path.relative(process.cwd(),renditionPath),