frishi/threetwo-core-service
1
0
Fork 0
Code Issues 2 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

🔧 Refactoring core-service to autowatch and import comics

Browse Source
This commit is contained in:
Rishi Ghan
2022-01-26 00:40:30 -08:00
parent 44940b51aa
commit 019493a6f3
7 changed files with 108 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
mixins/db.mixin.ts
View File

@@ -3,7 +3,7 @@ const mkdir = require("mkdirp").sync;
const DbService = require("moleculer-db");
const MongoAdapter = require("moleculer-db-adapter-mongoose");
export const DbMixin = (collection, model) => {
export const DbMixin = (collection, model) => {
if(process.env.MONGO_URI) {
return {
mixins: [DbService],
@@ -12,6 +12,7 @@ export const DbMixin = (collection, model) => {
pass: process.env.MONGO_INITDB_ROOT_PASSWORD,
keepAlive: true,
useUnifiedTopology: true,
family: 4,
}),
model,
collection,

2
package.json
View File

@@ -43,7 +43,7 @@
"@types/string-similarity": "^4.0.0",
"7zip-bin": "^5.1.1",
"7zip-min": "^1.4.0",
"chokidar": "^3.5.2",
"chokidar": "^3.5.3",
"dotenv": "^10.0.0",
"fs-extra": "^10.0.0",
"imghash": "^0.0.9",

171
services/api.service.ts
View File

@@ -90,7 +90,6 @@ export default class ApiService extends Service {
methods: {},
started(): any {
// Add a connect listener
io.on("connection", (client) => {
console.log("Client connected via websocket!");
@@ -112,94 +111,102 @@ export default class ApiService extends Service {
client.on("disconnect", () => {
console.log("Client disconnected");
});
});
// Filewatcher
const fileWatcher = chokidar.watch(path.resolve("./comics"), {
ignored: /(^|[\/\\])\../, // ignore dotfiles
persistent: true,
ignoreInitial: true,
atomic: true,
awaitWriteFinish: {
stabilityThreshold: 2000,
pollInterval: 100,
},
});
const fileCopyDelaySeconds = 10;
const checkFileCopyComplete = (path, previousPath) => {
fs.stat(path, async (err, stat) => {
if (err) {
throw err;
// Filewatcher
const fileWatcher = chokidar.watch(
path.resolve("./comics"),
{
ignored: /(^|[\/\\])\../, // ignore dotfiles
persistent: true,
usePolling: true,
ignoreInitial: true,
atomic: true,
depth: 1,
awaitWriteFinish: {
stabilityThreshold: 2000,
pollInterval: 100,
},
}
if (
stat.mtime.getTime() ===
previousPath.mtime.getTime()
) {
console.log("File detected, starting import...");
// this walking business needs to go, SACURATAYYY, SACURATAYYY!! This dude needs to go.
const walkedFolders: IFolderData =
await broker.call("import.walkFolders", {
basePathToWalk: path,
});
const extractionOptions: IExtractionOptions = {
extractTarget: "cover",
targetExtractionFolder: "./userdata/covers",
extractionMode: "single",
};
await this.broker.call(
"import.processAndImportToDB",
{ walkedFolders, extractionOptions }
);
} else {
setTimeout(
checkFileCopyComplete,
fileCopyDelaySeconds * 1000,
path,
stat
);
}
});
};
fileWatcher
.on("add", async (path, stats) => {
console.log("Watcher detected new files.");
console.log(
`File ${path} has been added with stats: ${JSON.stringify(
stats
)}`
);
console.log("File copy started...");
fs.stat(path, function (err, stat) {
);
const fileCopyDelaySeconds = 10;
const checkFileCopyComplete = (path, previousPath) => {
fs.stat(path, async (err, stat) => {
if (err) {
console.log(
"Error watching file for copy completion. ERR: " +
err.message
);
console.log(
"Error file not processed. PATH: " + path
);
throw err;
}
setTimeout(
checkFileCopyComplete,
fileCopyDelaySeconds * 1000,
path,
stat
);
if (
stat.mtime.getTime() ===
previousPath.mtime.getTime()
) {
console.log(
"File detected, starting import..."
);
// this walking business needs to go, SACURATAYYY, SACURATAYYY!! This dude needs to go.
const walkedFolder: IFolderData =
await broker.call("import.walkFolders", {
basePathToWalk: path,
});
console.log(walkedFolder);
await this.broker.call(
"import.processAndImportToDB",
{ walkedFolder }
);
} else {
setTimeout(
checkFileCopyComplete,
fileCopyDelaySeconds * 1000,
path,
stat
);
}
});
})
.on("change", (path, stats) =>
console.log(
`File ${path} has been changed. Stats: ${stats}`
};
fileWatcher
.on("add", async (path, stats) => {
console.log("Watcher detected new files.");
console.log(
`File ${path} has been added with stats: ${JSON.stringify(
stats
)}`
);
console.log("File copy started...");
fs.stat(path, function (err, stat) {
if (err) {
console.log(
"Error watching file for copy completion. ERR: " +
err.message
);
console.log(
"Error file not processed. PATH: " +
path
);
throw err;
}
setTimeout(
checkFileCopyComplete,
fileCopyDelaySeconds * 1000,
path,
stat
);
client.emit("action", {
type: "LS_COMIC_ADDED",
result: path,
});
});
})
.on("change", (path, stats) =>
console.log(
`File ${path} has been changed. Stats: ${stats}`
)
)
)
.on("unlink", (path) =>
console.log(`File ${path} has been removed`)
)
.on("addDir", (path) =>
console.log(`Directory ${path} has been added`)
);
.on("unlink", (path) =>
console.log(`File ${path} has been removed`)
)
.on("addDir", (path) =>
console.log(`Directory ${path} has been added`)
);
});
},
});
}

17
services/import.service.ts
View File

@@ -27,6 +27,7 @@ import klaw from "klaw";
import path from "path";
import { COMICS_DIRECTORY, USERDATA_DIRECTORY } from "../constants/directories";
console.log(process.env.MONGO_URI);
export default class ImportService extends Service {
public constructor(public broker: ServiceBroker) {
super(broker);
@@ -128,8 +129,7 @@ export default class ImportService extends Service {
params: {},
async handler(
ctx: Context<{
extractionOptions: any;
walkedFolders: {
walkedFolder: {
name: string;
path: string;
extension: string;
@@ -141,10 +141,9 @@ export default class ImportService extends Service {
}>
) {
try {
const { extractionOptions, walkedFolders } =
ctx.params;
const { walkedFolder } = ctx.params;
let comicExists = await Comic.exists({
"rawFileDetails.name": `${walkedFolders.name}`,
"rawFileDetails.name": `${walkedFolder.name}`,
});
// rough flow of import process
// 1. Walk folder
@@ -157,7 +156,7 @@ export default class ImportService extends Service {
| IExtractedComicBookCoverFile
| IExtractComicBookCoverErrorResponse
| IExtractedComicBookCoverFile[] = await extractCoverFromFile2(
extractionOptions
walkedFolder[0]
);
// 2. Add to mongo
@@ -185,7 +184,7 @@ export default class ImportService extends Service {
};
} else {
console.info(
`Comic: \"${walkedFolders.name}\" already exists in the database`
`Comic: \"${walkedFolder.name}\" already exists in the database`
);
}
} catch (error) {
@@ -459,7 +458,9 @@ export default class ImportService extends Service {
});
resp.on("end", () => {
console.log(`${apiDetailURL} returned data.`)
console.log(
`${apiDetailURL} returned data.`
);
const volumeInformation =
JSON.parse(data);
resolve(volumeInformation.results);

1
services/libraryqueue.service.ts
View File

@@ -14,6 +14,7 @@ import { extractCoverFromFile2 } from "../utils/uncompression.utils";
import { io } from "./api.service";
const REDIS_URI = process.env.REDIS_URI || `redis://0.0.0.0:6379`;
console.log(`REDIS -> ${REDIS_URI}`);
export default class LibraryQueueService extends Service {
public constructor(public broker: ServiceBroker) {
super(broker);

4
utils/file.utils.ts
View File

@@ -17,7 +17,7 @@ export const walkFolder = async (folder: string, formats: string[]): Promise<IFo
const result: IFolderData[] = [];
let walkResult: IFolderData = {
name: "",
path: "",
filePath: "",
extension: "",
containedIn: "",
isFile: false,
@@ -34,7 +34,7 @@ export const walkFolder = async (folder: string, formats: string[]): Promise<IFo
if ([...formats].includes(path.extname(dirent.name))) {
walkResult = {
name: path.basename(dirent.name, path.extname(dirent.name)),
path: path.dirname(pathname),
filePath: path.resolve(pathname),
extension: path.extname(dirent.name),
fileSize: fs.statSync(path.resolve(pathname)).size,
containedIn: path.dirname(pathname),

6
utils/uncompression.utils.ts
View File

@@ -55,7 +55,9 @@ export const extractCoverFromFile2 = async (
fileObject: any
): Promise<any> => {
try {
const { filePath, size} = fileObject;
console.log("ASDASD!@#!#!@#!@#");
console.log(fileObject);
const { filePath, fileSize} = fileObject;
const calibre = new Calibre();
console.info(`Initiating extraction process for path ${filePath}`);
@@ -111,7 +113,7 @@ export const extractCoverFromFile2 = async (
return {
name: fileNameWithoutExtension,
path: filePath,
fileSize: size,
fileSize,
extension: path.extname(filePath),
cover: {
filePath: path.relative(process.cwd(),renditionPath),