frishi/threetwo
1
0
Fork 0
Code Issues 9 Pull Requests 6 Actions Packages Projects Releases 2 Wiki Activity

🐰 RabbitMQ for enqueuing comic import jobs

Browse Source
This commit is contained in:
Rishi Ghan
2021-09-14 23:13:04 -07:00
parent e95eeb5f27
commit 0e039fb0a8
12 changed files with 276 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
src/client/actions/fileops.actions.tsx
View File

@@ -1,7 +1,6 @@
import axios from "axios";
import { IFolderData, IExtractedComicBookCoverFile } from "threetwo-ui-typings";
import { API_BASE_URI, SOCKET_BASE_URI } from "../constants/endpoints";
import { io } from "socket.io-client";
import {
IMS_COMICBOOK_METADATA_FETCHED,
IMS_SOCKET_CONNECTION_CONNECTED,
@@ -12,9 +11,11 @@ import {
IMS_CV_METADATA_IMPORT_CALL_IN_PROGRESS,
IMS_CV_METADATA_IMPORT_SUCCESSFUL,
IMS_CV_METADATA_IMPORT_FAILED,
RMQ_SOCKET_CONNECTED,
} from "../constants/action-types";
import { refineQuery } from "../shared/utils/filenameparser.utils";
import sortBy from "array-sort-by";
import { io } from "socket.io-client";
export async function walkFolder(path: string): Promise<Array<IFolderData>> {
return axios
@@ -43,6 +44,22 @@ export async function walkFolder(path: string): Promise<Array<IFolderData>> {
* @return {Promise<string>} HTML of the page
*/
export const fetchComicBookMetadata = (options) => async (dispatch) => {
const socket = io(SOCKET_BASE_URI, {
reconnectionDelayMax: 10000,
secure: false,
rejectUnauthorized: false,
});
socket.on("connect", () => {
console.log(`connect ${socket.id}`);
dispatch({
type: RMQ_SOCKET_CONNECTED,
isSocketConnected: true,
socketId: socket.id,
});
});
socket.on("disconnect", () => {
console.log(`disconnect`);
});
const extractionOptions = {
sourceFolder: options,
extractTarget: "cover",
@@ -55,37 +72,20 @@ export const fetchComicBookMetadata = (options) => async (dispatch) => {
};
const walkedFolders = await walkFolder("./comics");
const socket = io(SOCKET_BASE_URI, {
reconnectionDelayMax: 10000,
});
socket.on("connect", () => {
console.log(`connect ${socket.id}`);
dispatch({
type: IMS_SOCKET_CONNECTION_CONNECTED,
socketConnected: true,
await axios
.request({
url: "http://localhost:8050/api/getComicCovers",
method: "POST",
data: {
extractionOptions,
walkedFolders,
},
})
.then((response) => {
console.log("Response from import call", response);
});
});
socket.on("disconnect", () => {
console.log(`disconnect`);
});
socket.emit("importComicsToDB", {
action: "getComicCovers",
params: {
extractionOptions,
walkedFolders,
},
});
socket.on("comicBookCoverMetadata", (data: IExtractedComicBookCoverFile) => {
dispatch({
type: IMS_COMICBOOK_METADATA_FETCHED,
data,
dataTransferred: true,
});
});
socket.on("comicBookExists", (data) => {
socket.on("coverExtracted", (data) => {
console.log(data);
});
};

43
src/client/components/Import.tsx
View File

@@ -3,9 +3,8 @@ import { isUndefined } from "lodash";
import { connect } from "react-redux";
import { fetchComicBookMetadata } from "../actions/fileops.actions";
import { IFolderData } from "threetwo-ui-typings";
import { io, Socket } from "socket.io-client";
import { SOCKET_BASE_URI } from "../constants/endpoints";
import DynamicList, { createCache } from "react-window-dynamic-list";
import toast, { Toaster } from "react-hot-toast";
interface IProps {
matches: unknown;
@@ -18,7 +17,6 @@ interface IState {
searchPaneIndex: number;
fileOps: any;
}
let socket: Socket;
class Import extends React.Component<IProps, IState> {
/**
* Returns the average of two numbers.
@@ -47,16 +45,17 @@ class Import extends React.Component<IProps, IState> {
});
}
public initiateSocketConnection = () => {
public initiateImport = () => {
if (typeof this.props.path !== "undefined") {
socket = io(SOCKET_BASE_URI, {
reconnectionDelayMax: 10000,
});
socket.on("connect", () => {
console.log(`connect ${socket.id}`);
});
this.props.fetchComicMetadata();
toast.custom(
<div className="card">
<div className="card-content">Saokaaate</div>
</div>,
{
position: "top-right",
},
);
}
};
@@ -113,11 +112,9 @@ class Import extends React.Component<IProps, IState> {
</p>
</div>
</article>
<Toaster />
<p className="buttons">
<button
className="button is-medium"
onClick={this.initiateSocketConnection}
>
<button className="button is-medium" onClick={this.initiateImport}>
<span className="icon">
<i className="fas fa-file-import"></i>
</span>
@@ -132,18 +129,7 @@ class Import extends React.Component<IProps, IState> {
</button>
</p>
{!isUndefined(this.state.folderWalkResults) ? (
<div>
<DynamicList
data={this.props.covers}
cache={this.cache}
height={1000}
width={"100%"}
>
{this.renderRow}
</DynamicList>
</div>
) : null}
{!isUndefined(this.state.folderWalkResults) ? <div></div> : null}
</section>
</div>
);
@@ -154,7 +140,7 @@ function mapStateToProps(state: IState) {
console.log("state", state);
return {
// matches: state.comicInfo.searchResults,
covers: state.fileOps.comicBookMetadata,
// covers: state.fileOps.comicBookMetadata,
};
}
@@ -165,4 +151,3 @@ const mapDispatchToProps = (dispatch, ownProps) => ({
});
export default connect(mapStateToProps, mapDispatchToProps)(Import);
export { socket };

10
src/client/components/Navbar.tsx
View File

@@ -1,10 +1,7 @@
import React from "react";
import { useSelector } from "react-redux";
import { Link } from "react-router-dom";
import { RootState } from "threetwo-ui-typings";
const Navbar: React.FunctionComponent = (props) => {
const socketConnection = useSelector((state: RootState) => state.fileOps);
return (
<nav className="navbar is-fixed-top">
@@ -195,13 +192,6 @@ const Navbar: React.FunctionComponent = (props) => {
<a className="navbar-item is-hidden-desktop-only"></a>
<div className="navbar-item">
<div className="field is-grouped">
<p className="control">
{socketConnection.socketConnected ? (
<span className="icon is-small has-text-success">
<i className="fas fa-plug"></i>
</span>
) : null}
</p>
<p className="control">
<Link to="/settings" className="navbar-item">
Settings

10
src/client/constants/action-types.ts
View File

@@ -6,11 +6,11 @@ export const CV_CLEANUP = "CV_CLEANUP";
export const CV_API_GENERIC_FAILURE = "CV_API_GENERIC_FAILURE";
export const IMS_COMICBOOK_METADATA_FETCHED = "IMS_SOCKET_DATA_FETCHED";
export const IMS_SOCKET_CONNECTION_CONNECTED =
"IMS_SOCKET_CONNECTION_CONNECTED";
export const IMS_SOCKET_CONNECTION_DISCONNECTED =
"IMS_SOCKET_CONNECTION_DISCONNECTED";
export const IMS_SOCKET_ERROR = "IMS_SOCKET_ERROR";
// rabbitmq
export const RMQ_SOCKET_CONNECTED = "RMQ_SOCKET_CONNECTED";
export const RMQ_SOCKET_DISCONNECTED = "RMQ_SOCKET_DISCONNECTED";
export const RMQ_SOCKET_ERROR = "RMQ_SOCKET_ERROR";
export const IMS_RAW_IMPORT_SUCCESSFUL = "IMS_RAW_IMPORT_SUCCESSFUL";
export const IMS_RAW_IMPORT_FAILED = "IMS_RAW_IMPORT_FAILED";

2
src/client/constants/endpoints.ts
View File

@@ -1,3 +1,3 @@
export const COMICBOOKINFO_SERVICE_URI = "http://localhost:3080/api/comicvine/";
export const API_BASE_URI = "http://localhost:8050/api/";
export const SOCKET_BASE_URI = "ws://localhost:3000/";
export const SOCKET_BASE_URI = "ws://localhost:8051/";

12
src/client/reducers/fileops.reducer.ts
View File

@@ -1,8 +1,8 @@
import {
IMS_SOCKET_CONNECTION_CONNECTED,
IMS_SOCKET_CONNECTION_DISCONNECTED,
RMQ_SOCKET_CONNECTED,
RMQ_SOCKET_DISCONNECTED,
IMS_COMICBOOK_METADATA_FETCHED,
IMS_SOCKET_ERROR,
RMQ_SOCKET_ERROR,
IMS_RAW_IMPORT_SUCCESSFUL,
IMS_RAW_IMPORT_FAILED,
IMS_RECENT_COMICS_FETCHED,
@@ -14,7 +14,7 @@ import {
const initialState = {
dataTransferred: false,
comicBookMetadata: [],
socketConnected: false,
isSocketConnected: false,
isComicVineMetadataImportInProgress: false,
comicVineMetadataImportError: {},
rawImportError: {},
@@ -29,10 +29,10 @@ function fileOpsReducer(state = initialState, action) {
dataTransferred: true,
};
case IMS_SOCKET_CONNECTION_CONNECTED:
case RMQ_SOCKET_CONNECTED:
return {
...state,
socketConnected: action.socketConnected,
isSocketConnected: action.isSocketConnected,
};
case IMS_RAW_IMPORT_SUCCESSFUL:
return {

100
src/server/index.ts
View File

@@ -1,20 +1,14 @@
import express, { Request, Response, Router, Express } from "express";
import bodyParser from "body-parser";
import { basename, extname, join } from "path";
import { lookup } from "mime-types";
import { promises as fs } from "fs";
import { responseStream } from "http-response-stream";
import { isUndefined } from "lodash";
import { buildAsync } from "calibre-opds";
import initMain from "calibre-opds/lib/index";
import { EnumLinkRel } from "opds-extra/lib/const";
import { async as FastGlob } from "@bluelovers/fast-glob/bluebird";
import { Entry, Feed } from "opds-extra/lib/v1";
import { Link } from "opds-extra/lib/v1/core";
import { createServer } from "http";
import { Server, Socket } from "socket.io";
import router from "./route";
// call express
const app: Express = express(); // define our app using express
const router = Router();
const httpServer = createServer();
export const io = new Server(httpServer, {});
// configure app to use bodyParser for
// Getting data from body of requests
@@ -27,78 +21,28 @@ const port: number = Number(process.env.PORT) || 8050; // set our port
app.use(express.static("dist"));
app.use(express.static("public"));
export const opdsRouter = () => {
const path_of_books = "/Users/rishi/work/threetwo/src/server/comics";
router.use("/opds", async (req, res, next) => {
return buildAsync(
initMain({
title: `title`,
subtitle: `subtitle`,
icon: "/favicon.ico",
}),
[
async (feed: Feed) => {
feed.books = feed.books || [];
await FastGlob(["*.cbr", "*.cbz", "*.cb7", "*.cba", "*.cbt"], {
cwd: path_of_books,
}).each((file) => {
const ext = extname(file);
const title = basename(file, ext);
const href = encodeURI(`/file/${file}`);
const type = lookup(ext) || "application/octet-stream";
const entry = Entry.deserialize<Entry>({
title,
links: [
{
rel: EnumLinkRel.ACQUISITION,
href,
type,
} as Link,
],
});
if (!isUndefined(feed) && !isUndefined(feed.books)) {
console.log("haramzada", feed.books);
feed.books.push(entry);
}
});
return feed;
},
],
).then((feed) => {
res.setHeader("Content-Type", "application/xml");
return res.end(feed.toXML());
});
});
router.use("/file/*", async (req, res) => {
const file: string = req.params[0];
const ext = extname(file);
if ([".cbr", ".cbz", ".cb7", ".cba", ".cbt"].includes(ext)) {
const content = await fs.readFile(join(path_of_books, file));
const mime = lookup(ext) || "application/octet-stream";
res.set("Content-Type", mime);
return responseStream(res, content);
}
res.status(404).end(`'${file}' not exists`);
});
return router;
};
app.get("/", (req: Request, res: Response) => {
console.log("sending index.html");
res.sendFile("/dist/index.html");
});
app.use(opdsRouter());
// REGISTER ROUTES
// all of the routes will be prefixed with /api
const routes: Router[] = Object.values(router);
app.use("/api", routes);
app.listen(port);
console.log(`App listening on ${port}`);
io.on("connection", (socket) => {
console.log("Socket connected");
//Whenever someone disconnects this piece of code executed
socket.on("disconnect", () => {
console.log("Socket disconnected");
});
});
// socket server
httpServer.listen(8051);
console.log(`Socket server is listening on 8051`);
console.log(`Server is listening on ${port}`);

4
src/server/route/index.ts Normal file
View File

@@ -0,0 +1,4 @@
import extra from "./routes/importComics.routes";
import opds from "./routes/opds.routes";
export default { extra, opds };

54
src/server/route/routes/importComics.routes.ts Normal file
View File

@@ -0,0 +1,54 @@
import router from "../router";
import { Request, Response } from "express";
const amqp = require("amqplib/callback_api");
import axios from "axios";
import { io } from "../../index";
router.route("/getComicCovers").post(async (req: Request, res: Response) => {
typeof req.body === "object" ? req.body : {};
await axios.request({
url: "http://localhost:3000/api/import/importComicsToDB",
method: "POST",
data: {
extractionOptions: req.body.extractionOptions,
walkedFolders: req.body.walkedFolders,
},
});
const queueConsumer = amqp.connect(
"amqp://localhost",
(error0, connection) => {
if (error0) {
throw error0;
}
connection.createChannel((error1, channel) => {
if (error1) {
throw error1;
}
const queue = "comicBookCovers";
channel.assertQueue(queue, {
durable: false,
});
console.log(`Connected to ${queue}`);
console.log(`Waiting for comic book cover data in ${queue}`);
channel.consume(
queue,
(data) => {
//Socket Trigger All Clients
io.sockets.emit(
"coverExtracted",
JSON.parse(data.content.toString()),
);
},
{
noAck: true,
},
);
});
},
);
res.send({ queue: queueConsumer });
});
export default router;

73
src/server/route/routes/opds.routes.ts Normal file
View File

@@ -0,0 +1,73 @@
import { basename, extname, join } from "path";
import { lookup } from "mime-types";
import { promises as fs } from "fs";
import { responseStream } from "http-response-stream";
import { isUndefined } from "lodash";
import { buildAsync } from "calibre-opds";
import initMain from "calibre-opds/lib/index";
import { EnumLinkRel } from "opds-extra/lib/const";
import { async as FastGlob } from "@bluelovers/fast-glob/bluebird";
import { Entry, Feed } from "opds-extra/lib/v1";
import { Link } from "opds-extra/lib/v1/core";
import router from "../router";
const path_of_books = "/Users/rishi/work/threetwo/src/server/comics";
router.use("/opds", async (req, res, next) => {
return buildAsync(
initMain({
title: `title`,
subtitle: `subtitle`,
icon: "/favicon.ico",
}),
[
async (feed: Feed) => {
feed.books = feed.books || [];
await FastGlob(["*.cbr", "*.cbz", "*.cb7", "*.cba", "*.cbt"], {
cwd: path_of_books,
}).each((file) => {
const ext = extname(file);
const title = basename(file, ext);
const href = encodeURI(`/file/${file}`);
const type = lookup(ext) || "application/octet-stream";
const entry = Entry.deserialize<Entry>({
title,
links: [
{
rel: EnumLinkRel.ACQUISITION,
href,
type,
} as Link,
],
});
if (!isUndefined(feed) && !isUndefined(feed.books)) {
console.log("haramzada", feed.books);
feed.books.push(entry);
}
});
return feed;
},
],
).then((feed) => {
res.setHeader("Content-Type", "application/xml");
return res.end(feed.toXML());
});
});
router.use("/file/*", async (req, res) => {
const file: string = req.params[0];
const ext = extname(file);
if ([".cbr", ".cbz", ".cb7", ".cba", ".cbt"].includes(ext)) {
const content = await fs.readFile(join(path_of_books, file));
const mime = lookup(ext) || "application/octet-stream";
res.set("Content-Type", mime);
return responseStream(res, content);
}
res.status(404).end(`'${file}' not exists`);
});
export default router;