From d065225d8e530d5fe37ae8f363f1d22f5f7578db Mon Sep 17 00:00:00 2001 From: Rishi Ghan Date: Wed, 21 Dec 2022 21:17:38 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=8F=97=EF=B8=8F=20Refactoring=20archive?= =?UTF-8?q?=20uncompression=20for=20"Read=20Comic"=20and=20"Analysis"=20us?= =?UTF-8?q?er=20flows=20(#46)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🔧 Refactoring uncompression methods on client-side * ✏️ Refactoring * 👁️ Updates to the comic viewer * 🖼️ Added screenshots from December 2022 * ✏️ Fixed typo in README * 🏗️ Massive refactor around archive uncompression for reading/analysis * 🔧 Tweaked state vars for reading and analysis * 🏗️ Refactor to support DC++ and socket.io integration This refactor covers the following workflows: 1. Adding a comic from LOCG or ComicVine adds it to the wanted list 2. Downloading that comic from DC++ correctly adds download metadata to the corresponding comic object in mongo 3. Successful download triggers automatic import to library and cover extraction, metadata application --- package.json | 2 +- src/client/actions/fileops.actions.tsx | 91 +++++++++---------- src/client/components/App.tsx | 47 +++++++--- .../ComicDetail/AcquisitionPanel.tsx | 8 +- .../components/ComicDetail/ComicDetail.tsx | 4 +- .../components/ComicDetail/RawFileDetails.tsx | 3 +- .../ComicDetail/Tabs/ArchiveOperations.tsx | 12 ++- src/client/components/Dashboard/Dashboard.tsx | 9 +- src/client/components/Dashboard/PullList.tsx | 2 +- .../components/Dashboard/RecentlyImported.tsx | 10 +- src/client/constants/action-types.ts | 2 + src/client/reducers/fileops.reducer.ts | 54 +++++++++-- src/client/shared/utils/metadata.utils.ts | 5 +- yarn.lock | 2 +- 14 files changed, 158 insertions(+), 93 deletions(-) diff --git a/package.json b/package.json index d8433f2..3c1f1f0 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "pretty-bytes": "^5.6.0", "prop-types": "^15.8.1", "qs": "^6.10.5", - "react": "^18.1.0", + "react": "^18.2.0", "react-collapsible": "^2.9.0", "react-comic-viewer": "^0.4.0", "react-day-picker": "^8.0.6", diff --git a/src/client/actions/fileops.actions.tsx b/src/client/actions/fileops.actions.tsx index ee03372..1c25dc0 100644 --- a/src/client/actions/fileops.actions.tsx +++ b/src/client/actions/fileops.actions.tsx @@ -36,7 +36,7 @@ import { CV_WEEKLY_PULLLIST_FETCHED, } from "../constants/action-types"; import { success } from "react-notification-system-redux"; -import { removeLeadingPeriod } from "../shared/utils/formatting.utils"; + import { isNil, map } from "lodash"; export async function walkFolder(path: string): Promise> { @@ -60,15 +60,6 @@ export async function walkFolder(path: string): Promise> { * @return the comic book metadata */ export const fetchComicBookMetadata = () => async (dispatch) => { - const extractionOptions = { - extractTarget: "cover", - targetExtractionFolder: "./userdata/covers", - extractionMode: "bulk", - paginationOptions: { - pageLimit: 25, - page: 1, - }, - }; dispatch({ type: LS_IMPORT_CALL_IN_PROGRESS, }); @@ -86,7 +77,7 @@ export const fetchComicBookMetadata = () => async (dispatch) => { dispatch({ type: LS_IMPORT, meta: { remote: true }, - data: { extractionOptions }, + data: {}, }); }; export const toggleImportQueueStatus = (options) => async (dispatch) => { @@ -136,21 +127,24 @@ export const getComicBooks = (options) => async (dispatch) => { * @returns Nothing. * @param payload */ -export const importToDB = (sourceName: string, payload?: any) => (dispatch) => { +export const importToDB = (sourceName: string, metadata?: any) => (dispatch) => { try { const comicBookMetadata = { - rawFileDetails: { - name: "", - }, - importStatus: { - isImported: true, - tagged: false, - matchedResult: { - score: "0", + importType: "new", + payload: { + rawFileDetails: { + name: "", }, - }, - sourcedMetadata: payload || null, - acquisition: { source: { wanted: true, name: sourceName } }, + importStatus: { + isImported: true, + tagged: false, + matchedResult: { + score: "0", + }, + }, + sourcedMetadata: metadata || null, + acquisition: { source: { wanted: true, name: sourceName } }, + } }; dispatch({ type: IMS_CV_METADATA_IMPORT_CALL_IN_PROGRESS, @@ -260,34 +254,31 @@ export const fetchComicVineMatches = * @returns {any} */ export const extractComicArchive = - (path: string, options: any): any => async (dispatch) => { - const comicBookPages: string[] = []; - console.log(options); - dispatch({ - type: IMS_COMIC_BOOK_ARCHIVE_EXTRACTION_CALL_IN_PROGRESS, - }); - const extractedComicBookArchive = await axios({ - method: "POST", - url: `${LIBRARY_SERVICE_BASE_URI}/uncompressFullArchive`, - headers: { - "Content-Type": "application/json; charset=utf-8", - }, - data: { - filePath: path, - options, - }, - }); - map(extractedComicBookArchive.data, (page) => { - const pageFilePath = removeLeadingPeriod(page); - const imagePath = encodeURI(`${LIBRARY_SERVICE_HOST}${pageFilePath}`); - comicBookPages.push(imagePath); - }); - dispatch({ - type: IMS_COMIC_BOOK_ARCHIVE_EXTRACTION_SUCCESS, - extractedComicBookArchive: comicBookPages, - }); - }; + (path: string, options: any): any => + async (dispatch) => { + dispatch({ + type: IMS_COMIC_BOOK_ARCHIVE_EXTRACTION_CALL_IN_PROGRESS, + }); + await axios({ + method: "POST", + url: `${LIBRARY_SERVICE_BASE_URI}/uncompressFullArchive`, + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + data: { + filePath: path, + options, + }, + }); + }; + +/** + * Description + * @param {any} query + * @param {any} options + * @returns {any} + */ export const searchIssue = (query, options) => async (dispatch) => { dispatch({ type: SS_SEARCH_IN_PROGRESS, diff --git a/src/client/components/App.tsx b/src/client/components/App.tsx index 2efa3e7..6446767 100644 --- a/src/client/components/App.tsx +++ b/src/client/components/App.tsx @@ -18,18 +18,22 @@ import { AirDCPPSocketContext, } from "../context/AirDCPPSocket"; import { isEmpty, isUndefined } from "lodash"; -import { AIRDCPP_DOWNLOAD_PROGRESS_TICK } from "../constants/action-types"; -import { useDispatch } from "react-redux"; +import { + AIRDCPP_DOWNLOAD_PROGRESS_TICK, + LS_SINGLE_IMPORT, +} from "../constants/action-types"; +import { useDispatch, useSelector } from "react-redux"; /** * Method that initializes an AirDC++ socket connection * 1. Initializes event listeners for download init, tick and complete events - * 2. Handles errors in case the connection to AirDC++ is not established or terminated - * @returns void + * 2. Handles errors in case the connection to AirDC++ is not established or terminated + * @returns void */ const AirDCPPSocketComponent = (): ReactElement => { const airDCPPConfiguration = useContext(AirDCPPSocketContext); const dispatch = useDispatch(); + useEffect(() => { const initializeAirDCPPEventListeners = async () => { if ( @@ -42,9 +46,7 @@ const AirDCPPSocketComponent = (): ReactElement => { "queue_bundle_added", async (data) => { console.log("JEMEN:", data); - - - } + }, ); // download tick listener await airDCPPConfiguration.airDCPPState.socket.addListener( @@ -62,9 +64,18 @@ const AirDCPPSocketComponent = (): ReactElement => { `queue`, "queue_bundle_status", async (bundleData) => { + let count = 0; if (bundleData.status.completed && bundleData.status.downloaded) { // dispatch the action for raw import, with the metadata - console.log("IM THE MAN UP IN THIS") + if (count < 1) { + console.log(`[AirDCPP]: Download complete.`); + dispatch({ + type: LS_SINGLE_IMPORT, + meta: { remote: true }, + data: bundleData, + }); + count += 1; + } } }, ); @@ -92,7 +103,10 @@ export const App = (): ReactElement => { } /> } /> - } /> + } + /> } /> } /> } /> @@ -105,9 +119,18 @@ export const App = (): ReactElement => { element={} /> } /> - } /> - } /> - } /> + } + /> + } + /> + } + /> diff --git a/src/client/components/ComicDetail/AcquisitionPanel.tsx b/src/client/components/ComicDetail/AcquisitionPanel.tsx index 97d0c3d..a6c8ec2 100644 --- a/src/client/components/ComicDetail/AcquisitionPanel.tsx +++ b/src/client/components/ComicDetail/AcquisitionPanel.tsx @@ -16,6 +16,7 @@ import ellipsize from "ellipsize"; import { Form, Field } from "react-final-form"; import { isEmpty, isNil, map } from "lodash"; import { AirDCPPSocketContext } from "../../context/AirDCPPSocket"; + interface IAcquisitionPanelProps { query: any; comicObjectId: any; @@ -96,9 +97,12 @@ export const AcquisitionPanel = ( (searchInstanceId, resultId, name, size, type) => { dispatch( downloadAirDCPPItem( - searchInstanceId, resultId, + searchInstanceId, + resultId, props.comicObjectId, - name, size, type, + name, + size, + type, airDCPPConfiguration.airDCPPState.socket, { username: `${airDCPPConfiguration.airDCPPState.settings.directConnect.client.host.username}`, diff --git a/src/client/components/ComicDetail/ComicDetail.tsx b/src/client/components/ComicDetail/ComicDetail.tsx index 0145c09..66076eb 100644 --- a/src/client/components/ComicDetail/ComicDetail.tsx +++ b/src/client/components/ComicDetail/ComicDetail.tsx @@ -66,7 +66,7 @@ export const ComicDetail = (data: ComicDetailProps): ReactElement => { ); const extractedComicBook = useSelector( - (state: RootState) => state.fileOps.extractedComicBookArchive, + (state: RootState) => state.fileOps.extractedComicBookArchive.reading, ); const { comicObjectId } = useParams<{ comicObjectId: string }>(); @@ -77,7 +77,7 @@ export const ComicDetail = (data: ComicDetailProps): ReactElement => { dispatch( extractComicArchive(filePath, { type: "full", - purpose: "readComicBook", + purpose: "reading", imageResizeOptions: { baseWidth: 1024, }, diff --git a/src/client/components/ComicDetail/RawFileDetails.tsx b/src/client/components/ComicDetail/RawFileDetails.tsx index 7db0d4f..f2fa448 100644 --- a/src/client/components/ComicDetail/RawFileDetails.tsx +++ b/src/client/components/ComicDetail/RawFileDetails.tsx @@ -38,7 +38,8 @@ export const RawFileDetails = (props): ReactElement => { -
+ +
{/* inferred metadata */}
Inferred Issue Metadata
diff --git a/src/client/components/ComicDetail/Tabs/ArchiveOperations.tsx b/src/client/components/ComicDetail/Tabs/ArchiveOperations.tsx index c9655ed..839dcdc 100644 --- a/src/client/components/ComicDetail/Tabs/ArchiveOperations.tsx +++ b/src/client/components/ComicDetail/Tabs/ArchiveOperations.tsx @@ -14,7 +14,7 @@ export const ArchiveOperations = (props): ReactElement => { (state: RootState) => state.fileOps.comicBookExtractionInProgress, ); const extractedComicBookArchive = useSelector( - (state: RootState) => state.fileOps.extractedComicBookArchive, + (state: RootState) => state.fileOps.extractedComicBookArchive.analysis, ); const imageAnalysisResult = useSelector((state: RootState) => { @@ -23,7 +23,15 @@ export const ArchiveOperations = (props): ReactElement => { const dispatch = useDispatch(); const unpackComicArchive = useCallback(() => { - dispatch(extractComicArchive(data.rawFileDetails.filePath)); + dispatch( + extractComicArchive(data.rawFileDetails.filePath, { + type: "full", + purpose: "analysis", + imageResizeOptions: { + baseWidth: 275, + }, + }), + ); }, []); // sliding panel config diff --git a/src/client/components/Dashboard/Dashboard.tsx b/src/client/components/Dashboard/Dashboard.tsx index 4a724c7..77db77a 100644 --- a/src/client/components/Dashboard/Dashboard.tsx +++ b/src/client/components/Dashboard/Dashboard.tsx @@ -11,7 +11,7 @@ import { getComicBooks, } from "../../actions/fileops.actions"; import { getLibraryStatistics } from "../../actions/comicinfo.actions"; -import { isEmpty } from "lodash"; +import { isEmpty, isNil } from "lodash"; export const Dashboard = (): ReactElement => { const dispatch = useDispatch(); @@ -43,7 +43,7 @@ export const Dashboard = (): ReactElement => { }, []); const recentComics = useSelector( - (state: RootState) => state.fileOps.recentComics, + (state: RootState) => state.fileOps.recentComics ); const wantedComics = useSelector( (state: RootState) => state.fileOps.wantedComics, @@ -60,7 +60,7 @@ export const Dashboard = (): ReactElement => {

Dashboard

- {!isEmpty(recentComics) && !isEmpty(recentComics.docs) ? ( + {!isEmpty(recentComics) ? ( <> {/* Pull List */} @@ -74,9 +74,8 @@ export const Dashboard = (): ReactElement => { )} {/* Recent imports */} - {!isEmpty(recentComics) && ( - )} + {/* Volumes */} {!isEmpty(volumeGroups) && ( diff --git a/src/client/components/Dashboard/PullList.tsx b/src/client/components/Dashboard/PullList.tsx index 4b62d86..b00fe82 100644 --- a/src/client/components/Dashboard/PullList.tsx +++ b/src/client/components/Dashboard/PullList.tsx @@ -20,7 +20,7 @@ export const PullList = ({ issues }: PullListProps): ReactElement => { useEffect(() => { dispatch( getWeeklyPullList({ - startDate: "2022-11-15", + startDate: "2022-12-25", pageSize: "15", currentPage: "1", }), diff --git a/src/client/components/Dashboard/RecentlyImported.tsx b/src/client/components/Dashboard/RecentlyImported.tsx index ad784e2..b26b954 100644 --- a/src/client/components/Dashboard/RecentlyImported.tsx +++ b/src/client/components/Dashboard/RecentlyImported.tsx @@ -24,7 +24,6 @@ export const RecentlyImported = ({ 700: 2, 600: 2, }; - return ( <>
@@ -41,7 +40,7 @@ export const RecentlyImported = ({ columnClassName="recent-comics-column" > {map( - comicBookCovers.docs, + comicBookCovers, ( { _id, @@ -53,6 +52,7 @@ export const RecentlyImported = ({ }, idx, ) => { + console.log(comicvine); const { issueName, url } = determineCoverFile({ rawFileDetails, comicvine, @@ -64,7 +64,7 @@ export const RecentlyImported = ({ comicInfo, locg, }); - + console.log(name); const isComicBookMetadataAvailable = !isUndefined(comicvine) && !isUndefined(comicvine.volumeInformation); @@ -123,7 +123,7 @@ export const RecentlyImported = ({
{/* metadata card */} - {!isNil(name) ? ( + {!isNil(name) && (
@@ -138,7 +138,7 @@ export const RecentlyImported = ({
- ) : null} + )} ); }, diff --git a/src/client/constants/action-types.ts b/src/client/constants/action-types.ts index a6be92d..f058853 100644 --- a/src/client/constants/action-types.ts +++ b/src/client/constants/action-types.ts @@ -81,6 +81,8 @@ export const IMS_COMIC_BOOK_ARCHIVE_EXTRACTION_CALL_IN_PROGRESS = export const IMS_COMIC_BOOK_ARCHIVE_EXTRACTION_CALL_FAILED = "IMS_COMIC_BOOK_ARCHIVE_EXTRACTION_CALL_FAILED"; +export const COMICBOOK_EXTRACTION_SUCCESS = "COMICBOOK_EXTRACTION_SUCCESS"; + // Image file stats export const IMG_ANALYSIS_CALL_IN_PROGRESS = "IMG_ANALYSIS_CALL_IN_PROGRESS"; export const IMG_ANALYSIS_DATA_FETCH_SUCCESS = diff --git a/src/client/reducers/fileops.reducer.ts b/src/client/reducers/fileops.reducer.ts index 2060797..cdbda61 100644 --- a/src/client/reducers/fileops.reducer.ts +++ b/src/client/reducers/fileops.reducer.ts @@ -29,7 +29,11 @@ import { SS_SEARCH_FAILED, SS_SEARCH_RESULTS_FETCHED_SPECIAL, VOLUMES_FETCHED, + COMICBOOK_EXTRACTION_SUCCESS, } from "../constants/action-types"; +import { removeLeadingPeriod } from "../shared/utils/formatting.utils"; +import { LIBRARY_SERVICE_HOST } from "../constants/endpoints"; + const initialState = { IMSCallInProgress: false, IMGCallInProgress: false, @@ -42,7 +46,10 @@ const initialState = { isComicVineMetadataImportInProgress: false, comicVineMetadataImportError: {}, rawImportError: {}, - extractedComicBookArchive: [], + extractedComicBookArchive: { + reading: [], + analysis: [], + }, recentComics: [], wantedComics: [], libraryComics: [], @@ -81,7 +88,7 @@ function fileOpsReducer(state = initialState, action) { case IMS_RECENT_COMICS_FETCHED: return { ...state, - recentComics: action.data, + recentComics: action.data.docs, }; case IMS_WANTED_COMICS_FETCHED: return { @@ -131,13 +138,7 @@ function fileOpsReducer(state = initialState, action) { comicBookExtractionInProgress: true, }; } - case IMS_COMIC_BOOK_ARCHIVE_EXTRACTION_SUCCESS: { - return { - ...state, - extractedComicBookArchive: action.extractedComicBookArchive, - comicBookExtractionInProgress: false, - }; - } + case LOCATION_CHANGE: { return { ...state, @@ -152,11 +153,44 @@ function fileOpsReducer(state = initialState, action) { } case LS_COVER_EXTRACTED: { console.log("BASH", action); + if(state.recentComics.length === 5) { + state.recentComics.pop(); + } return { ...state, librarySearchResultCount: state.librarySearchResultCount + 1, + recentComics: [...state.recentComics, action.result.data.importResult] }; } + + case COMICBOOK_EXTRACTION_SUCCESS: { + const comicBookPages: string[] = []; + map(action.result.files, (page) => { + const pageFilePath = removeLeadingPeriod(page); + const imagePath = encodeURI(`${LIBRARY_SERVICE_HOST}${pageFilePath}`); + comicBookPages.push(imagePath); + }); + + switch (action.result.purpose) { + case "reading": + return { + ...state, + extractedComicBookArchive: { + reading: comicBookPages, + }, + comicBookExtractionInProgress: false, + }; + + case "analysis": + return { + ...state, + extractedComicBookArchive: { + analysis: comicBookPages, + }, + comicBookExtractionInProgress: false, + }; + } + } case LS_QUEUE_DRAINED: { console.log("drained", action); return { @@ -229,7 +263,7 @@ function fileOpsReducer(state = initialState, action) { volumes: action.data, SSCallInProgress: false, }; - + case SS_SEARCH_FAILED: { return { ...state, diff --git a/src/client/shared/utils/metadata.utils.ts b/src/client/shared/utils/metadata.utils.ts index 4130fe0..f32c325 100644 --- a/src/client/shared/utils/metadata.utils.ts +++ b/src/client/shared/utils/metadata.utils.ts @@ -67,7 +67,10 @@ export const determineCoverFile = (data) => { } }; -export const determineExternalMetadata = (metadataSource, source) => { +export const determineExternalMetadata = ( + metadataSource: string, + source: any +) => { switch (metadataSource) { case "comicvine": return { diff --git a/yarn.lock b/yarn.lock index c4af02d..a11303c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15607,7 +15607,7 @@ react-transition-group@4.4.2, react-transition-group@^4.3.0: loose-envify "^1.4.0" prop-types "^15.6.2" -react@^18.1.0: +react@^18.2.0: version "18.2.0" resolved "https://registry.yarnpkg.com/react/-/react-18.2.0.tgz#555bd98592883255fa00de14f1151a917b5d77d5" integrity sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==