diff --git a/services/comicvine.service.ts b/services/comicvine.service.ts index ef5ae66..09db55c 100644 --- a/services/comicvine.service.ts +++ b/services/comicvine.service.ts @@ -2,15 +2,16 @@ import { Service, ServiceBroker, Context } from "moleculer"; import axios from "axios"; -import delay from "delay"; import { isNil, isUndefined } from "lodash"; import { fetchReleases, FilterTypes, SortTypes } from "comicgeeks"; import { matchScorer, rankVolumes } from "../utils/searchmatchscorer.utils"; import { scrapeIssuesFromSeriesPage, scrapeIssuePage, + getWeeklyPullList, } from "../utils/scraping.utils"; const { calculateLimitAndOffset, paginate } = require("paginate-info"); +const { MoleculerError } = require("moleculer").Errors; const CV_BASE_URL = "https://comicvine.gamespot.com/api/"; console.log("ComicVine API Key: ", process.env.COMICVINE_API_KEY); @@ -28,7 +29,7 @@ export default class ComicVineService extends Service { format: string; sort: string; query: string; - fieldList: string; + field_list: string; limit: string; offset: string; resources: string; @@ -53,18 +54,23 @@ export default class ComicVineService extends Service { handler: async ( ctx: Context<{ volumeURI: string; - data: {}; + fieldList: string; }> ) => { + const { volumeURI, fieldList } = ctx.params; const response = await axios.request({ url: - ctx.params.volumeURI + + volumeURI + "?api_key=" + process.env.COMICVINE_API_KEY, params: { format: "json", + field_list: fieldList, + }, + headers: { + Accept: "application/json", + "User-Agent": "ThreeTwo", }, - headers: { Accept: "application/json" }, }); const { data } = response; return data; @@ -72,14 +78,14 @@ export default class ComicVineService extends Service { }, getIssuesForSeries: { rest: "POST /getIssuesForSeries", - params: {}, handler: async ( - ctx: Context<{ comicObjectID: string }> + ctx: Context<{ comicObjectId: string }> ) => { + const { comicObjectId } = ctx.params; // 1. Query mongo to get the comic document by its _id const comicBookDetails: any = await this.broker.call( "library.getComicBookById", - { id: ctx.params.comicObjectID } + { id: comicObjectId } ); // 2. Query CV and get metadata for them const issues = await axios({ @@ -131,22 +137,8 @@ export default class ComicVineService extends Service { pageSize ); - const response = await fetchReleases( - new Date(ctx.params.startDate), - { - publishers: [ - "DC Comics", - "Marvel Comics", - "Image Comics", - ], - filter: [ - FilterTypes.Regular, - FilterTypes.Digital, - FilterTypes.Annual, - ], - sort: SortTypes.AlphaAsc, - } - ); + const response = await getWeeklyPullList(); + console.log(JSON.stringify(response, null, 4)); const count = response.length; const paginatedData = response.slice( @@ -161,6 +153,47 @@ export default class ComicVineService extends Service { return { result: paginatedData, meta: paginationInfo }; }, }, + getResource: { + rest: "POST /getResource", + handler: async ( + ctx: Context<{ + resources: string; + filter: string; + fieldList: string; + }> + ) => { + const { resources, filter, fieldList } = ctx.params; + console.log(JSON.stringify(ctx.params, null, 2)); + console.log( + CV_BASE_URL + + `${resources}` + + "?api_key=" + + process.env.COMICVINE_API_KEY + ); + // 2. Query CV and get metadata for them + const response = await axios({ + method: "GET", + url: + CV_BASE_URL + + `${resources}` + + "?api_key=" + + process.env.COMICVINE_API_KEY, + params: { + resources: `${resources}`, + limit: "100", + format: "json", + filter: `${filter}`, + field_list: `${fieldList}`, + }, + headers: { + Accept: "application/json", + "User-Agent": "ThreeTwo", + }, + }); + console.log(response.data); + return response.data; + }, + }, volumeBasedSearch: { rest: "POST /volumeBasedSearch", params: {}, @@ -223,7 +256,7 @@ export default class ComicVineService extends Service { } ); - // 2b. cover_date:2014-01-01|2016-12-31 for the issue year 2015 + // 2b. E.g.: cover_date:2014-01-01|2016-12-31 for the issue year 2015 let coverDateFilter = ""; if ( !isNil( @@ -330,6 +363,172 @@ export default class ComicVineService extends Service { ); }, }, + getStoryArcs: { + rest: "POST /getStoryArcs", + handler: async ( + ctx: Context<{ volumeUrl: string; volumeId: number }> + ) => { + const { volumeUrl, volumeId } = ctx.params; + try { + const volumeResponse = await axios({ + url: + volumeUrl + + "?api_key=" + + process.env.COMICVINE_API_KEY, + method: "GET", + params: { + limit: "100", + format: "json", + resources: "volumes", + }, + headers: { + Accept: "application/json", + "User-Agent": "ThreeTwo", + }, + }); + const volumeData = volumeResponse.data; + + if (volumeData.results.issues.length > 0) { + const issuePromises = + volumeData.results.issues.map( + async (issue: any) => { + const issueUrl = `${CV_BASE_URL}issue/4000-${issue.id}/?api_key=${process.env.COMICVINE_API_KEY}&format=json&field_list=story_arc_credits,description,image`; + try { + const issueResponse = + await axios.get(issueUrl, { + params: { + limit: "100", + format: "json", + }, + headers: { + Accept: "application/json", + "User-Agent": + "ThreeTwo", + }, + }); + const issueData = + issueResponse.data.results; + + // Transform each story arc to include issue's description and image + return ( + issueData.story_arc_credits?.map( + (arc: any) => ({ + ...arc, + issueDescription: + issueData.description, + issueImage: + issueData.image, + }) + ) || [] + ); + } catch (error) { + console.error( + "An error occurred while fetching issue data:", + error.message + ); + return []; // Return an empty array on error + } + } + ); + + try { + const storyArcsResults: any = + await Promise.all(issuePromises); + // Flatten the array of arrays + const flattenedStoryArcs = + storyArcsResults.flat(); + + // Deduplicate based on arc ID, while preserving the last seen issueDescription and issueImage + const uniqueStoryArcs = Array.from( + new Map( + flattenedStoryArcs.map( + (arc: any) => [arc.id, arc] + ) + ).values() + ); + + console.log( + `Found ${uniqueStoryArcs.length} unique story arc(s) for volume ID ${volumeId}:` + ); + uniqueStoryArcs.forEach((arc: any) => { + console.log( + `- ${arc.name} (ID: ${arc.id}) with issueDescription and issueImage` + ); + }); + + return uniqueStoryArcs; + } catch (error) { + console.error( + "An error occurred while processing story arcs:", + error + ); + } + } else { + console.log( + "No issues found for the specified volume." + ); + } + } catch (error) { + console.error( + "An error occurred while fetching data from ComicVine:", + error + ); + } + }, + }, + + getIssuesForVolume: { + rest: "POST /getIssuesForVolume", + async handler(ctx: Context<{ volumeId: number }>) { + const { volumeId } = ctx.params; + const issuesUrl = `${CV_BASE_URL}issues/?api_key=${process.env.COMICVINE_API_KEY}`; + try { + const response = await axios.get(issuesUrl, { + params: { + api_key: process.env.COMICVINE_API_KEY, + filter: `volume:${volumeId}`, + format: "json", + field_list: + "id,name,image,issue_number,cover_date,description", + limit: 100, + }, + headers: { + Accept: "application/json", + "User-Agent": "ThreeTwo", + }, + }); + + // Map over the issues to include the year extracted from cover_date + const issuesWithDescriptionImageAndYear = + response.data.results.map((issue: any) => { + const year = issue.cover_date + ? new Date( + issue.cover_date + ).getFullYear() + : null; // Extract the year from cover_date + return { + ...issue, + year: year, + description: issue.description || "", + image: issue.image || {}, + }; + }); + + return issuesWithDescriptionImageAndYear; + } catch (error) { + this.logger.error( + "Error fetching issues from ComicVine:", + error.message + ); + throw new MoleculerError( + "Failed to fetch issues", + 500, + "FETCH_ERROR", + { error: error.message } + ); + } + }, + }, }, methods: { fetchVolumesFromCV: async (payload, output: any[] = []) => { diff --git a/services/metron.service.ts b/services/metron.service.ts index 2476206..ec53b50 100644 --- a/services/metron.service.ts +++ b/services/metron.service.ts @@ -1,7 +1,7 @@ "use strict"; -import { Service, ServiceBroker, Context } from "moleculer"; import axios from "axios"; +import { Context, Service, ServiceBroker } from "moleculer"; const METRON_BASE_URL = "https://metron.cloud/api/"; @@ -24,7 +24,7 @@ export default class MetronService extends Service { }; }> ) => { - console.log(ctx.params); + console.log(ctx.params); const results = await axios({ method: "GET", url: `https://metron.cloud/api/${ctx.params.resource}`, @@ -32,7 +32,14 @@ export default class MetronService extends Service { name: ctx.params.query.name, page: ctx.params.query.page, }, - + headers: { + "Authorization": "Basic ZnJpc2hpOlRpdHVAMTU4OA==" + }, + auth: { + "username": "frishi", + "password": "Titu@1588" + } + }); return results.data; }, diff --git a/utils/scraping.utils.ts b/utils/scraping.utils.ts index 51246c6..a244168 100644 --- a/utils/scraping.utils.ts +++ b/utils/scraping.utils.ts @@ -55,3 +55,34 @@ export const scrapeIssuePage = async (url: string) => { .querySelector("div.series-pagination > a.series").getAttribute("href"); return seriesDOMElement; }; + + +export const getWeeklyPullList = async () => { + const url = "https://www.tfaw.com/comics/new-releases.html"; + const response = await axios(url); + const dom = new JSDOM(response.data, { + url, + referrer: url, + contentType: "text/html", + includeNodeLocations: true, + storageQuota: 10000000, + }); + + const pullList: any[] = []; + // Node for the comics container + const issueNodes = dom.window.document.querySelectorAll("ol.products > li"); + + issueNodes.forEach(node => { + const coverImageUrl = node.querySelector("img.photo").getAttribute("data-src"); + const name = node.querySelector("div.product > a.product").textContent.trim(); + const publicationDate = node.querySelector("div.product-item-date").textContent.trim(); + pullList.push({ + coverImageUrl, + name, + publicationDate, + }); + }); + + return pullList; + +}; diff --git a/utils/searchmatchscorer.utils.ts b/utils/searchmatchscorer.utils.ts index c847ebf..a6f96fc 100644 --- a/utils/searchmatchscorer.utils.ts +++ b/utils/searchmatchscorer.utils.ts @@ -31,7 +31,7 @@ SOFTWARE. * Initial: 2021/07/29 Rishi Ghan */ -import { createWriteStream } from "fs"; +import { createWriteStream, existsSync, mkdirSync } from "fs"; import path from "path"; import https from "https"; import stringSimilarity from "string-similarity"; @@ -151,6 +151,10 @@ const calculateLevenshteinDistance = async (match: any, rawFileDetails: any) => https.get(match.image.small_url, (response: any) => { console.log(rawFileDetails.cover.filePath); const fileName = match.id + "_" + rawFileDetails.name + ".jpg"; + // Ensure the `temporary` directory exists + if (!existsSync("temporary")) { + mkdirSync("temporary", { recursive: true }); + } const file = createWriteStream( `${process.env.USERDATA_DIRECTORY}/temporary/${fileName}` );