diff --git a/services/comicvine.service.ts b/services/comicvine.service.ts index 5d3eb03..21eec21 100644 --- a/services/comicvine.service.ts +++ b/services/comicvine.service.ts @@ -176,7 +176,7 @@ export default class ComicVineService extends Service { field_list: `${fieldList}`, }, headers: { - Accept: "application/json", + "Accept": "application/json", "User-Agent": "ThreeTwo", }, }); @@ -279,7 +279,7 @@ export default class ComicVineService extends Service { filter: filterString, }, headers: { - Accept: "application/json", + "Accept": "application/json", "User-Agent": "ThreeTwo", }, }); @@ -335,7 +335,7 @@ export default class ComicVineService extends Service { rest: "POST /getComicVineMatchScores", handler: async ( ctx: Context<{ - finalMatches: Array; + finalMatches: any[]; rawFileDetails: any; scorerConfiguration: any; }> @@ -372,7 +372,7 @@ export default class ComicVineService extends Service { resources: "volumes", }, headers: { - Accept: "application/json", + "Accept": "application/json", "User-Agent": "ThreeTwo", }, }); @@ -391,7 +391,7 @@ export default class ComicVineService extends Service { format: "json", }, headers: { - Accept: "application/json", + "Accept": "application/json", "User-Agent": "ThreeTwo", }, @@ -483,7 +483,7 @@ export default class ComicVineService extends Service { limit: 100, }, headers: { - Accept: "application/json", + "Accept": "application/json", "User-Agent": "ThreeTwo", }, }); @@ -498,7 +498,7 @@ export default class ComicVineService extends Service { : null; // Extract the year from cover_date return { ...issue, - year: year, + year, description: issue.description || "", image: issue.image || {}, }; @@ -538,7 +538,7 @@ export default class ComicVineService extends Service { resources, }, headers: { - Accept: "application/json", + "Accept": "application/json", "User-Agent": "ThreeTwo", }, }); diff --git a/utils/scraping.utils.ts b/utils/scraping.utils.ts index bc53790..b81ae4d 100644 --- a/utils/scraping.utils.ts +++ b/utils/scraping.utils.ts @@ -36,8 +36,8 @@ export const getWeeklyPullList = async (url: string) => { try { await page.goto(url, { - waitUntil: "domcontentloaded", // faster and more reliable for JS-rendered content - timeout: 30000, // give it time on Tor or slow networks + waitUntil: "domcontentloaded", // Faster and more reliable for JS-rendered content + timeout: 30000, // Give it time on Tor or slow networks }); await page.waitForSelector(".issue", { timeout: 30000 }); @@ -96,45 +96,6 @@ export const getWeeklyPullList = async (url: string) => { } }; -// export const scrapeIssuesFromSeriesPage = async (url: string) => { -// const response = await axios(url); -// const dom = new JSDOM(response.data, { -// url, -// referrer: url, -// contentType: "text/html", -// includeNodeLocations: true, -// storageQuota: 10000000, -// }); -// const seriesId = dom.window.document -// .querySelector("#comic-list-block") -// .getAttribute("data-series-id"); -// const issueNodes = dom.window.document.querySelectorAll( -// "ul.comic-list-thumbs > li" -// ); - -// const issues: any = []; -// issueNodes.forEach(node => { -// const comicHref = node.querySelector("a").getAttribute("href"); -// const issueCoverImage = node.querySelector("img").getAttribute("src"); -// const issueDetails = node.querySelector("img").getAttribute("alt"); -// const issueDate = node.querySelector("span.date").getAttribute("data-date"); -// const formattedIssueDate = node.querySelector("span.date").textContent.trim(); -// const publisher = node.querySelector("div.publisher").textContent.trim(); - -// issues.push({ -// comicHref, -// issueCoverImage, -// issueDetails, -// issueDate, -// formattedIssueDate, -// publisher, -// }); -// }); -// return { -// seriesId, -// issues, -// }; -// }; export const scrapeIssuePage = async (url: string) => { const response = await axios(url);