diff --git a/package-lock.json b/package-lock.json index 709eb94..90f2bff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,8 @@ "@types/mkdirp": "^1.0.0", "@types/node": "^13.9.8", "axios": "^0.21.1", + "axios-extensions": "^3.1.3", + "date-fns": "^2.27.0", "dotenv": "^10.0.0", "imghash": "^0.0.9", "ioredis": "^4.28.1", @@ -1536,6 +1538,11 @@ "integrity": "sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==", "dev": true }, + "node_modules/@types/lru-cache": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz", + "integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw==" + }, "node_modules/@types/mkdirp": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz", @@ -2023,6 +2030,33 @@ "follow-redirects": "^1.14.0" } }, + "node_modules/axios-extensions": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/axios-extensions/-/axios-extensions-3.1.3.tgz", + "integrity": "sha512-/OB9OcJLNOIx9pdW4m4/hFRvNo12wlX5BaprIzqpMaLR02I88Mr98/wW4QN9rhx0/yg9rM7i6Af/RpV4MyxXjA==", + "dependencies": { + "@types/lru-cache": "^4.1.1", + "lru-cache": "^5.1.1", + "tslib": "^1.9.0", + "util": "^0.11.1" + }, + "peerDependencies": { + "axios": "*" + } + }, + "node_modules/axios-extensions/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/axios-extensions/node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + }, "node_modules/babel-jest": { "version": "25.5.1", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-25.5.1.tgz", @@ -2998,6 +3032,18 @@ "whatwg-url": "^7.0.0" } }, + "node_modules/date-fns": { + "version": "2.27.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.27.0.tgz", + "integrity": "sha512-sj+J0Mo2p2X1e306MHq282WS4/A8Pz/95GIFcsPNMPMZVI3EUrAdSv90al1k+p74WGLCruMXk23bfEDZa71X9Q==", + "engines": { + "node": ">=0.11" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/date-fns" + } + }, "node_modules/debug": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", @@ -11092,8 +11138,7 @@ "node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/tsutils": { "version": "3.21.0", @@ -11322,12 +11367,25 @@ "node": ">=0.10.0" } }, + "node_modules/util": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", + "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "dependencies": { + "inherits": "2.0.3" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", "dev": true }, + "node_modules/util/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, "node_modules/uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", @@ -13019,6 +13077,11 @@ "integrity": "sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==", "dev": true }, + "@types/lru-cache": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz", + "integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw==" + }, "@types/mkdirp": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz", @@ -13366,6 +13429,32 @@ "follow-redirects": "^1.14.0" } }, + "axios-extensions": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/axios-extensions/-/axios-extensions-3.1.3.tgz", + "integrity": "sha512-/OB9OcJLNOIx9pdW4m4/hFRvNo12wlX5BaprIzqpMaLR02I88Mr98/wW4QN9rhx0/yg9rM7i6Af/RpV4MyxXjA==", + "requires": { + "@types/lru-cache": "^4.1.1", + "lru-cache": "^5.1.1", + "tslib": "^1.9.0", + "util": "^0.11.1" + }, + "dependencies": { + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "requires": { + "yallist": "^3.0.2" + } + }, + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + } + } + }, "babel-jest": { "version": "25.5.1", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-25.5.1.tgz", @@ -14155,6 +14244,11 @@ "whatwg-url": "^7.0.0" } }, + "date-fns": { + "version": "2.27.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.27.0.tgz", + "integrity": "sha512-sj+J0Mo2p2X1e306MHq282WS4/A8Pz/95GIFcsPNMPMZVI3EUrAdSv90al1k+p74WGLCruMXk23bfEDZa71X9Q==" + }, "debug": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", @@ -20344,8 +20438,7 @@ "tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "tsutils": { "version": "3.21.0", @@ -20524,6 +20617,21 @@ "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", "dev": true }, + "util": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", + "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "requires": { + "inherits": "2.0.3" + }, + "dependencies": { + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + } + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index 79f1f7e..b7c30f5 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,8 @@ "@types/mkdirp": "^1.0.0", "@types/node": "^13.9.8", "axios": "^0.21.1", + "axios-extensions": "^3.1.3", + "date-fns": "^2.27.0", "dotenv": "^10.0.0", "imghash": "^0.0.9", "ioredis": "^4.28.1", diff --git a/services/comicvine.service.ts b/services/comicvine.service.ts index e5697c9..9e99140 100644 --- a/services/comicvine.service.ts +++ b/services/comicvine.service.ts @@ -2,8 +2,9 @@ import qs from "querystring"; import { Service, ServiceBroker, Context } from "moleculer"; -import axios, { AxiosResponse } from "axios"; -import { matchScorer } from "../utils/searchmatchscorer.utils"; +import axios from "axios"; +import { cacheAdapterEnhancer, throttleAdapterEnhancer } from "axios-extensions"; +import { matchScorer, rankVolumes } from "../utils/searchmatchscorer.utils"; const CV_BASE_URL = "https://comicvine.gamespot.com/api/"; console.log("KEYYYYYYYY", process.env.COMICVINE_API_KEY); @@ -176,7 +177,8 @@ export default class ComicVineService extends Service { "?api_key=" + process.env.COMICVINE_API_KEY, params, - headers: { Accept: "application/json" }, + headers: { Accept: "application/json"}, + adapter: throttleAdapterEnhancer(cacheAdapterEnhancer(axios.defaults.adapter)), }); const { data } = response; @@ -187,6 +189,10 @@ export default class ComicVineService extends Service { ); if (currentPage < totalPages) { output.push(...data.results); + + // 1a. Run the current batch of volumes through the matcher + // Check for: issue year falling in the range of the volume run + rankVolumes(output, params.scorerConfiguration); currentPage += 1; params.page = currentPage; console.log(`Fetching results for page ${currentPage}...`); @@ -199,12 +205,4 @@ export default class ComicVineService extends Service { }); } - // Action - public ActionHello(): string { - return "Hello Moleculer"; - } - - public ActionWelcome(name: string): string { - return `Welcome, ${name}`; - } } diff --git a/utils/searchmatchscorer.utils.ts b/utils/searchmatchscorer.utils.ts index 8445b3a..a1b40ea 100644 --- a/utils/searchmatchscorer.utils.ts +++ b/utils/searchmatchscorer.utils.ts @@ -37,6 +37,7 @@ import https from "https"; import stringSimilarity from "string-similarity"; import { isNil, map, isUndefined } from "lodash"; import leven from "leven"; +import { isAfter, isSameYear, parseISO } from "date-fns"; const imghash = require("imghash"); @@ -84,6 +85,38 @@ export const matchScorer = async ( return Promise.all(scoredMatches); }; +export const rankVolumes = (volumes: any, scorerConfiguration: any) => { + // Iterate over volumes, checking to see: + // 1. If the detected year of the issue falls in the range (end_year >= {detected year for issue} >= start_year ) + // 2. If there is a strong string comparison between the volume name and the issue name ?? + const issueNumber = parseInt( + scorerConfiguration.searchParams.searchTerms.number, + 10 + ); + let count = 0; + const issueYear = parseISO( + scorerConfiguration.searchParams.searchTerms.year + ); + volumes.map(async (volume: any, idx: number) => { + const volumeStartYear = parseISO(volume.start_year); + const firstIssueNumber = parseInt(volume.first_issue.issue_number, 10); + const lastIssueNumber = parseInt(volume.last_issue.issue_number, 10); + const issueNameMatchScore = stringSimilarity.compareTwoStrings(scorerConfiguration.searchParams.searchTerms.name, volume.name); + if ( + (isSameYear(issueYear, volumeStartYear) || + isAfter(issueYear, volumeStartYear)) && + (firstIssueNumber <= issueNumber && + issueNumber <= lastIssueNumber) && + issueNameMatchScore > 0.5 + ) { + count += 1; + console.log("issue name match score", issueNameMatchScore); + console.log(volume); + } + }); + console.log("final count ", count); +}; + const calculateLevenshteinDistance = async (match: any, rawFileDetails: any) => new Promise((resolve, reject) => { https.get(match.image.small_url, (response: any) => { @@ -113,7 +146,7 @@ const calculateLevenshteinDistance = async (match: any, rawFileDetails: any) => } resolve(match); } else { - reject({ error: "bastard couldn't calculate hashes" }); + reject({ error: "Couldn't calculate hashes." }); } }); });