frishi/threetwo-metadata-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🤼 Volume matching and filtering WIP

Browse Source
This commit is contained in:
Rishi Ghan
2021-12-23 12:54:48 -08:00
parent f66f0f283e
commit 71223fb8ed
4 changed files with 157 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
package-lock.json generated
View File

@@ -13,6 +13,8 @@
"@types/mkdirp": "^1.0.0",
"@types/node": "^13.9.8",
"axios": "^0.21.1",
"axios-extensions": "^3.1.3",
"date-fns": "^2.27.0",
"dotenv": "^10.0.0",
"imghash": "^0.0.9",
"ioredis": "^4.28.1",
@@ -1536,6 +1538,11 @@
"integrity": "sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==",
"dev": true
},
"node_modules/@types/lru-cache": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz",
"integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw=="
},
"node_modules/@types/mkdirp": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz",
@@ -2023,6 +2030,33 @@
"follow-redirects": "^1.14.0"
}
},
"node_modules/axios-extensions": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/axios-extensions/-/axios-extensions-3.1.3.tgz",
"integrity": "sha512-/OB9OcJLNOIx9pdW4m4/hFRvNo12wlX5BaprIzqpMaLR02I88Mr98/wW4QN9rhx0/yg9rM7i6Af/RpV4MyxXjA==",
"dependencies": {
"@types/lru-cache": "^4.1.1",
"lru-cache": "^5.1.1",
"tslib": "^1.9.0",
"util": "^0.11.1"
},
"peerDependencies": {
"axios": "*"
}
},
"node_modules/axios-extensions/node_modules/lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
"dependencies": {
"yallist": "^3.0.2"
}
},
"node_modules/axios-extensions/node_modules/yallist": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
},
"node_modules/babel-jest": {
"version": "25.5.1",
"resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-25.5.1.tgz",
@@ -2998,6 +3032,18 @@
"whatwg-url": "^7.0.0"
}
},
"node_modules/date-fns": {
"version": "2.27.0",
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.27.0.tgz",
"integrity": "sha512-sj+J0Mo2p2X1e306MHq282WS4/A8Pz/95GIFcsPNMPMZVI3EUrAdSv90al1k+p74WGLCruMXk23bfEDZa71X9Q==",
"engines": {
"node": ">=0.11"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/date-fns"
}
},
"node_modules/debug": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz",
@@ -11092,8 +11138,7 @@
"node_modules/tslib": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
"dev": true
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
},
"node_modules/tsutils": {
"version": "3.21.0",
@@ -11322,12 +11367,25 @@
"node": ">=0.10.0"
}
},
"node_modules/util": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz",
"integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==",
"dependencies": {
"inherits": "2.0.3"
}
},
"node_modules/util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
"dev": true
},
"node_modules/util/node_modules/inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
},
"node_modules/uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
@@ -13019,6 +13077,11 @@
"integrity": "sha512-/BHF5HAx3em7/KkzVKm3LrsD6HZAXuXO1AJZQ3cRRBZj4oHZDviWPYu0aEplAqDFNHZPW6d3G7KN+ONcCCC7pw==",
"dev": true
},
"@types/lru-cache": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/@types/lru-cache/-/lru-cache-4.1.3.tgz",
"integrity": "sha512-QjCOmf5kYwekcsfEKhcEHMK8/SvgnneuSDXFERBuC/DPca2KJIO/gpChTsVb35BoWLBpEAJWz1GFVEArSdtKtw=="
},
"@types/mkdirp": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@types/mkdirp/-/mkdirp-1.0.2.tgz",
@@ -13366,6 +13429,32 @@
"follow-redirects": "^1.14.0"
}
},
"axios-extensions": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/axios-extensions/-/axios-extensions-3.1.3.tgz",
"integrity": "sha512-/OB9OcJLNOIx9pdW4m4/hFRvNo12wlX5BaprIzqpMaLR02I88Mr98/wW4QN9rhx0/yg9rM7i6Af/RpV4MyxXjA==",
"requires": {
"@types/lru-cache": "^4.1.1",
"lru-cache": "^5.1.1",
"tslib": "^1.9.0",
"util": "^0.11.1"
},
"dependencies": {
"lru-cache": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
"integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
"requires": {
"yallist": "^3.0.2"
}
},
"yallist": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
}
}
},
"babel-jest": {
"version": "25.5.1",
"resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-25.5.1.tgz",
@@ -14155,6 +14244,11 @@
"whatwg-url": "^7.0.0"
}
},
"date-fns": {
"version": "2.27.0",
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.27.0.tgz",
"integrity": "sha512-sj+J0Mo2p2X1e306MHq282WS4/A8Pz/95GIFcsPNMPMZVI3EUrAdSv90al1k+p74WGLCruMXk23bfEDZa71X9Q=="
},
"debug": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz",
@@ -20344,8 +20438,7 @@
"tslib": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
"dev": true
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
},
"tsutils": {
"version": "3.21.0",
@@ -20524,6 +20617,21 @@
"integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==",
"dev": true
},
"util": {
"version": "0.11.1",
"resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz",
"integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==",
"requires": {
"inherits": "2.0.3"
},
"dependencies": {
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
}
}
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",

2
package.json
View File

@@ -40,6 +40,8 @@
"@types/mkdirp": "^1.0.0",
"@types/node": "^13.9.8",
"axios": "^0.21.1",
"axios-extensions": "^3.1.3",
"date-fns": "^2.27.0",
"dotenv": "^10.0.0",
"imghash": "^0.0.9",
"ioredis": "^4.28.1",

20
services/comicvine.service.ts
View File

@@ -2,8 +2,9 @@
import qs from "querystring";
import { Service, ServiceBroker, Context } from "moleculer";
import axios, { AxiosResponse } from "axios";
import { matchScorer } from "../utils/searchmatchscorer.utils";
import axios from "axios";
import { cacheAdapterEnhancer, throttleAdapterEnhancer } from "axios-extensions";
import { matchScorer, rankVolumes } from "../utils/searchmatchscorer.utils";
const CV_BASE_URL = "https://comicvine.gamespot.com/api/";
console.log("KEYYYYYYYY", process.env.COMICVINE_API_KEY);
@@ -176,7 +177,8 @@ export default class ComicVineService extends Service {
"?api_key=" +
process.env.COMICVINE_API_KEY,
params,
headers: { Accept: "application/json" },
headers: { Accept: "application/json"},
adapter: throttleAdapterEnhancer(cacheAdapterEnhancer(axios.defaults.adapter)),
});
const { data } = response;
@@ -187,6 +189,10 @@ export default class ComicVineService extends Service {
);
if (currentPage < totalPages) {
output.push(...data.results);
// 1a. Run the current batch of volumes through the matcher
// Check for: issue year falling in the range of the volume run
rankVolumes(output, params.scorerConfiguration);
currentPage += 1;
params.page = currentPage;
console.log(`Fetching results for page ${currentPage}...`);
@@ -199,12 +205,4 @@ export default class ComicVineService extends Service {
});
}
// Action
public ActionHello(): string {
return "Hello Moleculer";
}
public ActionWelcome(name: string): string {
return `Welcome, ${name}`;
}
}

35
utils/searchmatchscorer.utils.ts
View File

@@ -37,6 +37,7 @@ import https from "https";
import stringSimilarity from "string-similarity";
import { isNil, map, isUndefined } from "lodash";
import leven from "leven";
import { isAfter, isSameYear, parseISO } from "date-fns";
const imghash = require("imghash");
@@ -84,6 +85,38 @@ export const matchScorer = async (
return Promise.all(scoredMatches);
};
export const rankVolumes = (volumes: any, scorerConfiguration: any) => {
// Iterate over volumes, checking to see:
// 1. If the detected year of the issue falls in the range (end_year >= {detected year for issue} >= start_year )
// 2. If there is a strong string comparison between the volume name and the issue name ??
const issueNumber = parseInt(
scorerConfiguration.searchParams.searchTerms.number,
10
);
let count = 0;
const issueYear = parseISO(
scorerConfiguration.searchParams.searchTerms.year
);
volumes.map(async (volume: any, idx: number) => {
const volumeStartYear = parseISO(volume.start_year);
const firstIssueNumber = parseInt(volume.first_issue.issue_number, 10);
const lastIssueNumber = parseInt(volume.last_issue.issue_number, 10);
const issueNameMatchScore = stringSimilarity.compareTwoStrings(scorerConfiguration.searchParams.searchTerms.name, volume.name);
if (
(isSameYear(issueYear, volumeStartYear) ||
isAfter(issueYear, volumeStartYear)) &&
(firstIssueNumber <= issueNumber &&
issueNumber <= lastIssueNumber) &&
issueNameMatchScore > 0.5
) {
count += 1;
console.log("issue name match score", issueNameMatchScore);
console.log(volume);
}
});
console.log("final count ", count);
};
const calculateLevenshteinDistance = async (match: any, rawFileDetails: any) =>
new Promise((resolve, reject) => {
https.get(match.image.small_url, (response: any) => {
@@ -113,7 +146,7 @@ const calculateLevenshteinDistance = async (match: any, rawFileDetails: any) =>
}
resolve(match);
} else {
reject({ error: "bastard couldn't calculate hashes" });
reject({ error: "Couldn't calculate hashes." });
}
});
});