From d9364679a51ff80db8e5948ab089d749da36a6b2 Mon Sep 17 00:00:00 2001 From: Geo Halkiadakis Date: Wed, 17 Apr 2024 18:32:13 +0300 Subject: dockerize the app --- app/utils/kb-util.js | 88 +++++++++++++++++++++++++++ app/utils/match-util.js | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ app/utils/mem-usage.js | 20 +++++++ app/utils/url-util.js | 23 ++++++++ 4 files changed, 285 insertions(+) create mode 100644 app/utils/kb-util.js create mode 100644 app/utils/match-util.js create mode 100644 app/utils/mem-usage.js create mode 100644 app/utils/url-util.js (limited to 'app/utils') diff --git a/app/utils/kb-util.js b/app/utils/kb-util.js new file mode 100644 index 0000000..aac88f9 --- /dev/null +++ b/app/utils/kb-util.js @@ -0,0 +1,88 @@ +/** + * fast string manipulation utilities + * for bi-lingual (EL/EN) words/phrases + * based on the keyboard layout + */ + +// suplamentary arrays (mostly for cache) +// --- -- -- - - - + +var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + +var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + +var map = new Map(); +for (var i=0; i { + ap = pair.split(' '); + accented_vowels.push({ + a: ap[0], // accented + p: ap[1] // pure = non accended + }); +}); + + +// translates string to keyboard-latin keys +// (the ones that used whan typing each letter of the word) +const keyboardize = (str) => { + str = str.replace('\'',''); + var out = ''; + // [map]'s implementation is 40x faster than [for]'s + for (var i=0 ; i< str.length; i++) out += map.get(str[i]); + return out; +} + +// keyboardize an array of strings +const keyb_array = (arr) => { + kb_arr = []; + arr.forEach( w => { + kb_arr.push(keyboardize(w)); + }); + return kb_arr; +} + + +// transforms to lowercase; handles sigma-teliko +const sanitizeGR = (str) => { + str = str.toLowerCase(); + + // replace accended vowels with pure ones + accented_vowels.forEach( v => { + str = str.replaceAll(v.a, v.p); + }); + + // replace sigma on the end of words + str = str + ' '; + str = str.replaceAll('σ-', 'ς-'); + str = str.replaceAll('σ ', 'ς '); + + return str; +} + +// removes non keyword characters [+ . , !] and internal multiple-spaces +// @param txt (string): product description +const clean = (txt) => { + return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space + .replace('!','').replace('\"', '') // remove character + .replace(' ',' ').replace(' ',' '); // remove multiple spaces +} + + +// exports +// --- -- -- - - - + +module.exports = { + keyboardize, + keyb_array, + sanitizeGR, + clean +}; \ No newline at end of file diff --git a/app/utils/match-util.js b/app/utils/match-util.js new file mode 100644 index 0000000..5e8c648 --- /dev/null +++ b/app/utils/match-util.js @@ -0,0 +1,154 @@ +/** + * match utility; + * includes fuzzy and partial match functions too; + * many of them return a match-rate + */ + + +// fuzzy match +//////////////////////////////////////////////////////////////////////////////// + +/** Ngram fuzzy match algorithm + * (simple and fast) + */ +const createNgram = (word, n) => { // Ngram creation + if (word.length <3) return word; + const vector = []; + for (let i = 0; i < word.length-n+1; ++i) { + vector.push(word.slice(i, i + n)); + } + return vector; +}; + +/** similarity + * rates similarity between 2 words + * based on Ngram matches of N = n letters; + * implements a 2-dim check (all a-Ngrams vs all all b-Ngrams) + * + * @param {string} a : first word + * @param {string} b : second word + * @param {int} n : Ngram base + * @returns {float} : match percentage as a float in [0, 1] + */ +const similarity = (a, b, n) => { // Ngram match score + if (a.length > 0 && b.length > 0) { + const aNgram = createNgram(a, n); + const bNgram = createNgram(b, n); + let hits = 0; + for (let x = 0; x < aNgram.length; ++x) { + for (let y = 0; y < bNgram.length; ++y) { + if (aNgram[x] === bNgram[y]) { + hits += 1; + } + } + } + if (hits > 0) { + const union = aNgram.length + bNgram.length; + return (2.0 * hits) / union; + } + } + return 0; +}; + +/** resemblance + * is an alternative similarity rating; + * implements an 1-dim Ngram similarity check + * and it's much faster than similarity() + */ +const resemblance = (a, b, n) => { + if (a.length > n && b.length >= a.length) { + const aNgram = createNgram(a, n); + let hits = 0; + for (let i = 0; i < aNgram.length; ++i) { + if (b.includes(aNgram[i])) { + hits++; + } + } + if (hits > 0) { + // rate resemblance based on hits and length-similarity + return (hits / aNgram.length) * (a.length / b.length); + } + } + return 0; +} + + +// exact and partial match +//////////////////////////////////////////////////////////////////////////////// + +/** is_exact_match + * check if a searching string -> query (string/latin in kb-format) + * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) + * + * @param {string} query: searching string; string/latin in kb-format + * @param {array} chkArr: array of synonyms; (array of utf-8/strings) + * @return {boolean}: true|false + */ +function exact( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (w == query) found = true }); + return found; +} + +function partial( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (w.includes(query)) found = true }); + return found; +} + +/** is exact match + weight rating + * @returns {float} weight rates depth of array when a match is found + */ +function weighted_exact( query, chkArr ) { + let weight = 0; // closer to left/begin rating + let len = chkArr.length; + for(let i = 0; i < len ; i++) { // i ~ depth + if (chkArr[i] == query) { + // weights array depth + weight = (len - i + 1.0) / len; + break; + } + } + return weight; +} + + +/** is partial match + weight rating + * + * @param query (string): searching string; string/latin in kb-format + * @param chkArr (array): array of synonyms; (array of utf-8/strings) + * @returns {float} weight rates both match position and depth of match + * + * (*) optimization NOTE: + * Given the weight `W` and the depth `i`, + * the best weight for next `i` shall be: `(L - (i+1)) / L` + * To be imposibbe to have a better weight, should: + * W > (L - (i+1)) / L => ... => i > (L - L*W - 1) + */ +function weighted_partial( query, chkArr ) { + let rate = 0; + let weight = 0; + let len = chkArr.length; + for( let i = 0 ; i < len ; i++ ) { + let chk = chkArr[i].indexOf(query) + if (chk != -1) { + rate = (len - i) / (len + 2.0 * chk); + weight = rate > weight ? rate : weight; + } + if (i > (len - len * weight - 1)) { + break; // better rating is not possible (*) + } + } + return weight; +} + + +// exports +module.exports = { + exact, + partial, + weighted_exact, + weighted_partial, + similarity, + resemblance +} diff --git a/app/utils/mem-usage.js b/app/utils/mem-usage.js new file mode 100644 index 0000000..766a93e --- /dev/null +++ b/app/utils/mem-usage.js @@ -0,0 +1,20 @@ +/** + * memory usage report utility + */ + +const formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`; + +function report() { + let memoryData = process.memoryUsage(); + + let memoryUsage = { + rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`, + heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`, + heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`, + external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`, + }; + + console.log(memoryUsage); +} + +module.exports = { report } diff --git a/app/utils/url-util.js b/app/utils/url-util.js new file mode 100644 index 0000000..e86ad9b --- /dev/null +++ b/app/utils/url-util.js @@ -0,0 +1,23 @@ +/** + * url utility + */ + +const querystring = require('querystring'); + +function struct(req, url) { + let url_parts = url.split('?'); + let query = (url_parts.length > 1) + ? querystring.decode(url_parts[1]) + : {}; + return { + method: req.method, + host: req.host, + path: url_parts[0], + query: query + } +} + +/** + * exports + */ +module.exports = { struct } \ No newline at end of file -- cgit v1.2.3