From b122279ece06a9381e805c6087f130b41616fe3d Mon Sep 17 00:00:00 2001 From: Geo Halkiadakis Date: Fri, 12 Apr 2024 18:31:00 +0300 Subject: added utils folder; added several suplamentary modules --- .gitignore | 1 + README.md | 4 + app.js | 5 +- benchmark/find.js | 27 ++-- benchmark/match-str.js | 22 ++-- package-lock.json | 12 ++ package.json | 1 + paths.js | 47 ++++--- pieces/kb-util.js | 84 ------------- pieces/match-util.js | 92 -------------- pieces/prepare-streams.js | 5 +- pieces/retro-search.js | 304 ++++++++++++++++++++++++++++++++++++++++++++++ pieces/search.js | 303 --------------------------------------------- pieces/suggest.js | 4 +- utils/kb-util.js | 88 ++++++++++++++ utils/match-util.js | 92 ++++++++++++++ utils/mem-usage.js | 20 +++ utils/url-util.js | 23 ++++ 18 files changed, 618 insertions(+), 516 deletions(-) delete mode 100644 pieces/kb-util.js delete mode 100644 pieces/match-util.js create mode 100644 pieces/retro-search.js delete mode 100644 pieces/search.js create mode 100644 utils/kb-util.js create mode 100644 utils/match-util.js create mode 100644 utils/mem-usage.js create mode 100644 utils/url-util.js diff --git a/.gitignore b/.gitignore index 3f33deb..1575068 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules/ data/* +.env diff --git a/README.md b/README.md index e550493..477d0cf 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,7 @@ run npm start +prepare source for production: + +* remove ``/bench`` routes from ``paths.js`` +* remove ``/bench`` folder diff --git a/app.js b/app.js index c0ad4f2..aa79f62 100644 --- a/app.js +++ b/app.js @@ -2,6 +2,8 @@ const Koa = require('koa'); const { koaBody } = require('koa-body'); +require('dotenv').config(); + const app = new Koa(); // middleware @@ -13,4 +15,5 @@ let paths = require('./paths.js'); // use the routes app.use(paths.routes()); -app.listen(3000); +// app.listen(3000); +app.listen(process.env.APP_PORT); diff --git a/benchmark/find.js b/benchmark/find.js index cc95a44..2d18657 100644 --- a/benchmark/find.js +++ b/benchmark/find.js @@ -1,5 +1,11 @@ +/** + * benchmark: find a product in product list + * using: for vs forEach vs find + */ + +const microtime = require('microtime'); + const products = require('../data/products.json'); -var microtime = require('microtime'); var selected = []; @@ -12,25 +18,32 @@ products.forEach( pr => { function compare() { + let n = 4; + var f0 = microtime.nowDouble(); - for(i=0 ; i < 4 ; i++) byFor(); + for(i=0 ; i < n ; i++) byFor(); var f1 = microtime.nowDouble(); // var e0 = microtime.nowDouble(); - for(i=0 ; i < 4 ; i++) byEach(); + for(i=0 ; i < n ; i++) byEach(); var e1 = microtime.nowDouble(); // var b0 = microtime.nowDouble(); - for(i=0 ; i < 4 ; i++) byFind(); + for(i=0 ; i < n ; i++) byFind(); var b1 = microtime.nowDouble(); return { + n: n, + for: f1-f0, - ifor: byFor(), + items_for: byFor(), + each: e1-e0, - ieach: byEach(), + items_each: byEach(), + find: b1-b0, - ifind: byFind(), + items_find: byFind(), + sel: selected, } } diff --git a/benchmark/match-str.js b/benchmark/match-str.js index 7760b4e..fa802c1 100644 --- a/benchmark/match-str.js +++ b/benchmark/match-str.js @@ -1,17 +1,23 @@ -const products = require('../data/products.json'); -const match = require('../pieces/match-util.js'); var microtime = require('microtime'); +const match = require('../utils/match-util.js'); +const memory_usage = require('../utils/mem-usage.js'); + +const products = require('../data/products.json'); -query = 'solokata'; +// memory_usage.report(); -function run() { +// test runner +function run(query) { var f0 = microtime.nowDouble(); - // for(let i = 0 ; i < 100 ; i++) - let result = matchQuery(query); + let result; + // for(let i = 0 ; i < 20 ; i++) + result = matchQuery(query); var f1 = microtime.nowDouble(); + memory_usage.report(); + return { t: f1-f0, q: query, @@ -19,7 +25,7 @@ function run() { } } - +// sumple search implementation function matchQuery(q) { var result = []; @@ -43,4 +49,4 @@ function matchQuery(q) { } -module.exports = { run } \ No newline at end of file +module.exports = { run } diff --git a/package-lock.json b/package-lock.json index 7b611b6..dea6a07 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "dotenv": "^16.4.5", "koa": "^2.15.2", "koa-body": "^6.0.1", "koa-router": "^12.0.1", @@ -447,6 +448,17 @@ "wrappy": "1" } }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", diff --git a/package.json b/package.json index 2b131cb..2ffe852 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "author": "Geo-Xalkiadakis@Sklavenitis-SA", "license": "ISC", "dependencies": { + "dotenv": "^16.4.5", "koa": "^2.15.2", "koa-body": "^6.0.1", "koa-router": "^12.0.1", diff --git a/paths.js b/paths.js index a2af434..6e5fbef 100644 --- a/paths.js +++ b/paths.js @@ -1,16 +1,24 @@ +/** + * defines routes + * exports router + */ + const Router = require('koa-router'); +const urler = require('./utils/url-util'); -const kb = require('./pieces/kb-util.js'); const data = require('./pieces/prepare-streams'); -const products = require('./data/products.json'); + const bench = require('./benchmark/find.js'); -const macthStr = require('./benchmark/match-str.js'); +const matchStr = require('./benchmark/match-str.js'); + // Prefix all routes with: /items const router = new Router({ // prefix: '/items' }); + + /* simple route example let items = [ @@ -50,7 +58,6 @@ const router = new Router({ // Routes - router.get('/', (ctx, next) => { ctx.body = { success: true, @@ -68,6 +75,7 @@ router.get('/search', (ctx, next) => { router.get('/search/:title', (ctx, next) => { // console.log(ctx); + const kb = require('./utils/kb-util.js'); let words = kb.keyboardize(kb.clean(ctx.params.title)).split(' '); ctx.body = { params: ctx.params, @@ -93,8 +101,7 @@ router.get('/search/:title', (ctx, next) => { router.get('/test', (ctx, next) => { // easy test route // test anything ... - let result = { success: true, data: [1, 2, 3] }; - ctx.body = result; + ctx.body = { params: urler.struct(ctx.request, ctx.url), ctx: ctx } next(); }); @@ -111,24 +118,30 @@ router.get('/test/do-data', (ctx, next) => { // easy test route next(); }); -router.get('/test/json', (ctx, next) => { // easy test route - // test anything ... - ctx.body = { t: + new Date(), p: products }; - next(); -}); +// benchmark routes; +// shall be removed from production +//////////////////////////////////////////////////////////////////////////////// + +/// router.get('/test/json', (ctx, next) => { // easy test route +/// // test anything ... +/// ctx.body = { t: + new Date(), p: products }; +/// next(); +/// }); -router.get('/bench/fe', (ctx, next) => { // easy test route +router.get('/bench/find', (ctx, next) => { // easy test route // test anything ... - // ctx.body = bench.compare(); ctx.body = bench.compare(); next(); }); -router.get('/bench/match', (ctx, next) => { // easy test route - // test anything ... +router.get('/bench/match/:title', (ctx, next) => { // easy test route + // test anything ... [ query = 'solokata' ] // ctx.body = bench.compare(); - ctx.body = macthStr.run(); + ctx.body = matchStr.run(ctx.params.title); next(); }); -module.exports = router; \ No newline at end of file + +// export routes + +module.exports = router; diff --git a/pieces/kb-util.js b/pieces/kb-util.js deleted file mode 100644 index 5441861..0000000 --- a/pieces/kb-util.js +++ /dev/null @@ -1,84 +0,0 @@ -// suplamentary arrays (mostly for cache) -// --- -- -- - - - - -var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); - -var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); - -var map = new Map(); -for (var i=0; i { - ap = pair.split(' '); - accented_vowels.push({ - a: ap[0], // accented - p: ap[1] // pure = non accended - }); -}); - - -// translates string to keyboard-latin keys -// (the ones that used whan typing each letter of the word) -const keyboardize = (str) => { - str = str.replace('\'',''); - var out = ''; - // [map]'s implementation is 40x faster than [for]'s - for (var i=0 ; i< str.length; i++) out += map.get(str[i]); - return out; -} - -// keyboardize an array of strings -const keyb_array = (arr) => { - kb_arr = []; - arr.forEach( w => { - kb_arr.push(keyboardize(w)); - }); - return kb_arr; -} - - -// transforms to lowercase; handles sigma-teliko -const sanitizeGR = (str) => { - str = str.toLowerCase(); - - // replace accended vowels with pure ones - accented_vowels.forEach( v => { - str = str.replaceAll(v.a, v.p); - }); - - // replace sigma on the end of words - str = str + ' '; - str = str.replaceAll('σ-', 'ς-'); - str = str.replaceAll('σ ', 'ς '); - - return str; -} - -// removes non keyword characters [+ . , !] and internal multiple-spaces -// @param txt (string): product description -const clean = (txt) => { - return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space - .replace('!','').replace('\"', '') // remove character - .replace(' ',' ').replace(' ',' '); // remove multiple spaces -} - - -// exports -// --- -- -- - - - - -module.exports = { - map, - accented_vowels, - keyboardize, - keyb_array, - sanitizeGR, - clean -}; \ No newline at end of file diff --git a/pieces/match-util.js b/pieces/match-util.js deleted file mode 100644 index a13fddb..0000000 --- a/pieces/match-util.js +++ /dev/null @@ -1,92 +0,0 @@ -/** Ngram fuzzy match algorithm - * (simple and fast) - */ -const createNgram = (word, n) => { // Ngram creation - if (word.length <3) return word; - const vector = []; - for (let i = 0; i < word.length-n+1; ++i) { - vector.push(word.slice(i, i + n)); - } - return vector; -}; - -/** similarity - * rates similarity between 2 words - * based on Ngram matches of N = n letters; - * implements a 2-dim check (all a-Ngrams vs all all b-Ngrams) - * - * @param {string} a : first word - * @param {string} b : second word - * @param {int} n : Ngram base - * @returns {float} : match percentage as a float in [0, 1] - */ -const similarity = (a, b, n) => { // Ngram match score - if (a.length > 0 && b.length > 0) { - const aNgram = createNgram(a, n); - const bNgram = createNgram(b, n); - let hits = 0; - for (let x = 0; x < aNgram.length; ++x) { - for (let y = 0; y < bNgram.length; ++y) { - if (aNgram[x] === bNgram[y]) { - hits += 1; - } - } - } - if (hits > 0) { - const union = aNgram.length + bNgram.length; - return (2.0 * hits) / union; - } - } - return 0; -}; - -/** resemblance - * is an alternative similarity rating; - * implements an 1-dim Ngram similarity check - * and it's much faster than similarity() - */ -const resemblance = (a, b, n) => { - if (a.length > n && b.length >= a.length) { - const aNgram = createNgram(a, n); - let hits = 0; - for (let i = 0; i < aNgram.length; ++i) { - if (b.includes(aNgram[i])) { - hits++; - } - } - if (hits > 0) { - // rate resemblance based on hits and length-similarity - return (hits / aNgram.length) * (a.length / b.length); - } - } - return 0; -} - - -/** is_exact_match - * - * check if a searching string -> query (string/latin in kb-format) - * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) - * - * @param query (string): searching string; string/latin in kb-format - * @param chkArr (array): array of synonyms; (array of utf-8/strings) - * @return (boolean): true|false - */ -function exact( query, chkArr ) { - found = false; - chkArr.forEach( w => { if (w == query) found = true }); - return found; -} - -function partial( query, chkArr ) { - found = false; - chkArr.forEach( w => { if (w.includes(query)) found = true }); - return found; -} - -module.exports = { - exact, - partial, - similarity, - resemblance -} diff --git a/pieces/prepare-streams.js b/pieces/prepare-streams.js index 4fa0a7e..ee7ba8e 100644 --- a/pieces/prepare-streams.js +++ b/pieces/prepare-streams.js @@ -1,8 +1,9 @@ const fs = require('fs'); -const kb = require('./kb-util.js'); - var request = require('request'); +const kb = require('../utils/kb-util.js'); + + // const https = require("https"); diff --git a/pieces/retro-search.js b/pieces/retro-search.js new file mode 100644 index 0000000..ba4d937 --- /dev/null +++ b/pieces/retro-search.js @@ -0,0 +1,304 @@ +const kb = require('../utils/kb-util.js'); +const match = require('../utils/match-util.js'); + +const products = require('../data/products.json'); + +/** VARIABLES + * may passed as module arguments + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + +var _allowFuzzy = true; // enable|disable fuzzy search +var n = 2; // Ngram base +var _fuzzyLimit = .5; // minimum bigram score for being considered a match +var max_list = 24; +var tolerance = 42; +var STORE = { id: 904 }; + +var _kwlinks; // keyword links (word-connections; imported via ajax-get) +var _products = []; // all products (imported via ajax-get) + +// setup options +var _maxResults = options.max_list; // limit suggestions +var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions +var _Ngram_base = 2; // number of N in Ngram spliting algorithm +var _isReady = false; // whether the searchbox is ready to be used + +// product keywords +var keywordsURL = options.keywords_json; + +var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } + + + + + + +/** SUPPLEMENTARY FUNCTIONS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + + +// callback function for sorting resulrs per r (=rating) property +function compare_rate(a,b) { + return (a.r < b.r); +} + + + +/** SEARCH ENGINE + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + + +function matchWordInList(q, list = false) { + + if (list !== false && list.length == 0) return []; // no results + + var result = []; + var firstPass = false; + + if (list === false) { + firstPass = true; // on first pass + list = products; // list is all products + } + + for(let i = 0 ; i < list.length ; i++) { + + // check exact + rate + + // else check partial + rate + + // else check similarity + rate + + let found = false; + let similarity = 0; + products[i].kb.split(' ').forEach( w => { + let sim = match.resemblance(q, w, 2); + if (sim > 0.5) { + found = true; + similarity = (sim > similarity) ? sim : similarity; + } + }); + if (found) { + products[i].similarity = similarity; + products[i].rating = (firstPass) + ? similarity * 2.0 + : products[i].rating + similarity * 2.0 + result.push(products[i]); + } + } + + return result +} + + + +// suggestions engine ////////////////////////////////////////////////// +// --- +function suggestions_engine(query) { + var results = []; + var pot = []; pot.length = 0; + + // clean and sanitize and mark links onto q(uery) string + var q = kb.keyboardize( kb.sanitizeGR( kb.clean(query.trim()) ) ).trim(); + + // TODO: + // construct direct-linked words + // = do unequivocally replaces + // steps: + // 1. replace accented vowels with non accented ones + // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'` + + var qAr = q.split(' '); // split to words + /// if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array + /// + /// if (qAr.slice(-1) == "") { + /// qAr.pop(); + /// } + + + pot = _products; // potential results // NOTE: CRITICAL: BY REFERENCE + + var wi = 0; // word index (from list) + var wc = qAr.length; + + qAr.forEach( w => { + + let sf = []; // (matches) so far + let mi; // position of match + wi++; + + pot.forEach( it => { + let matched = false; + let tester = ' '+ it.kb + ' '; + + // reset previous history and ratings + if (wi == 1) { + it.r = 0; + it.history = []; + } + + + // rate word-match > start-match > simple-match + // ... up to 8 points + + if (tester.indexOf(' '+ w +' ') != -1) { + it.r += 9; + it.history.push({ w: w, rate: 9 }); + matched = true; + } + else if (tester.indexOf(' '+ w) != -1) { + it.r += 5; + it.history.push({ w: w, rate: 5 }); + matched = true; + } + else if (tester.indexOf(w) != -1) { + it.r += 2; + it.history.push({ w: w, rate: 2 }); + matched = true; + } + + // rate `near-to-start` matching .. up to 7p + // rate `earlyness` of word in query .. up to 7p + + if ((mi = tester.indexOf(' '+w)) != -1) { + let fc1 = 100 - ((mi < 99) ? mi : 99); // near-to-start factor + let fc2 = wc - wi + 1; // query earlyness factor + let r1 = Math.floor(7*fc1/100); + let r2 = Math.floor(7*fc2/wc); + + it.r += (r1 + r2); + it.history.push({ w: w, left: [fc1, r1], early: [fc2, r2] }); + } + if (matched) sf.push(it); + }); + + if ((sf.length > (_maxResults + Math.floor(_maxResults/2))) + || (wi == 1) ) { + // ..if pot has a fair amount (= max + 50%) of results + // ..or these are results of '1st-query-word' + // set sf as new source + pot.lenght = 0; pot = []; + pot = JSON.parse(JSON.stringify(sf)); // copy by value + + } else { + // else.. keep the source list and increase of 'so-far rating' + // console.log('found small list', sf, pot) + pot.forEach( it => { + sf.forEach( si => { + if (it.id == si.id) { + it.r += 10; + it.history.push({ w: w, plus: '+10'}); + } + }); + }); + } + }); + + // sort results, get max-list of best rated + results = (pot.length > _maxResults) + ? pot.sort(compare_rate).slice(0, _maxResults) + : pot.sort(compare_rate) + + if (options.debug) console.log(results); + + return results; +} + + +// sub-module (start) +//////////////////////////////////////////////////////////////////////////// + +function update_common_search_results(q, results) { + let queries = getSessionObj('sr'); + let newSRlist = []; + let isnewQ = true; + if (queries === null) { + setSessionObj('sr', [{ + q: q, + result: result, + t: + new Date() + }]); + return true; + + } else { + + queries.forEach(it => { + if (it.q == q) { + newSRlist.push({ + q:q, + result: result, + t: + new Date() + }); + isnewQ = false; + } else { newSRlist.push(it); } + }); + + if (isnewQ) { + newSRlist.push({ + q:q, + result: result, + t: + new Date() + }); + } + + return true; + } +} + + + +//////////////////////////////////////////////////////////////////////////// +// sub-module (end) + + +function common_search(query) { + // clear ; sanitize ; split + var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' '); + + // if last item is empty, remove it + if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop() + + var results = _products; + + // for each key fitler results + qAr.forEach( key => { + results = key_sublist(key, results) + }); + + // echo products (and prepare list to POST) + var list_ = []; + results.forEach( item => { + if (options.debug) console.log(item.id, ':', item.w); + list_.push(item.id) + }) + + // *** TODO: keep results in local storage (or on session storage) + + // update_common_search_results(query, list_); + + let l = list_.join(','); + var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`); + + console.log('common search: search query > location = search') + window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`); + +} + +/** return from list only items that include 'key' + */ +function key_sublist(key, list) { + var result = []; + list.forEach( item => { + if (item.kb.includes(key)) { + result.push(item); + } + }); + + return result; +} diff --git a/pieces/search.js b/pieces/search.js deleted file mode 100644 index 0c05708..0000000 --- a/pieces/search.js +++ /dev/null @@ -1,303 +0,0 @@ -const kb = require('./kb-util.js'); -const match = require('./match-util.js'); -const products = require('../data/products.json'); - -/** VARIABLES - * may passed as module arguments - * ----------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////////// - - -var _allowFuzzy = true; // enable|disable fuzzy search -var n = 2; // Ngram base -var _fuzzyLimit = .5; // minimum bigram score for being considered a match -var max_list = 24; -var tolerance = 42; -var STORE = { id: 904 }; - -var _kwlinks; // keyword links (word-connections; imported via ajax-get) -var _products = []; // all products (imported via ajax-get) - -// setup options -var _maxResults = options.max_list; // limit suggestions -var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions -var _Ngram_base = 2; // number of N in Ngram spliting algorithm -var _isReady = false; // whether the searchbox is ready to be used - -// product keywords -var keywordsURL = options.keywords_json; - -var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } - - - - - - -/** SUPPLEMENTARY FUNCTIONS - * ----------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////////// - - - -// callback function for sorting resulrs per r (=rating) property -function compare_rate(a,b) { - return (a.r < b.r); -} - - - -/** SEARCH ENGINE - * ------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////// - - - - -function matchWordInList(q, list = false) { - - if (list !== false && list.length == 0) return []; // no results - - var result = []; - var firstPass = false; - - if (list === false) { - firstPass = true; // on first pass - list = products; // list is all products - } - - for(let i = 0 ; i < list.length ; i++) { - - // check exact + rate - - // else check partial + rate - - // else check similarity + rate - - let found = false; - let similarity = 0; - products[i].kb.split(' ').forEach( w => { - let sim = match.resemblance(q, w, 2); - if (sim > 0.5) { - found = true; - similarity = (sim > similarity) ? sim : similarity; - } - }); - if (found) { - products[i].similarity = similarity; - products[i].rating = (firstPass) - ? similarity * 2.0 - : products[i].rating + similarity * 2.0 - result.push(products[i]); - } - } - - return result -} - - - -// suggestions engine ////////////////////////////////////////////////// -// --- -function suggestions_engine(query) { - var results = []; - var pot = []; pot.length = 0; - - // clean and sanitize and mark links onto q(uery) string - var q = kb.keyboardize( kb.sanitizeGR( kb.clean(query.trim()) ) ).trim(); - - // TODO: - // construct direct-linked words - // = do unequivocally replaces - // steps: - // 1. replace accented vowels with non accented ones - // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'` - - var qAr = q.split(' '); // split to words - /// if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array - /// - /// if (qAr.slice(-1) == "") { - /// qAr.pop(); - /// } - - - pot = _products; // potential results // NOTE: CRITICAL: BY REFERENCE - - var wi = 0; // word index (from list) - var wc = qAr.length; - - qAr.forEach( w => { - - let sf = []; // (matches) so far - let mi; // position of match - wi++; - - pot.forEach( it => { - let matched = false; - let tester = ' '+ it.kb + ' '; - - // reset previous history and ratings - if (wi == 1) { - it.r = 0; - it.history = []; - } - - - // rate word-match > start-match > simple-match - // ... up to 8 points - - if (tester.indexOf(' '+ w +' ') != -1) { - it.r += 9; - it.history.push({ w: w, rate: 9 }); - matched = true; - } - else if (tester.indexOf(' '+ w) != -1) { - it.r += 5; - it.history.push({ w: w, rate: 5 }); - matched = true; - } - else if (tester.indexOf(w) != -1) { - it.r += 2; - it.history.push({ w: w, rate: 2 }); - matched = true; - } - - // rate `near-to-start` matching .. up to 7p - // rate `earlyness` of word in query .. up to 7p - - if ((mi = tester.indexOf(' '+w)) != -1) { - let fc1 = 100 - ((mi < 99) ? mi : 99); // near-to-start factor - let fc2 = wc - wi + 1; // query earlyness factor - let r1 = Math.floor(7*fc1/100); - let r2 = Math.floor(7*fc2/wc); - - it.r += (r1 + r2); - it.history.push({ w: w, left: [fc1, r1], early: [fc2, r2] }); - } - if (matched) sf.push(it); - }); - - if ((sf.length > (_maxResults + Math.floor(_maxResults/2))) - || (wi == 1) ) { - // ..if pot has a fair amount (= max + 50%) of results - // ..or these are results of '1st-query-word' - // set sf as new source - pot.lenght = 0; pot = []; - pot = JSON.parse(JSON.stringify(sf)); // copy by value - - } else { - // else.. keep the source list and increase of 'so-far rating' - // console.log('found small list', sf, pot) - pot.forEach( it => { - sf.forEach( si => { - if (it.id == si.id) { - it.r += 10; - it.history.push({ w: w, plus: '+10'}); - } - }); - }); - } - }); - - // sort results, get max-list of best rated - results = (pot.length > _maxResults) - ? pot.sort(compare_rate).slice(0, _maxResults) - : pot.sort(compare_rate) - - if (options.debug) console.log(results); - - return results; -} - - -// sub-module (start) -//////////////////////////////////////////////////////////////////////////// - -function update_common_search_results(q, results) { - let queries = getSessionObj('sr'); - let newSRlist = []; - let isnewQ = true; - if (queries === null) { - setSessionObj('sr', [{ - q: q, - result: result, - t: + new Date() - }]); - return true; - - } else { - - queries.forEach(it => { - if (it.q == q) { - newSRlist.push({ - q:q, - result: result, - t: + new Date() - }); - isnewQ = false; - } else { newSRlist.push(it); } - }); - - if (isnewQ) { - newSRlist.push({ - q:q, - result: result, - t: + new Date() - }); - } - - return true; - } -} - - - -//////////////////////////////////////////////////////////////////////////// -// sub-module (end) - - -function common_search(query) { - // clear ; sanitize ; split - var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' '); - - // if last item is empty, remove it - if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop() - - var results = _products; - - // for each key fitler results - qAr.forEach( key => { - results = key_sublist(key, results) - }); - - // echo products (and prepare list to POST) - var list_ = []; - results.forEach( item => { - if (options.debug) console.log(item.id, ':', item.w); - list_.push(item.id) - }) - - // *** TODO: keep results in local storage (or on session storage) - - // update_common_search_results(query, list_); - - let l = list_.join(','); - var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`); - - console.log('common search: search query > location = search') - window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`); - -} - -/** return from list only items that include 'key' - */ -function key_sublist(key, list) { - var result = []; - list.forEach( item => { - if (item.kb.includes(key)) { - result.push(item); - } - }); - - return result; -} diff --git a/pieces/suggest.js b/pieces/suggest.js index a420edd..c0a6e04 100644 --- a/pieces/suggest.js +++ b/pieces/suggest.js @@ -1,5 +1,5 @@ -const kb = require('./kb-util.js'); -const match = require('./match-util.js'); +const kb = require('../utils/kb-util.js'); +const match = require('../utils/match-util.js'); var _allowFuzzy = true; // enable|disable fuzzy search var n = 2; // Ngram base diff --git a/utils/kb-util.js b/utils/kb-util.js new file mode 100644 index 0000000..aac88f9 --- /dev/null +++ b/utils/kb-util.js @@ -0,0 +1,88 @@ +/** + * fast string manipulation utilities + * for bi-lingual (EL/EN) words/phrases + * based on the keyboard layout + */ + +// suplamentary arrays (mostly for cache) +// --- -- -- - - - + +var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + +var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + +var map = new Map(); +for (var i=0; i { + ap = pair.split(' '); + accented_vowels.push({ + a: ap[0], // accented + p: ap[1] // pure = non accended + }); +}); + + +// translates string to keyboard-latin keys +// (the ones that used whan typing each letter of the word) +const keyboardize = (str) => { + str = str.replace('\'',''); + var out = ''; + // [map]'s implementation is 40x faster than [for]'s + for (var i=0 ; i< str.length; i++) out += map.get(str[i]); + return out; +} + +// keyboardize an array of strings +const keyb_array = (arr) => { + kb_arr = []; + arr.forEach( w => { + kb_arr.push(keyboardize(w)); + }); + return kb_arr; +} + + +// transforms to lowercase; handles sigma-teliko +const sanitizeGR = (str) => { + str = str.toLowerCase(); + + // replace accended vowels with pure ones + accented_vowels.forEach( v => { + str = str.replaceAll(v.a, v.p); + }); + + // replace sigma on the end of words + str = str + ' '; + str = str.replaceAll('σ-', 'ς-'); + str = str.replaceAll('σ ', 'ς '); + + return str; +} + +// removes non keyword characters [+ . , !] and internal multiple-spaces +// @param txt (string): product description +const clean = (txt) => { + return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space + .replace('!','').replace('\"', '') // remove character + .replace(' ',' ').replace(' ',' '); // remove multiple spaces +} + + +// exports +// --- -- -- - - - + +module.exports = { + keyboardize, + keyb_array, + sanitizeGR, + clean +}; \ No newline at end of file diff --git a/utils/match-util.js b/utils/match-util.js new file mode 100644 index 0000000..a13fddb --- /dev/null +++ b/utils/match-util.js @@ -0,0 +1,92 @@ +/** Ngram fuzzy match algorithm + * (simple and fast) + */ +const createNgram = (word, n) => { // Ngram creation + if (word.length <3) return word; + const vector = []; + for (let i = 0; i < word.length-n+1; ++i) { + vector.push(word.slice(i, i + n)); + } + return vector; +}; + +/** similarity + * rates similarity between 2 words + * based on Ngram matches of N = n letters; + * implements a 2-dim check (all a-Ngrams vs all all b-Ngrams) + * + * @param {string} a : first word + * @param {string} b : second word + * @param {int} n : Ngram base + * @returns {float} : match percentage as a float in [0, 1] + */ +const similarity = (a, b, n) => { // Ngram match score + if (a.length > 0 && b.length > 0) { + const aNgram = createNgram(a, n); + const bNgram = createNgram(b, n); + let hits = 0; + for (let x = 0; x < aNgram.length; ++x) { + for (let y = 0; y < bNgram.length; ++y) { + if (aNgram[x] === bNgram[y]) { + hits += 1; + } + } + } + if (hits > 0) { + const union = aNgram.length + bNgram.length; + return (2.0 * hits) / union; + } + } + return 0; +}; + +/** resemblance + * is an alternative similarity rating; + * implements an 1-dim Ngram similarity check + * and it's much faster than similarity() + */ +const resemblance = (a, b, n) => { + if (a.length > n && b.length >= a.length) { + const aNgram = createNgram(a, n); + let hits = 0; + for (let i = 0; i < aNgram.length; ++i) { + if (b.includes(aNgram[i])) { + hits++; + } + } + if (hits > 0) { + // rate resemblance based on hits and length-similarity + return (hits / aNgram.length) * (a.length / b.length); + } + } + return 0; +} + + +/** is_exact_match + * + * check if a searching string -> query (string/latin in kb-format) + * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) + * + * @param query (string): searching string; string/latin in kb-format + * @param chkArr (array): array of synonyms; (array of utf-8/strings) + * @return (boolean): true|false + */ +function exact( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (w == query) found = true }); + return found; +} + +function partial( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (w.includes(query)) found = true }); + return found; +} + +module.exports = { + exact, + partial, + similarity, + resemblance +} diff --git a/utils/mem-usage.js b/utils/mem-usage.js new file mode 100644 index 0000000..766a93e --- /dev/null +++ b/utils/mem-usage.js @@ -0,0 +1,20 @@ +/** + * memory usage report utility + */ + +const formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`; + +function report() { + let memoryData = process.memoryUsage(); + + let memoryUsage = { + rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`, + heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`, + heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`, + external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`, + }; + + console.log(memoryUsage); +} + +module.exports = { report } diff --git a/utils/url-util.js b/utils/url-util.js new file mode 100644 index 0000000..e86ad9b --- /dev/null +++ b/utils/url-util.js @@ -0,0 +1,23 @@ +/** + * url utility + */ + +const querystring = require('querystring'); + +function struct(req, url) { + let url_parts = url.split('?'); + let query = (url_parts.length > 1) + ? querystring.decode(url_parts[1]) + : {}; + return { + method: req.method, + host: req.host, + path: url_parts[0], + query: query + } +} + +/** + * exports + */ +module.exports = { struct } \ No newline at end of file -- cgit v1.2.3