diff options
| author | Geo Halkiadakis <gchalkiadakis@sklavenitis.co.gr> | 2024-04-17 18:32:13 +0300 |
|---|---|---|
| committer | Geo Halkiadakis <gchalkiadakis@sklavenitis.co.gr> | 2024-04-17 18:32:13 +0300 |
| commit | d9364679a51ff80db8e5948ab089d749da36a6b2 (patch) | |
| tree | f847a03a017be8857923aaaae9f8dcf7177b2ba9 /app | |
| parent | 6248d2cf7a2214ac32628bc58248694108b4d8bf (diff) | |
| download | oseine-master.tar.gz oseine-master.tar.bz2 oseine-master.zip | |
dockerize the appHEADmasterdevelopment
Diffstat (limited to 'app')
| -rw-r--r-- | app/app.js | 21 | ||||
| -rw-r--r-- | app/benchmark/find.js | 112 | ||||
| -rw-r--r-- | app/benchmark/match-str.js | 80 | ||||
| -rw-r--r-- | app/pieces/prepare.js | 128 | ||||
| -rw-r--r-- | app/pieces/retro-search.js | 305 | ||||
| -rw-r--r-- | app/pieces/suggest.js | 786 | ||||
| -rw-r--r-- | app/routes/dev.js | 103 | ||||
| -rw-r--r-- | app/routes/index.js | 26 | ||||
| -rw-r--r-- | app/routes/v1.js | 68 | ||||
| -rw-r--r-- | app/utils/kb-util.js | 88 | ||||
| -rw-r--r-- | app/utils/match-util.js | 154 | ||||
| -rw-r--r-- | app/utils/mem-usage.js | 20 | ||||
| -rw-r--r-- | app/utils/url-util.js | 23 |
13 files changed, 1914 insertions, 0 deletions
diff --git a/app/app.js b/app/app.js new file mode 100644 index 0000000..42ff8c1 --- /dev/null +++ b/app/app.js @@ -0,0 +1,21 @@ +// app.js + +const Koa = require('koa'); +// const { koaBody } = require('koa-body'); + +// load parameters +require('dotenv').config(); + +// define app +const app = new Koa(); + +// middleware +// app.use(koaBody()); + +// load routes +let _r = require('./routes'); +app.use(_r.routes()).use(_r.allowedMethods()); + +// start server listening on APP_PORT +// app.listen(process.env.APP_PORT); +app.listen(3000); diff --git a/app/benchmark/find.js b/app/benchmark/find.js new file mode 100644 index 0000000..1e86c24 --- /dev/null +++ b/app/benchmark/find.js @@ -0,0 +1,112 @@ +/** + * benchmark: find a product in product list + * using: for vs forEach vs find + */ + +const microtime = require('microtime'); + +var products = require('../../data/products.json'); +if (products.length == 0) { + const prepare = require('../pieces/prepare.js'); + prepare.load_products( + 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' + ); +} + + + +var selected = []; +products.forEach( pr => { + if (Math.floor(Math.random() * 100) > 85) { + selected.push(pr.id); + } +}); + + +function compare() { + + let n = 4; + + var f0 = microtime.nowDouble(); + for(i=0 ; i < n ; i++) byFor(); + var f1 = microtime.nowDouble(); + // + var e0 = microtime.nowDouble(); + for(i=0 ; i < n ; i++) byEach(); + var e1 = microtime.nowDouble(); + // + var b0 = microtime.nowDouble(); + for(i=0 ; i < n ; i++) byFind(); + var b1 = microtime.nowDouble(); + + return { + n: n, + + for: f1-f0, + items_for: byFor(), + + each: e1-e0, + items_each: byEach(), + + find: b1-b0, + items_find: byFind(), + + sel: selected, + } +} + + +function byFor() { + var items = []; + var notFound = []; + var found; + selected.forEach( id => { + found = false; + for(i = 0; i < products.length; i++) { + if (products[i].id == id) { + items.push(products[i]); + found = true; + break; + } + } + if (!found) notFound.push(id) + + }) + return {items: items, nf: notFound}; +} + + +function byEach() { + var items = []; + var notFound = []; + var found; + selected.forEach( id => { + found = false; + products.forEach( pr => { + if (pr.id == id) { + items.push(pr); + found = true; + } + }); + if (!found) notFound.push(id) + }); + return {items: items, nf: notFound}; +} + +function byFind() { + var items = []; + var notFound = []; + var result; + selected.forEach( id => { + found = false; + result = products.find((pr) => pr.id == id); + if (result === undefined) notFound.push(id) + else items.push(result); + }); + return {items: items, nf: notFound}; +} + +module.exports = { + compare, + byFor +}
\ No newline at end of file diff --git a/app/benchmark/match-str.js b/app/benchmark/match-str.js new file mode 100644 index 0000000..225cbc2 --- /dev/null +++ b/app/benchmark/match-str.js @@ -0,0 +1,80 @@ +var microtime = require('microtime'); + +const match = require('../utils/match-util.js'); +const memory_usage = require('../utils/mem-usage.js'); +const kb = require('../utils/kb-util.js'); + +// const products = require('../../data/products.json'); +var products; +try { + products = require('../../data/products.json'); +} catch (e) { + products = []; +} + +// memory_usage.report(); + +// test runner +function run(query) { + + var f0 = microtime.nowDouble(); + let result; + // for(let i = 0 ; i < 20 ; i++) + result = matchQuery(query); + var f1 = microtime.nowDouble(); + + memory_usage.report(); + + return { + t: f1-f0, + q: query, + result: result + } +} + +// sample search implementation +function matchQuery(query) { + var result = []; + + let q = kb.keyboardize(query); + + for(let i = 0; i < products.length; i++) { + + let rate = 0; + let found = false; + let x; + let src = products[i].kb.split(' '); // (array) source of product (key-)words + if ((x = match.weighted_exact(q, src)) > 0) { + // console.log(products[i].kb, x); + rate = 5.0 + x; + found = true; + } + else if ((x = match.weighted_partial(q, src)) > 0) { + rate = 3.0 + x; + found = true; + } + else { + let similarity = 0; + src.forEach( w => { + let sim = match.resemblance(q, w, 2); + if (sim > 0.6) { + found = true; + similarity = (sim > similarity) ? sim : similarity; + } + }); + if (found) { + rate = 2 * similarity; + } + } + + if (found) { + products[i].rate = rate; + result.push(products[i]); + } + } + + return result.sort((a,b) => b.rate - a.rate).slice(0, 48);; +} + + +module.exports = { run } diff --git a/app/pieces/prepare.js b/app/pieces/prepare.js new file mode 100644 index 0000000..20825cd --- /dev/null +++ b/app/pieces/prepare.js @@ -0,0 +1,128 @@ +const fs = require('fs'); +var request = require('request'); + +const kb = require('../utils/kb-util.js'); + + +// const https = require("https"); + + +/* TODO: ?? parallel read + // var request = require('request-promise'); + var calls = [ + request({ + url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', + // headers: { ... } + }), + request({ + url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', + // headers: { ... } + }), + // + linked-terms and replaces + // and more.. + ]; + + Promise.all(calls).then(function(results) { + // do something with results[0] + // do something with results[1] + // ... + }); +*/ + + +/** create entity file from url + * used for entities like 'linked-words', 'synonyms', etc. + * + * @param {string} entity: use the entity name + * @param {string} url: prepared entity json-file in remote server + */ +function create(entity, url) { + request(url, + function (error, response, body) { + if (!error && response.statusCode == 200) { + // body is a ready json-string; no need to parse and (re-)stringify + try { + fs.writeFileSync(`${__dirname}/../../data/${entity}.json`, body, 'utf-8'); + // file written successfully + } catch (err) { + console.error(err); + } + } + } + ); +} + +function get_data_structure() { + // data structure is an array of `{ url:.., expiration:.., path:.. }` objects +} + +function set_data_structure(node) { + +} + + +function load_products(url) { + request(url, + function (error, response, body) { + if (!error && response.statusCode == 200) { + // body is a ready json-string; no need to parse and (re-)stringify + json = JSON.parse(body); + newJson = []; + + // TODO: + // + attach handle synonyms + // + mark brand-names + // + attach category-names and SAP-categories + // + construct combo words + // + remove non-important words + // + normalize popularity + + json.forEach( p => { + newJson.push({ + id: p.id, + w: p.w, + kb: kb.keyboardize(kb.clean(p.w)) + }); + }); + + try { + + fs.writeFileSync( + `${__dirname}/../../data/products.json`, + JSON.stringify(newJson), + 'utf-8' + ); + return true; // file written successfully + + } catch (err) { + console.error(err); + return false; + } + } + } + ); +} + + +// create data +// testing an implemenatatin of a custom local-fs-cache system +function random() { + var data = []; + for( let i = 0; i < (Math.floor(Math.random() * 100) +15) ; i++) { + data.push({ id: i, x5: i*5 }); + } + + try { + fs.writeFileSync(__dirname + '/../data/random.json', JSON.stringify(data), 'utf-8'); + // file written successfully + } catch (err) { + console.error(err); + } +} + + +module.exports = { + create, + random, + load_products +}
\ No newline at end of file diff --git a/app/pieces/retro-search.js b/app/pieces/retro-search.js new file mode 100644 index 0000000..4e26720 --- /dev/null +++ b/app/pieces/retro-search.js @@ -0,0 +1,305 @@ +const kb = require('../utils/kb-util.js'); +const match = require('../utils/match-util.js'); + +const products = require('../../data/products.json'); + + +/** VARIABLES + * may passed as module arguments + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + +var _allowFuzzy = true; // enable|disable fuzzy search +var n = 2; // Ngram base +var _fuzzyLimit = .5; // minimum bigram score for being considered a match +var max_list = 24; +var tolerance = 42; +var STORE = { id: 904 }; + +var _kwlinks; // keyword links (word-connections; imported via ajax-get) +var _products = []; // all products (imported via ajax-get) + +// setup options +var _maxResults = options.max_list; // limit suggestions +var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions +var _Ngram_base = 2; // number of N in Ngram spliting algorithm +var _isReady = false; // whether the searchbox is ready to be used + +// product keywords +var keywordsURL = options.keywords_json; + +var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } + + + + + + +/** SUPPLEMENTARY FUNCTIONS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + + +// callback function for sorting resulrs per r (=rating) property +function compare_rate(a,b) { + return (a.r < b.r); +} + + + +/** SEARCH ENGINE + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + + +function matchWordInList(q, list = false) { + + if (list !== false && list.length == 0) return []; // no results + + var result = []; + var firstPass = false; + + if (list === false) { + firstPass = true; // on first pass + list = products; // list is all products + } + + for(let i = 0 ; i < list.length ; i++) { + + // check exact + rate + + // else check partial + rate + + // else check similarity + rate + + let found = false; + let similarity = 0; + products[i].kb.split(' ').forEach( w => { + let sim = match.resemblance(q, w, 2); + if (sim > 0.5) { + found = true; + similarity = (sim > similarity) ? sim : similarity; + } + }); + if (found) { + products[i].similarity = similarity; + products[i].rating = (firstPass) + ? similarity * 2.0 + : products[i].rating + similarity * 2.0 + result.push(products[i]); + } + } + + return result +} + + + +// suggestions engine ////////////////////////////////////////////////// +// --- +function suggestions_engine(query) { + var results = []; + var pot = []; pot.length = 0; + + // clean and sanitize and mark links onto q(uery) string + var q = kb.keyboardize( kb.sanitizeGR( kb.clean(query.trim()) ) ).trim(); + + // TODO: + // construct direct-linked words + // = do unequivocally replaces + // steps: + // 1. replace accented vowels with non accented ones + // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'` + + var qAr = q.split(' '); // split to words + /// if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array + /// + /// if (qAr.slice(-1) == "") { + /// qAr.pop(); + /// } + + + pot = _products; // potential results // NOTE: CRITICAL: BY REFERENCE + + var wi = 0; // word index (from list) + var wc = qAr.length; + + qAr.forEach( w => { + + let sf = []; // (matches) so far + let mi; // position of match + wi++; + + pot.forEach( it => { + let matched = false; + let tester = ' '+ it.kb + ' '; + + // reset previous history and ratings + if (wi == 1) { + it.r = 0; + it.history = []; + } + + + // rate word-match > start-match > simple-match + // ... up to 8 points + + if (tester.indexOf(' '+ w +' ') != -1) { + it.r += 9; + it.history.push({ w: w, rate: 9 }); + matched = true; + } + else if (tester.indexOf(' '+ w) != -1) { + it.r += 5; + it.history.push({ w: w, rate: 5 }); + matched = true; + } + else if (tester.indexOf(w) != -1) { + it.r += 2; + it.history.push({ w: w, rate: 2 }); + matched = true; + } + + // rate `near-to-start` matching .. up to 7p + // rate `earlyness` of word in query .. up to 7p + + if ((mi = tester.indexOf(' '+w)) != -1) { + let fc1 = 100 - ((mi < 99) ? mi : 99); // near-to-start factor + let fc2 = wc - wi + 1; // query earlyness factor + let r1 = Math.floor(7*fc1/100); + let r2 = Math.floor(7*fc2/wc); + + it.r += (r1 + r2); + it.history.push({ w: w, left: [fc1, r1], early: [fc2, r2] }); + } + if (matched) sf.push(it); + }); + + if ((sf.length > (_maxResults + Math.floor(_maxResults/2))) + || (wi == 1) ) { + // ..if pot has a fair amount (= max + 50%) of results + // ..or these are results of '1st-query-word' + // set sf as new source + pot.lenght = 0; pot = []; + pot = JSON.parse(JSON.stringify(sf)); // copy by value + + } else { + // else.. keep the source list and increase of 'so-far rating' + // console.log('found small list', sf, pot) + pot.forEach( it => { + sf.forEach( si => { + if (it.id == si.id) { + it.r += 10; + it.history.push({ w: w, plus: '+10'}); + } + }); + }); + } + }); + + // sort results, get max-list of best rated + results = (pot.length > _maxResults) + ? pot.sort(compare_rate).slice(0, _maxResults) + : pot.sort(compare_rate) + + if (options.debug) console.log(results); + + return results; +} + + +// sub-module (start) +//////////////////////////////////////////////////////////////////////////// + +function update_common_search_results(q, results) { + let queries = getSessionObj('sr'); + let newSRlist = []; + let isnewQ = true; + if (queries === null) { + setSessionObj('sr', [{ + q: q, + result: result, + t: + new Date() + }]); + return true; + + } else { + + queries.forEach(it => { + if (it.q == q) { + newSRlist.push({ + q:q, + result: result, + t: + new Date() + }); + isnewQ = false; + } else { newSRlist.push(it); } + }); + + if (isnewQ) { + newSRlist.push({ + q:q, + result: result, + t: + new Date() + }); + } + + return true; + } +} + + + +//////////////////////////////////////////////////////////////////////////// +// sub-module (end) + + +function common_search(query) { + // clear ; sanitize ; split + var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' '); + + // if last item is empty, remove it + if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop() + + var results = _products; + + // for each key fitler results + qAr.forEach( key => { + results = key_sublist(key, results) + }); + + // echo products (and prepare list to POST) + var list_ = []; + results.forEach( item => { + if (options.debug) console.log(item.id, ':', item.w); + list_.push(item.id) + }) + + // *** TODO: keep results in local storage (or on session storage) + + // update_common_search_results(query, list_); + + let l = list_.join(','); + var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`); + + console.log('common search: search query > location = search') + window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`); + +} + +/** return from list only items that include 'key' + */ +function key_sublist(key, list) { + var result = []; + list.forEach( item => { + if (item.kb.includes(key)) { + result.push(item); + } + }); + + return result; +} diff --git a/app/pieces/suggest.js b/app/pieces/suggest.js new file mode 100644 index 0000000..c0a6e04 --- /dev/null +++ b/app/pieces/suggest.js @@ -0,0 +1,786 @@ +const kb = require('../utils/kb-util.js'); +const match = require('../utils/match-util.js'); + +var _allowFuzzy = true; // enable|disable fuzzy search +var n = 2; // Ngram base +var _fuzzyLimit = .5; // minimum bigram score for being considered a match + + +/** (Search) SUGGESTIONS ENGINE + * --------------------------------------------------------------------------- + * + * Operates in dual mode; + * -- suggestions engine (interactive) + * -- classic-like mode (passive) + * + * TODO: + * CRITICAL: (optimization) + * Search initialization uses quite a lot of network sources; + * thus it should be started in a later time; + * lets say ... after `x` seconds + * or... when document/core-ui-elements are ready + * + * @parametres (json) : options + * --- + * @var keywords_json (sring) : endpoint url of linked keywords structure + * @var products_json (string) : endpoint od product descriptions + * @var search_tag (string) : selector of field that shall act as typeahead-suggestions + * @var visualize_search_results_url (str) : url that will visualize the sended "results-page" + * @var debug (bool) : if true sends several debug console messages; if false mesagges are eliminated + */ + +suggestions_module({ + keywords_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', + products_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', + search_tag: '#tagsInput', + visualize_search_results_url: '/product_list', + debug: ((location.hostname == 'localhost') || (location.hostname == '127.0.0.1')) +}); + +function suggestions_module(options) { + + /** CONTENTS + * + * +1: Variables + * + * +2: Purify string functions + * + keyboardize + * + sanitize_GR + * + clean + * + mark_explicit_links + * + * +3: Supplementary function (vanilla js) + * + ajax_get(url, callback) + * + createNgram (fuzzy) + * + checkSimilarity (fuzzy) + * + check_match + * + is_exact_match + * + match_one + * + * +4: Actual data loading (async) + * + * +5: Suggestions Engine (jQuery) + * + suggestions_engine + * + */ + + console.log('executing suggestions...'); + + + /** 1. VARIABLES + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + var _kwlinks; // keyword links (word-connections; imported via ajax-get) + var _products = []; // all products (imported via ajax-get) + + // setup options + var _maxResults = 24; // limit suggestions + var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions + var _timeout_ms = 100; // time (in ms) for the search engine to find matches (before rendering) + + var _Ngram_base = 2; // number of N in Ngram spliting algorithm + var _isReady = false; // whether the searchbox is ready to be used + + // product keywords + var keywordsURL = options.keywords_json; + + var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } + + + /** replaces (correcting descriptions) + * == construct unequivocally liked words ////////////////////////////////// + * ------------------------------------------------------------------------- + * + * NOTE: TODO: + * in future implementations multi-word keywords + * may use the non-breaking space as conecting character (\u00A0) instead of dush (-) + * (or maybe both of them) + * + * also TODO: + * in future implementaions linked words may passed via some endpoint + */ + replaces = []; + replaceSource = [ + '3Α;3-ΑΛΦΑ', + '3 ΑΛΦΑ;3-ΑΛΦΑ', + 'HEAD & SHOULDERS;HEAD&SHOULDERS', + 'HEAD N SHOULDERS;HEAD&SHOULDERS', + 'W.K Kellogg; W-K-Kellogg', + 'W.K Kellogg;', + '7 DAYS;7-DAYS', + '7 UP;7UP', + '7-UP;7UP', + 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', + 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', + 'COCA COLA;COCA-COLA', + 'J P. CHENET; J.P.CHENET', + 'J.P. CHENET; J.P.CHENET', + 'COCACOLA;COCA-COLA', + 'NES CAFE;NESCAFE', + 'NES-CAFE;NESCAFE', + 'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS', + 'PETIT MARSEILLAIS;PETIT-MARSEILLAIS', + 'Το Μάννα;Το-Μάννα', + 'Χωρίς Γλουτένη;Χωρίς-Γλουτένη', + 'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη', + 'Χωρίς Αλάτι;Χωρίς-Αλάτι', + 'Χωρίς Λακτόζη;Χωρίς-Λακτόζη', + 'Χωρίς Συντηρητικά;Χωρίς-Συντηρητικά', + 'Χωρίς Αλκοόλ;Χωρίς-Αλκοόλ', + 'Χωρίς Kαφεϊνη;Χωρίς-Kαφεϊνη', + 'Χωρίς Kαφεΐνη;Χωρίς-Kαφεϊνη', + 'Χωρίς Γλυκάνισο;Χωρίς-Γλυκάνισο', + 'Χωρίς Ανθρακικό;Χωρίς-Ανθρακικό', + 'Υψηλής Παστερίωσης;Υψηλής-Παστερίωσης', + 'Ολικής Άλεσης;Ολικής-Άλεσης', + 'Ολικής Aλέσεως;Ολικής-Aλέσεως', + 'Χαρτί Υγείας;Χαρτί-Υγείας', + 'ρολό υγείας;ρολό-υγείας', + 'χαρτί τουαλέτας;χαρτί-τουαλέτας', + 'Χαρτί Κουζίνας;Χαρτί-Κουζίνας', + 'ρολό κουζίνας;ρολό-κουζίνας', + 'Μπάρες Δημητριακών;Μπάρες-Δημητριακών', + 'Ας Μαγειρέψουμε;Ας-Μαγειρέψουμε', + 'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', + 'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', + 'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ', + 'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ', + 'ΕΛ ΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', + 'ΕΛΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', + 'FREE STEP;FREE-STEP', + 'EL SABOR;EL-SABOR', + 'ELSABOR;EL-SABOR', + 'DOUWE EGBERTS;DOUWE-EGBERTS', + 'DOUWEEGBERTS;DOUWE-EGBERTS', + 'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', + 'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', + 'SPIN SPAN;SPIN-SPAN', + 'SPINSPAN;SPIN-SPAN', + 'CRETA-FARMS;CRETA-FARM', + 'CRETA-FARM;CRETA-FARM', + 'CRETAFARM;CRETA-FARM', + 'Ολες-τις-Χρήσεις;Ολες-τις-Χρήσεις', + 'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη', + 'φρουι ζελε, φρουί-ζελε', + 'DR BECKMANN, DR-BECKMANN' + ]; + replaceSource.forEach( it => { + st = sanitize_GR( + it.toLowerCase() + ).split(';'); + replaces.push({ + src: ' '+ st[0] +' ', // encolse between spaces + trg: ' '+ st[1] +' ' // to separate from before/after words + }); + }); + + /** mark_explicit_links + * + * mark linked words shall be handled as one-(key)word + * also, edit common mistakes with suggested replaces + * + * @param str + * @return + */ + function mark_explicit_links(str) { + str = ' '+ str +' '; + replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); }); + return str.replaceAll(' ', ' ').trim(); + } + + + + + + /** 3. SUPPLEMENTARY FUNCTIONS + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + // TODO: + // exclude some generic non-critial words when proccessing user's query + // ** example code to work with: + // var ignoredKeys_kb = []; // keywords to ignore (in kb-format) + // 'μας με σε για του της των από στο στον &'.split(' ').forEach(w => { ignoredKeys_kb.push(keyboardize(w)); }); + + // pure JS ajax GET request; return data as JSON + // no fancy things like UTF8; if needed use base64 + function ajax_get(url, callback) { + var xmlhttp = new XMLHttpRequest(); + xmlhttp.onreadystatechange = function() { + if (xmlhttp.readyState == 4 && xmlhttp.status == 200) { + try { + var data = JSON.parse(xmlhttp.responseText); + } catch(err) { + if (options.debug) console.log(err.message + " in " + xmlhttp.responseText); + return; + } + callback(data); + } + }; + xmlhttp.open("GET", url, true); + xmlhttp.send(); + } + + + + + // Search Endine's match functions + ////////////////////////////////////////////////////////////////////////////// + + /** check_match + * --- + * check if a searching string -> query (string/latin in kb-format) + * matches an item of the array of synonyms -> chkArr (array of utf-8/strings) + * + * + option to use fyzzy (bigram) match + * + * @return: matched string (utf-8) --or-- false (if not matched) + */ + function check_match( query, chkArr, fuzzy = false ) { + var result = ''; + var found = false; + + if (query == ' ') return chkArr[0]; + + chkArr.forEach( chk => { + if (!found) { + chk_kb = keyboardize(chk); + if ( (chk_kb.indexOf( query ) !== -1) + || (fuzzy && (match.similarity(chk_kb, query, n) > _fuzzyLimit)) ) { + found = true; + result = chk; + } + } + }); + return found ? result : false; + } + + /** is_exact_match + * + * check if a searching string -> query (string/latin in kb-format) + * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) + * + * @param query (string): searching string; string/latin in kb-format + * @param chkArr (array): array of synonyms; (array of utf-8/strings) + * @return (boolean): true|false + */ + function is_exact_match( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (keyboardize(w) == query) found = true }); + return found; + } + + /** match_one + * + * check if at-least-ONE item from an array of query-words -> qArr (array of string/latin in kb-format) + * matches any item of an array of synonyms -> chkArr (array of utf-8/strings) + * + * @return (boolean) true|false + */ + function match_one( qArr, chkArr ) { + found = false; + qArr.forEach( query => { + chkArr.forEach( w => { if (keyboardize(w) == query) found = true }); + }); + return found; + } + + + + + /** 4. ACTUAL DATA LOADING + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + + // TODO: + // control completion of async loads via .then() rather by this custom structrure + // Need to rewrite the folllowing code ............................. from here + // ........................................................................... + // ........................................................................... + + + /** workline + * ------------------------------------------------------------------------- + * custom object/data-structure + * to track status/completion of async svents + * (it does the job using a special set method) + */ + var workline = { + + trackerJL : 0, + set jsonLoaded(x) { + this.trackerJL = x; + + // fire event on certain values + if (x == 3) { + if (options.debug) console.info('suggestion-engine requirements fulfilled'); + _isReady = true; + // code to execute + // ... + } + }, + get jsonLoaded() { return this.trackerJL; } + }; + + + /** LOAD DATA (from endoints) + * --------------------------------------------------------------------------- + */ + + ajax_get( keywordsURL, function(data) { // get _kwlinks + _kwlinks = data; + if (options.debug) console.log('...keywords loaded;'); + workline.jsonLoaded++; + }); + + function load_store_products(storeID) { + ajax_get(options.products_json, function(data) { // get stor's _products + _products = data; + _products.forEach(p => {p.kb = keyboardize(p.w).toLowerCase()} ); + if (options.debug) console.log('...products loaded;'); + workline.jsonLoaded++; + CURRENT_STOREs_CATALOG = storeID; + }); + } + + var STORE = { id: 904 }; + + if (STORE.id != 0) load_store_products(STORE.id); + + // ........................................................................... + // ........................................................................... + // ................................................................ up to here + + + + + /** 5. SUGGESTIONS ENGINE + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + workline.jsonLoaded++; // notify workline that jQuery is ready!! + + + // suggestions engine ////////////////////////////////////////////////// + // --- + function suggestions_engine(qOrig) { + var results = []; // suggestions to respond + var proList = []; // list of products (for all suggestions) + var commonL = []; // list of common products (for multiple suggestions) + var possibleNext = []; // list of possible next suggestions + + var root, last; + + var space_ended = (qOrig.slice(-1) == ' ') ? true : false; + + // clean and sanitize and mark links onto q(uery) string + var q = mark_explicit_links( sanitize_GR( clean_text(qOrig) ) ); + + // TODO: + // construct direct-linked words + // = do unequivocally replaces + // steps: + // 1. replace accented vowels with non accented ones + // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'` + // --- -- -- - - - + // test: + // oneliner: var str = 'αλφα βητα ΑΛΦΑ Βητα world'; var src = 'αλφα βητα'; var reg = new RegExp(src, "gi"); var replacedOnce = str.toLowerCase().replace(reg, 'α-β'); console.log(replacedOnce); + + var qAr = q.split(' '); // split to words + if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array + + if (qAr.slice(-1) == "") { + qAr.pop(); + } + + if (options.debug) console.log(qAr); + + if (qAr.length == 1) { // suggest 1st word ////////////////////// + + var kbq = kb.keyboardize(q); + + // loop through root-words + // ... to match all possible suggestions; + _kwlinks.forEach( it => { + chk = check_match(kbq, it.w, true); + + if ( chk != false ) { + results.push({ w: chk }); + + // ... also keep possible products in a list + it.c.forEach( wo => { // for every Word-Link-Node + if (proList.length < _maxResults + 2) { // NOTE: about +2 (bellow) + + // concatenate this-suggestion's product sub-list (wo.p) + // to all-suggestions posiible products-list (proList) + proList = proList.concat(wo.p); + // keep unique products in the products-list + proList = proList.filter((item, i, ar) => ar.indexOf(item) === i); + } + }); + } + + }); + } + + if (qAr.length == 2) { // suggest 2nd word ///////////////////////// + root = qAr[0]; + last = qAr[1]; + kbroot = kb.keyboardize(root); + kblast = kb.keyboardize(last); + + // TODO: + // change [_kwlinks.forEach] to for loop + // --- + _kwlinks.forEach( it => { // locate the ... + if (match.exact(kbroot, kb.keyb_array(it.w))) { // exact match of root-word + + it.c.forEach ( wo => { // loop the word-links ... + chk = check_match(kblast, wo.w, true); + if ( chk !== false ) { // if a match is found + results.push({ // keep suggestion + w: root +' '+ chk, + f: wo.f + }); + if (proList.length < _maxResults + 2) { // plus... + proList = proList.concat(wo.p); // keep products-list + proList = proList.filter((item, i, ar) => ar.indexOf(item) === i); + } + } + }); + + } + }); + } + + if (qAr.length > 2) { // suggest N-th word (N>2) /////////////// + root = qAr.shift(); // isolate first item of qAr + last = qAr.pop(); // isolate last item too + // now qAr includes only the items after root and before last; + // so qAr includes all already selected suggestions (but root) + + // prepare/cache kb-formated string for any key we're going to use + var kb_qAr = []; // array of selected suggestions in kb-format + qAr.forEach( w => { kb_qAr.push(keyboardize(w)); }) + kbroot = keyboardize(root); + kblast = keyboardize(last); + + // TODO: + // change [_kwlinks.forEach] to for loop + // --- + _kwlinks.forEach( it => { + if (is_exact_match(kbroot, it.w)) { // locate root-word + + // calculate list of common items/products (commonL) + // for selected suggestions + is1stOcc = true; // 1st occurance flag + + it.c.forEach ( swo => { + if (match_one(kb_qAr, swo.w)) { + // swo is one of the already selected suggestions + // so... update the common-(products)-L(ist) + if (is1stOcc) { + commonL = swo.p; // init list (on 1st occurance) + is1stOcc = false; + } + else { + // caclulate list of common products + // = intersection of (so-far) commonL and swo.p + // commonL = commonL.filter(value => swo.p.includes(value)); + commonL = commonL.filter(function(n) { return swo.p.indexOf(n) !== -1; }); + } + } + else { // if swo is not already selected + // then This is a possible NEXT suggestion + possibleNext.push(swo); + } + }); + + // Now that we have all the possible next suggestions + // we'll match them with the last word of the query + + possibleNext.forEach( poss => { // for tthe possible next suggestions + // if last-word-of-query matches possible word(s) + // and list of word's products has commons with commonL + // then THiS is a Valid-Next-Suggestion + chk = check_match(kblast, poss.w, true); + if (chk !== false) { + // check intersection of commonL and suggestion's product-lists + tempL = commonL.filter(value => poss.p.includes(value)); + if (tempL.length > 0) { + results.push({ + w: root +' '+ qAr.join(' ') +' '+ chk, + f: poss.f + }); + // update proList too + proList = proList.concat(tempL); + } + } + }); + } + + }); + } + + // NOTE: about +2 (vs +1) + // after having calculated next suggestions and a banch of possible products + // the proccedure is going to decide what data will return; + // if pro(ducts)List includes less items than maximum suggestions + // ... this will be the array to return. So + // ... +2 ensures that this list will not become shorter than this limit + + // calculate unique products + // (credit: https://stackoverflow.com/questions/11246758/) + let unique = proList.filter((item, i, ar) => ar.indexOf(item) === i); + + + + if (unique.length < (_maxResults +1)) { // IF list is small ...... + + // serve products instead of suggestions + // ... + results = []; + notOnThisStore = [] + unique.forEach( pr => { + for (i=0 ; i< _products.length-1 ; i++) { // for makes things faster + pi = _products[i]; + if (pi.id == pr) { + results.push(pi); + break; + } + } + }); + + } else { // remove forced link character '-' from suggestions ..... + var dirty_results = results; + results = []; + for (i=0 ; i< dirty_results.length ; i++) { + results.push({ w: dirty_results[i].w.replaceAll('-', ' ') }) + } + } + + return results; + } + + // simple, fast search products by numeric code property + // --- + function search_by_code( num ) { + const str = num.toString(); + var results = []; + var p; + for (i=0 ; i< _products.length-1 ; i++) { + p = _products[i]; + if (results.length > _maxResults) { break; } + if ( (p.bp+'-'+p.bc).indexOf(str) !== -1 ) { results.push(p) } + } + return results; + } + + // suggestions caller (router function) + // arguments: + // * qOrig : original query string + // ** list : callback array structure to host results + // --- + var isuggest = function(qOrig, list) { + var results; + + qSanit = sanitize_GR(qOrig); // sanitize greel accended chars + + // TODO: + // check if all source-lists are ready + // if not you need to wait ... + // via async promishes or synced timouts + + // if query seems to be some king of 'code/id' + if (qSanit.length>2 && qSanit.match(/^[0-9]+$/) != null) { + results = search_by_code(qOrig); + + } else { + // string match procedure with suggestions engine + results = suggestions_engine(qOrig); + + } + list(results); + } + + /** + jQuery(function() { // on document ready code ////////////////////////// + /** + + // UI-dependent code + // uses reference to specific document element (passed via options) + // ----------------------------------------------------------------------- + const searchBox = $(options.search_tag); + searchBox.typeahead( + { + hint: true, + highlight: true, + minLength: 1 + }, + { + limit: _maxResults, + name: 'kwlinks', + displayKey: 'w', + source: isuggest, + templates: { + suggestion: function(data) { + if (data.id) + return '<div data-id="'+ data.id +'">'+ data.w +'</div>'; + return '<div>'+ data.w +'<span class="icon icon-arrowRight--b"></span></div>'; + }, + empty: '<div class="-empty-">Δεν υπάρχει στο κωδικολόγιο του καταστήματος</div>' + } + } + ) + .bind("typeahead:selected", function(obj, datum, name) { + if (datum.hasOwnProperty('id')) { // ** selected: PRODUCT + + // $('#js-add-product-to-order').attr('disabled', false); + // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img); + if (options.debug) console.log('selected: ', datum.id, datum.w); + + // TODO: GOTO product page + window.location.href = 'product_list?product=id-' + datum.id; + + // $("#product-quantity").trigger('focus'); + + } + else { // selected: SUGGESTION + + $('.typeahead').typeahead('val','').trigger('blur'); + $('.typeahead').typeahead('val', datum.w +' ') + .trigger("query"); + // give some time to the engine to calculate results + // then fire focus again... + setTimeout(() => { $('.typeahead').trigger('focus'); }, _timeout_ms); + } + + }) + .bind("typeahead:cursorchange", function( event, obj) { + // track cursor-chane to handle special keys after a final product is selected + + // TODO: + // handle exception where obj is undefined; + // this occures... + // when cusror returns from suggestions list back to the search field + + if (typeof obj === 'undefined') cursor_on = { none: true }; + else if (obj.hasOwnProperty('id')) cursor_on = obj; + else cursor_on = { none: true }; + }); + + // TRACK user search attempt /////////////////////////////////////////////// + $('.typeahead').on('keyup', function(e) { + if (_isReady) { + // console.log('on:', cursor_on); + // console.log('key:', e.key); + if ((e.key == ' ') && cursor_on.hasOwnProperty('id')) { + + // product is actually selected + var datum = cursor_on; + + // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img); + $("#product-quantity").trigger('focus'); + e.preventDefault(); + + } else { + if (e.key === "Enter") { + + // if curson is not on some option + if ((typeof curson_on === 'undefined') || (curson_on.none == true)) { + + // do a common search + if (options.debug) console.log('Do a Non-Suggestions search', searchBox.val()); + + // DEPRICATED: var location = encodeURI('/product_list?productSearch=%'+ searchBox.val() +'%'); + + search_results = common_search(searchBox.val()); + + + } else { + // launch a product page + var location = '/product_list?product=id-'+ cursor_on.id; + if (options.debug) console.log('product', cursor_on.id, cursor_on.w); + + console.log('go to product', location) + } + // window.location.href = location; + } + } + } + }); + + }); + */ + + + /* + function common_search(query) { + // clear ; sanitize ; split + var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' '); + + // if last item is empty, remove it + if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop() + + var results = _products; + + // for each key fitler results + qAr.forEach( key => { + results = key_sublist(key, results) + }); + + // echo products (and prepare list to POST) + var list_ = []; + results.forEach( item => { + if (options.debug) console.log(item.id, ':', item.w); + list_.push(item.id) + }) + + let l = list_.join(','); + var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`); + + if ($('.js-categories.selected').length + || $('.js-checkout.selected').length + ) { + url = encodeURI(`api/v1/products?eys_code=${l}&q=${query}`); + console.log('common search : fetch results > history.push', url, query); + fetchSearchProducts(url, query); + window.history.pushState('search', null, '/product_list?search='+query); + } else { + console.log('common search: search query > location = search') + window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`); + } + + // Redirect with POST (template) + // --- -- -- - - - + // var form = $([ + // `<form action="${url}" method="post">`, + // '<textarea name="products_list">', + // list_.join(','), + // '</textarea>', + // '</form>' + // ].join('')); + // $('body').append(form); + // form.submit(); + + } + */ + + /** return from list only items that include 'key' + */ + function key_sublist(key, list) { + var result = []; + list.forEach( item => { + if (item.kb.includes(key)) { + result.push(item); + } + }); + + return result; + } +}
\ No newline at end of file diff --git a/app/routes/dev.js b/app/routes/dev.js new file mode 100644 index 0000000..cd40822 --- /dev/null +++ b/app/routes/dev.js @@ -0,0 +1,103 @@ +/** + * defines routes + * exports router + */ + +const Router = require('koa-router'); +const urler = require('../utils/url-util.js'); + +const data = require('../pieces/prepare.js'); + +const bench = require('../benchmark/find.js'); +const matchStr = require('../benchmark/match-str.js'); + + +// Prefix all routes with: /items +const router = new Router(); + + + +// Routes + +/* simple route example + + let items = [ + { id: 100, iname: 'Quartz Analog Wrist Watch', price: 'US $4.99'}, + { id: 101, iname: 'Leather Peep Pump Heels', price: 'US $33.56'}, + { id: 102, iname: 'Apple iPod', price: 'US $219.99'}, + { id: 103, iname: 'Prince Phantom 97P Tennnis Racket', price: 'US $50.00'}, + ]; + + router.get('/items', (ctx, next) => { + ctx.body = items; + next(); + }); + + router.get('/items/:id', (ctx, next) => { + let getCurrentItem = items.filter(function(item) { + if (item.id == ctx.params.id) { + return true; + } + }); + if (getCurrentItem.length) { + ctx.body = getCurrentItem[0]; + } else { + ctx.response.status = 404; + ctx.body = 'Item Not Found'; + } + next(); + }); +*/ + + +// NOTE: set timeout per route (how-to) +// https://stackoverflow.com/questions/66634123/how-to-add-individual-timeout-value-per-specific-route-using-node-js + + +// TEST routes +//////////////////////////////////////////////////////////////////////////////// + +router.get('/test', (ctx) => { // easy test route + // test anything ... + ctx.body = { params: urler.struct(ctx.request, ctx.url), ctx: ctx } +}); + +router.get('/test/do-data', (ctx) => { // easy test route + // test anything ... + // data.create( + // 'products', + // 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' + // ); + data.load_products( + 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' + ); + ctx.body = { success: true, operation: 'create new data' }; +}); + +// benchmark routes; +// shall be removed from production +//////////////////////////////////////////////////////////////////////////////// + +/// router.get('/test/json', (ctx, next) => { // easy test route +/// // test anything ... +/// ctx.body = { t: + new Date(), p: products }; +/// next(); +/// }); + +router.get('/bench/find', (ctx) => { // easy test route + // test anything ... + ctx.body = bench.compare(); + // next(); +}); + +router.get('/bench/match/:title', (ctx) => { // easy test route + // test anything ... [ query = 'solokata' ] + // ctx.body = bench.compare(); + ctx.body = matchStr.run(ctx.params.title); + // next(); +}); + + +// export routes + +module.exports = router; diff --git a/app/routes/index.js b/app/routes/index.js new file mode 100644 index 0000000..e3d6230 --- /dev/null +++ b/app/routes/index.js @@ -0,0 +1,26 @@ +/** + * routes (index) + */ +const Router = require('koa-router'); + +const router = new Router(); + +// Require grouped routes + +let v1 = require('./v1.js'); +router.use(v1.routes()); + +let dev_paths = require('./dev.js'); +router.use(dev_paths.routes()); + +// define default route +router.get('/', (ctx) => { + ctx.body = { + success: true, + title: 'oseine', + description: 'oseine search engine is not elastic', + message: 'where are you now?' + } +}) + +module.exports = router; diff --git a/app/routes/v1.js b/app/routes/v1.js new file mode 100644 index 0000000..394dde6 --- /dev/null +++ b/app/routes/v1.js @@ -0,0 +1,68 @@ +/** + * api v1 routes + */ + +const Router = require('koa-router'); +const prepare = require('../pieces/prepare.js'); + + +// Prefix all routes with: /items +const router = new Router({ + prefix: '/v1' +}); + + + +// Routes + +router.get('/prepare/:title', (ctx) => { + let result = false; + switch (ctx.params.title) { + case 'products': + result = prepare.load_products( + 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' + ); + break; + + default: + break; + + } + console.log(result); + ctx.body = { success: result }; +}) + + +router.get('/search', (ctx) => { + ctx.body = []; +}); + +router.get('/search/:title', (ctx) => { + let result = [ + { id: 10, w: 'Ένα Προϊόν' }, + { id: 15, w: 'Άλλο Προϊόν' }, + { id: 20, w: 'Προϊόν 3' }, + { id: 25, w: 'Προϊόν 4' } + ] + // console.log(ctx); + /// const kb = require('./utils/kb-util.js'); + /// let words = kb.keyboardize(kb.clean(ctx.params.title)).split(' '); + /// ctx.body = { + /// params: ctx.params, + /// results: words, + /// nxt: next + /// }; + ctx.body = result; +}); + + +/** TODO: ?? other routes ?? + * ///////////////////////////////////////////////////////////////////////////// + * + /suggest/<some search string> + * + /prepare (products, keywords, linked-terms etc) + * + /stats + */ + + +// export routes +module.exports = router; diff --git a/app/utils/kb-util.js b/app/utils/kb-util.js new file mode 100644 index 0000000..aac88f9 --- /dev/null +++ b/app/utils/kb-util.js @@ -0,0 +1,88 @@ +/** + * fast string manipulation utilities + * for bi-lingual (EL/EN) words/phrases + * based on the keyboard layout + */ + +// suplamentary arrays (mostly for cache) +// --- -- -- - - - + +var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + +var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + +var map = new Map(); +for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]); + + +// cache (=create a global array) +// of accended to non-accended vowels mapping +// --- -- -- - - - +accented_vowels = []; +[ + 'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ΐ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω', + 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω' +].forEach( pair => { + ap = pair.split(' '); + accented_vowels.push({ + a: ap[0], // accented + p: ap[1] // pure = non accended + }); +}); + + +// translates string to keyboard-latin keys +// (the ones that used whan typing each letter of the word) +const keyboardize = (str) => { + str = str.replace('\'',''); + var out = ''; + // [map]'s implementation is 40x faster than [for]'s + for (var i=0 ; i< str.length; i++) out += map.get(str[i]); + return out; +} + +// keyboardize an array of strings +const keyb_array = (arr) => { + kb_arr = []; + arr.forEach( w => { + kb_arr.push(keyboardize(w)); + }); + return kb_arr; +} + + +// transforms to lowercase; handles sigma-teliko +const sanitizeGR = (str) => { + str = str.toLowerCase(); + + // replace accended vowels with pure ones + accented_vowels.forEach( v => { + str = str.replaceAll(v.a, v.p); + }); + + // replace sigma on the end of words + str = str + ' '; + str = str.replaceAll('σ-', 'ς-'); + str = str.replaceAll('σ ', 'ς '); + + return str; +} + +// removes non keyword characters [+ . , !] and internal multiple-spaces +// @param txt (string): product description +const clean = (txt) => { + return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space + .replace('!','').replace('\"', '') // remove character + .replace(' ',' ').replace(' ',' '); // remove multiple spaces +} + + +// exports +// --- -- -- - - - + +module.exports = { + keyboardize, + keyb_array, + sanitizeGR, + clean +};
\ No newline at end of file diff --git a/app/utils/match-util.js b/app/utils/match-util.js new file mode 100644 index 0000000..5e8c648 --- /dev/null +++ b/app/utils/match-util.js @@ -0,0 +1,154 @@ +/** + * match utility; + * includes fuzzy and partial match functions too; + * many of them return a match-rate + */ + + +// fuzzy match +//////////////////////////////////////////////////////////////////////////////// + +/** Ngram fuzzy match algorithm + * (simple and fast) + */ +const createNgram = (word, n) => { // Ngram creation + if (word.length <3) return word; + const vector = []; + for (let i = 0; i < word.length-n+1; ++i) { + vector.push(word.slice(i, i + n)); + } + return vector; +}; + +/** similarity + * rates similarity between 2 words + * based on Ngram matches of N = n letters; + * implements a 2-dim check (all a-Ngrams vs all all b-Ngrams) + * + * @param {string} a : first word + * @param {string} b : second word + * @param {int} n : Ngram base + * @returns {float} : match percentage as a float in [0, 1] + */ +const similarity = (a, b, n) => { // Ngram match score + if (a.length > 0 && b.length > 0) { + const aNgram = createNgram(a, n); + const bNgram = createNgram(b, n); + let hits = 0; + for (let x = 0; x < aNgram.length; ++x) { + for (let y = 0; y < bNgram.length; ++y) { + if (aNgram[x] === bNgram[y]) { + hits += 1; + } + } + } + if (hits > 0) { + const union = aNgram.length + bNgram.length; + return (2.0 * hits) / union; + } + } + return 0; +}; + +/** resemblance + * is an alternative similarity rating; + * implements an 1-dim Ngram similarity check + * and it's much faster than similarity() + */ +const resemblance = (a, b, n) => { + if (a.length > n && b.length >= a.length) { + const aNgram = createNgram(a, n); + let hits = 0; + for (let i = 0; i < aNgram.length; ++i) { + if (b.includes(aNgram[i])) { + hits++; + } + } + if (hits > 0) { + // rate resemblance based on hits and length-similarity + return (hits / aNgram.length) * (a.length / b.length); + } + } + return 0; +} + + +// exact and partial match +//////////////////////////////////////////////////////////////////////////////// + +/** is_exact_match + * check if a searching string -> query (string/latin in kb-format) + * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) + * + * @param {string} query: searching string; string/latin in kb-format + * @param {array} chkArr: array of synonyms; (array of utf-8/strings) + * @return {boolean}: true|false + */ +function exact( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (w == query) found = true }); + return found; +} + +function partial( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (w.includes(query)) found = true }); + return found; +} + +/** is exact match + weight rating + * @returns {float} weight rates depth of array when a match is found + */ +function weighted_exact( query, chkArr ) { + let weight = 0; // closer to left/begin rating + let len = chkArr.length; + for(let i = 0; i < len ; i++) { // i ~ depth + if (chkArr[i] == query) { + // weights array depth + weight = (len - i + 1.0) / len; + break; + } + } + return weight; +} + + +/** is partial match + weight rating + * + * @param query (string): searching string; string/latin in kb-format + * @param chkArr (array): array of synonyms; (array of utf-8/strings) + * @returns {float} weight rates both match position and depth of match + * + * (*) optimization NOTE: + * Given the weight `W` and the depth `i`, + * the best weight for next `i` shall be: `(L - (i+1)) / L` + * To be imposibbe to have a better weight, should: + * W > (L - (i+1)) / L => ... => i > (L - L*W - 1) + */ +function weighted_partial( query, chkArr ) { + let rate = 0; + let weight = 0; + let len = chkArr.length; + for( let i = 0 ; i < len ; i++ ) { + let chk = chkArr[i].indexOf(query) + if (chk != -1) { + rate = (len - i) / (len + 2.0 * chk); + weight = rate > weight ? rate : weight; + } + if (i > (len - len * weight - 1)) { + break; // better rating is not possible (*) + } + } + return weight; +} + + +// exports +module.exports = { + exact, + partial, + weighted_exact, + weighted_partial, + similarity, + resemblance +} diff --git a/app/utils/mem-usage.js b/app/utils/mem-usage.js new file mode 100644 index 0000000..766a93e --- /dev/null +++ b/app/utils/mem-usage.js @@ -0,0 +1,20 @@ +/** + * memory usage report utility + */ + +const formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`; + +function report() { + let memoryData = process.memoryUsage(); + + let memoryUsage = { + rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`, + heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`, + heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`, + external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`, + }; + + console.log(memoryUsage); +} + +module.exports = { report } diff --git a/app/utils/url-util.js b/app/utils/url-util.js new file mode 100644 index 0000000..e86ad9b --- /dev/null +++ b/app/utils/url-util.js @@ -0,0 +1,23 @@ +/** + * url utility + */ + +const querystring = require('querystring'); + +function struct(req, url) { + let url_parts = url.split('?'); + let query = (url_parts.length > 1) + ? querystring.decode(url_parts[1]) + : {}; + return { + method: req.method, + host: req.host, + path: url_parts[0], + query: query + } +} + +/** + * exports + */ +module.exports = { struct }
\ No newline at end of file |
