const kb = require('../utils/kb-util.js'); const match = require('../utils/match-util.js'); var _allowFuzzy = true; // enable|disable fuzzy search var n = 2; // Ngram base var _fuzzyLimit = .5; // minimum bigram score for being considered a match /** (Search) SUGGESTIONS ENGINE * --------------------------------------------------------------------------- * * Operates in dual mode; * -- suggestions engine (interactive) * -- classic-like mode (passive) * * TODO: * CRITICAL: (optimization) * Search initialization uses quite a lot of network sources; * thus it should be started in a later time; * lets say ... after `x` seconds * or... when document/core-ui-elements are ready * * @parametres (json) : options * --- * @var keywords_json (sring) : endpoint url of linked keywords structure * @var products_json (string) : endpoint od product descriptions * @var search_tag (string) : selector of field that shall act as typeahead-suggestions * @var visualize_search_results_url (str) : url that will visualize the sended "results-page" * @var debug (bool) : if true sends several debug console messages; if false mesagges are eliminated */ suggestions_module({ keywords_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', products_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', search_tag: '#tagsInput', visualize_search_results_url: '/product_list', debug: ((location.hostname == 'localhost') || (location.hostname == '127.0.0.1')) }); function suggestions_module(options) { /** CONTENTS * * +1: Variables * * +2: Purify string functions * + keyboardize * + sanitize_GR * + clean * + mark_explicit_links * * +3: Supplementary function (vanilla js) * + ajax_get(url, callback) * + createNgram (fuzzy) * + checkSimilarity (fuzzy) * + check_match * + is_exact_match * + match_one * * +4: Actual data loading (async) * * +5: Suggestions Engine (jQuery) * + suggestions_engine * */ console.log('executing suggestions...'); /** 1. VARIABLES * ------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////// var _kwlinks; // keyword links (word-connections; imported via ajax-get) var _products = []; // all products (imported via ajax-get) // setup options var _maxResults = 24; // limit suggestions var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions var _timeout_ms = 100; // time (in ms) for the search engine to find matches (before rendering) var _Ngram_base = 2; // number of N in Ngram spliting algorithm var _isReady = false; // whether the searchbox is ready to be used // product keywords var keywordsURL = options.keywords_json; var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } /** replaces (correcting descriptions) * == construct unequivocally liked words ////////////////////////////////// * ------------------------------------------------------------------------- * * NOTE: TODO: * in future implementations multi-word keywords * may use the non-breaking space as conecting character (\u00A0) instead of dush (-) * (or maybe both of them) * * also TODO: * in future implementaions linked words may passed via some endpoint */ replaces = []; replaceSource = [ '3Α;3-ΑΛΦΑ', '3 ΑΛΦΑ;3-ΑΛΦΑ', 'HEAD & SHOULDERS;HEAD&SHOULDERS', 'HEAD N SHOULDERS;HEAD&SHOULDERS', 'W.K Kellogg; W-K-Kellogg', 'W.K Kellogg;', '7 DAYS;7-DAYS', '7 UP;7UP', '7-UP;7UP', 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', 'COCA COLA;COCA-COLA', 'J P. CHENET; J.P.CHENET', 'J.P. CHENET; J.P.CHENET', 'COCACOLA;COCA-COLA', 'NES CAFE;NESCAFE', 'NES-CAFE;NESCAFE', 'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS', 'PETIT MARSEILLAIS;PETIT-MARSEILLAIS', 'Το Μάννα;Το-Μάννα', 'Χωρίς Γλουτένη;Χωρίς-Γλουτένη', 'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη', 'Χωρίς Αλάτι;Χωρίς-Αλάτι', 'Χωρίς Λακτόζη;Χωρίς-Λακτόζη', 'Χωρίς Συντηρητικά;Χωρίς-Συντηρητικά', 'Χωρίς Αλκοόλ;Χωρίς-Αλκοόλ', 'Χωρίς Kαφεϊνη;Χωρίς-Kαφεϊνη', 'Χωρίς Kαφεΐνη;Χωρίς-Kαφεϊνη', 'Χωρίς Γλυκάνισο;Χωρίς-Γλυκάνισο', 'Χωρίς Ανθρακικό;Χωρίς-Ανθρακικό', 'Υψηλής Παστερίωσης;Υψηλής-Παστερίωσης', 'Ολικής Άλεσης;Ολικής-Άλεσης', 'Ολικής Aλέσεως;Ολικής-Aλέσεως', 'Χαρτί Υγείας;Χαρτί-Υγείας', 'ρολό υγείας;ρολό-υγείας', 'χαρτί τουαλέτας;χαρτί-τουαλέτας', 'Χαρτί Κουζίνας;Χαρτί-Κουζίνας', 'ρολό κουζίνας;ρολό-κουζίνας', 'Μπάρες Δημητριακών;Μπάρες-Δημητριακών', 'Ας Μαγειρέψουμε;Ας-Μαγειρέψουμε', 'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', 'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', 'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ', 'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ', 'ΕΛ ΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', 'ΕΛΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', 'FREE STEP;FREE-STEP', 'EL SABOR;EL-SABOR', 'ELSABOR;EL-SABOR', 'DOUWE EGBERTS;DOUWE-EGBERTS', 'DOUWEEGBERTS;DOUWE-EGBERTS', 'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', 'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', 'SPIN SPAN;SPIN-SPAN', 'SPINSPAN;SPIN-SPAN', 'CRETA-FARMS;CRETA-FARM', 'CRETA-FARM;CRETA-FARM', 'CRETAFARM;CRETA-FARM', 'Ολες-τις-Χρήσεις;Ολες-τις-Χρήσεις', 'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη', 'φρουι ζελε, φρουί-ζελε', 'DR BECKMANN, DR-BECKMANN' ]; replaceSource.forEach( it => { st = sanitize_GR( it.toLowerCase() ).split(';'); replaces.push({ src: ' '+ st[0] +' ', // encolse between spaces trg: ' '+ st[1] +' ' // to separate from before/after words }); }); /** mark_explicit_links * * mark linked words shall be handled as one-(key)word * also, edit common mistakes with suggested replaces * * @param str * @return */ function mark_explicit_links(str) { str = ' '+ str +' '; replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); }); return str.replaceAll(' ', ' ').trim(); } /** 3. SUPPLEMENTARY FUNCTIONS * ------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////// // TODO: // exclude some generic non-critial words when proccessing user's query // ** example code to work with: // var ignoredKeys_kb = []; // keywords to ignore (in kb-format) // 'μας με σε για του της των από στο στον &'.split(' ').forEach(w => { ignoredKeys_kb.push(keyboardize(w)); }); // pure JS ajax GET request; return data as JSON // no fancy things like UTF8; if needed use base64 function ajax_get(url, callback) { var xmlhttp = new XMLHttpRequest(); xmlhttp.onreadystatechange = function() { if (xmlhttp.readyState == 4 && xmlhttp.status == 200) { try { var data = JSON.parse(xmlhttp.responseText); } catch(err) { if (options.debug) console.log(err.message + " in " + xmlhttp.responseText); return; } callback(data); } }; xmlhttp.open("GET", url, true); xmlhttp.send(); } // Search Endine's match functions ////////////////////////////////////////////////////////////////////////////// /** check_match * --- * check if a searching string -> query (string/latin in kb-format) * matches an item of the array of synonyms -> chkArr (array of utf-8/strings) * * + option to use fyzzy (bigram) match * * @return: matched string (utf-8) --or-- false (if not matched) */ function check_match( query, chkArr, fuzzy = false ) { var result = ''; var found = false; if (query == ' ') return chkArr[0]; chkArr.forEach( chk => { if (!found) { chk_kb = keyboardize(chk); if ( (chk_kb.indexOf( query ) !== -1) || (fuzzy && (match.similarity(chk_kb, query, n) > _fuzzyLimit)) ) { found = true; result = chk; } } }); return found ? result : false; } /** is_exact_match * * check if a searching string -> query (string/latin in kb-format) * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) * * @param query (string): searching string; string/latin in kb-format * @param chkArr (array): array of synonyms; (array of utf-8/strings) * @return (boolean): true|false */ function is_exact_match( query, chkArr ) { found = false; chkArr.forEach( w => { if (keyboardize(w) == query) found = true }); return found; } /** match_one * * check if at-least-ONE item from an array of query-words -> qArr (array of string/latin in kb-format) * matches any item of an array of synonyms -> chkArr (array of utf-8/strings) * * @return (boolean) true|false */ function match_one( qArr, chkArr ) { found = false; qArr.forEach( query => { chkArr.forEach( w => { if (keyboardize(w) == query) found = true }); }); return found; } /** 4. ACTUAL DATA LOADING * ------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////// // TODO: // control completion of async loads via .then() rather by this custom structrure // Need to rewrite the folllowing code ............................. from here // ........................................................................... // ........................................................................... /** workline * ------------------------------------------------------------------------- * custom object/data-structure * to track status/completion of async svents * (it does the job using a special set method) */ var workline = { trackerJL : 0, set jsonLoaded(x) { this.trackerJL = x; // fire event on certain values if (x == 3) { if (options.debug) console.info('suggestion-engine requirements fulfilled'); _isReady = true; // code to execute // ... } }, get jsonLoaded() { return this.trackerJL; } }; /** LOAD DATA (from endoints) * --------------------------------------------------------------------------- */ ajax_get( keywordsURL, function(data) { // get _kwlinks _kwlinks = data; if (options.debug) console.log('...keywords loaded;'); workline.jsonLoaded++; }); function load_store_products(storeID) { ajax_get(options.products_json, function(data) { // get stor's _products _products = data; _products.forEach(p => {p.kb = keyboardize(p.w).toLowerCase()} ); if (options.debug) console.log('...products loaded;'); workline.jsonLoaded++; CURRENT_STOREs_CATALOG = storeID; }); } var STORE = { id: 904 }; if (STORE.id != 0) load_store_products(STORE.id); // ........................................................................... // ........................................................................... // ................................................................ up to here /** 5. SUGGESTIONS ENGINE * ------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////// workline.jsonLoaded++; // notify workline that jQuery is ready!! // suggestions engine ////////////////////////////////////////////////// // --- function suggestions_engine(qOrig) { var results = []; // suggestions to respond var proList = []; // list of products (for all suggestions) var commonL = []; // list of common products (for multiple suggestions) var possibleNext = []; // list of possible next suggestions var root, last; var space_ended = (qOrig.slice(-1) == ' ') ? true : false; // clean and sanitize and mark links onto q(uery) string var q = mark_explicit_links( sanitize_GR( clean_text(qOrig) ) ); // TODO: // construct direct-linked words // = do unequivocally replaces // steps: // 1. replace accented vowels with non accented ones // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'` // --- -- -- - - - // test: // oneliner: var str = 'αλφα βητα ΑΛΦΑ Βητα world'; var src = 'αλφα βητα'; var reg = new RegExp(src, "gi"); var replacedOnce = str.toLowerCase().replace(reg, 'α-β'); console.log(replacedOnce); var qAr = q.split(' '); // split to words if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array if (qAr.slice(-1) == "") { qAr.pop(); } if (options.debug) console.log(qAr); if (qAr.length == 1) { // suggest 1st word ////////////////////// var kbq = kb.keyboardize(q); // loop through root-words // ... to match all possible suggestions; _kwlinks.forEach( it => { chk = check_match(kbq, it.w, true); if ( chk != false ) { results.push({ w: chk }); // ... also keep possible products in a list it.c.forEach( wo => { // for every Word-Link-Node if (proList.length < _maxResults + 2) { // NOTE: about +2 (bellow) // concatenate this-suggestion's product sub-list (wo.p) // to all-suggestions posiible products-list (proList) proList = proList.concat(wo.p); // keep unique products in the products-list proList = proList.filter((item, i, ar) => ar.indexOf(item) === i); } }); } }); } if (qAr.length == 2) { // suggest 2nd word ///////////////////////// root = qAr[0]; last = qAr[1]; kbroot = kb.keyboardize(root); kblast = kb.keyboardize(last); // TODO: // change [_kwlinks.forEach] to for loop // --- _kwlinks.forEach( it => { // locate the ... if (match.exact(kbroot, kb.keyb_array(it.w))) { // exact match of root-word it.c.forEach ( wo => { // loop the word-links ... chk = check_match(kblast, wo.w, true); if ( chk !== false ) { // if a match is found results.push({ // keep suggestion w: root +' '+ chk, f: wo.f }); if (proList.length < _maxResults + 2) { // plus... proList = proList.concat(wo.p); // keep products-list proList = proList.filter((item, i, ar) => ar.indexOf(item) === i); } } }); } }); } if (qAr.length > 2) { // suggest N-th word (N>2) /////////////// root = qAr.shift(); // isolate first item of qAr last = qAr.pop(); // isolate last item too // now qAr includes only the items after root and before last; // so qAr includes all already selected suggestions (but root) // prepare/cache kb-formated string for any key we're going to use var kb_qAr = []; // array of selected suggestions in kb-format qAr.forEach( w => { kb_qAr.push(keyboardize(w)); }) kbroot = keyboardize(root); kblast = keyboardize(last); // TODO: // change [_kwlinks.forEach] to for loop // --- _kwlinks.forEach( it => { if (is_exact_match(kbroot, it.w)) { // locate root-word // calculate list of common items/products (commonL) // for selected suggestions is1stOcc = true; // 1st occurance flag it.c.forEach ( swo => { if (match_one(kb_qAr, swo.w)) { // swo is one of the already selected suggestions // so... update the common-(products)-L(ist) if (is1stOcc) { commonL = swo.p; // init list (on 1st occurance) is1stOcc = false; } else { // caclulate list of common products // = intersection of (so-far) commonL and swo.p // commonL = commonL.filter(value => swo.p.includes(value)); commonL = commonL.filter(function(n) { return swo.p.indexOf(n) !== -1; }); } } else { // if swo is not already selected // then This is a possible NEXT suggestion possibleNext.push(swo); } }); // Now that we have all the possible next suggestions // we'll match them with the last word of the query possibleNext.forEach( poss => { // for tthe possible next suggestions // if last-word-of-query matches possible word(s) // and list of word's products has commons with commonL // then THiS is a Valid-Next-Suggestion chk = check_match(kblast, poss.w, true); if (chk !== false) { // check intersection of commonL and suggestion's product-lists tempL = commonL.filter(value => poss.p.includes(value)); if (tempL.length > 0) { results.push({ w: root +' '+ qAr.join(' ') +' '+ chk, f: poss.f }); // update proList too proList = proList.concat(tempL); } } }); } }); } // NOTE: about +2 (vs +1) // after having calculated next suggestions and a banch of possible products // the proccedure is going to decide what data will return; // if pro(ducts)List includes less items than maximum suggestions // ... this will be the array to return. So // ... +2 ensures that this list will not become shorter than this limit // calculate unique products // (credit: https://stackoverflow.com/questions/11246758/) let unique = proList.filter((item, i, ar) => ar.indexOf(item) === i); if (unique.length < (_maxResults +1)) { // IF list is small ...... // serve products instead of suggestions // ... results = []; notOnThisStore = [] unique.forEach( pr => { for (i=0 ; i< _products.length-1 ; i++) { // for makes things faster pi = _products[i]; if (pi.id == pr) { results.push(pi); break; } } }); } else { // remove forced link character '-' from suggestions ..... var dirty_results = results; results = []; for (i=0 ; i< dirty_results.length ; i++) { results.push({ w: dirty_results[i].w.replaceAll('-', ' ') }) } } return results; } // simple, fast search products by numeric code property // --- function search_by_code( num ) { const str = num.toString(); var results = []; var p; for (i=0 ; i< _products.length-1 ; i++) { p = _products[i]; if (results.length > _maxResults) { break; } if ( (p.bp+'-'+p.bc).indexOf(str) !== -1 ) { results.push(p) } } return results; } // suggestions caller (router function) // arguments: // * qOrig : original query string // ** list : callback array structure to host results // --- var isuggest = function(qOrig, list) { var results; qSanit = sanitize_GR(qOrig); // sanitize greel accended chars // TODO: // check if all source-lists are ready // if not you need to wait ... // via async promishes or synced timouts // if query seems to be some king of 'code/id' if (qSanit.length>2 && qSanit.match(/^[0-9]+$/) != null) { results = search_by_code(qOrig); } else { // string match procedure with suggestions engine results = suggestions_engine(qOrig); } list(results); } /** jQuery(function() { // on document ready code ////////////////////////// /** // UI-dependent code // uses reference to specific document element (passed via options) // ----------------------------------------------------------------------- const searchBox = $(options.search_tag); searchBox.typeahead( { hint: true, highlight: true, minLength: 1 }, { limit: _maxResults, name: 'kwlinks', displayKey: 'w', source: isuggest, templates: { suggestion: function(data) { if (data.id) return '
'+ data.w +'
'; return '
'+ data.w +'
'; }, empty: '
Δεν υπάρχει στο κωδικολόγιο του καταστήματος
' } } ) .bind("typeahead:selected", function(obj, datum, name) { if (datum.hasOwnProperty('id')) { // ** selected: PRODUCT // $('#js-add-product-to-order').attr('disabled', false); // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img); if (options.debug) console.log('selected: ', datum.id, datum.w); // TODO: GOTO product page window.location.href = 'product_list?product=id-' + datum.id; // $("#product-quantity").trigger('focus'); } else { // selected: SUGGESTION $('.typeahead').typeahead('val','').trigger('blur'); $('.typeahead').typeahead('val', datum.w +' ') .trigger("query"); // give some time to the engine to calculate results // then fire focus again... setTimeout(() => { $('.typeahead').trigger('focus'); }, _timeout_ms); } }) .bind("typeahead:cursorchange", function( event, obj) { // track cursor-chane to handle special keys after a final product is selected // TODO: // handle exception where obj is undefined; // this occures... // when cusror returns from suggestions list back to the search field if (typeof obj === 'undefined') cursor_on = { none: true }; else if (obj.hasOwnProperty('id')) cursor_on = obj; else cursor_on = { none: true }; }); // TRACK user search attempt /////////////////////////////////////////////// $('.typeahead').on('keyup', function(e) { if (_isReady) { // console.log('on:', cursor_on); // console.log('key:', e.key); if ((e.key == ' ') && cursor_on.hasOwnProperty('id')) { // product is actually selected var datum = cursor_on; // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img); $("#product-quantity").trigger('focus'); e.preventDefault(); } else { if (e.key === "Enter") { // if curson is not on some option if ((typeof curson_on === 'undefined') || (curson_on.none == true)) { // do a common search if (options.debug) console.log('Do a Non-Suggestions search', searchBox.val()); // DEPRICATED: var location = encodeURI('/product_list?productSearch=%'+ searchBox.val() +'%'); search_results = common_search(searchBox.val()); } else { // launch a product page var location = '/product_list?product=id-'+ cursor_on.id; if (options.debug) console.log('product', cursor_on.id, cursor_on.w); console.log('go to product', location) } // window.location.href = location; } } } }); }); */ /* function common_search(query) { // clear ; sanitize ; split var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' '); // if last item is empty, remove it if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop() var results = _products; // for each key fitler results qAr.forEach( key => { results = key_sublist(key, results) }); // echo products (and prepare list to POST) var list_ = []; results.forEach( item => { if (options.debug) console.log(item.id, ':', item.w); list_.push(item.id) }) let l = list_.join(','); var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`); if ($('.js-categories.selected').length || $('.js-checkout.selected').length ) { url = encodeURI(`api/v1/products?eys_code=${l}&q=${query}`); console.log('common search : fetch results > history.push', url, query); fetchSearchProducts(url, query); window.history.pushState('search', null, '/product_list?search='+query); } else { console.log('common search: search query > location = search') window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`); } // Redirect with POST (template) // --- -- -- - - - // var form = $([ // `
`, // '', // '
' // ].join('')); // $('body').append(form); // form.submit(); } */ /** return from list only items that include 'key' */ function key_sublist(key, list) { var result = []; list.forEach( item => { if (item.kb.includes(key)) { result.push(item); } }); return result; } }