From 575ca536c237eff331998714cdbebaf88fe3b5c6 Mon Sep 17 00:00:00 2001 From: Geo Halkiadakis Date: Tue, 9 Apr 2024 19:12:30 +0300 Subject: step-1: import main search implementations into node --- pieces/search.js | 730 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 730 insertions(+) create mode 100644 pieces/search.js (limited to 'pieces/search.js') diff --git a/pieces/search.js b/pieces/search.js new file mode 100644 index 0000000..6606272 --- /dev/null +++ b/pieces/search.js @@ -0,0 +1,730 @@ +/** Search Engine + * --------------------------------------------------------------------------- + * + * TODO: + * CRITICAL: (optimization) + * Search initialization uses quite a lot of network sources; + * thus it should be started in a later time; + * lets say ... after `x` seconds + * or... when document/core-ui-elements are ready + * + * @parametres (json) : options + * --- + * @var {string} products_json : endpoint od product descriptions + * @var {string} search_tag : selector of field that shall act as typeahead-suggestions + * @var {string} visualize_search_results_url : url that will visualize the sended "results-page" + * @var {int} max_list : max-size of (rated) results expected + * @var {int} tolerance + * @var {boolean} debug : if true sends several debug console messages; if false mesagges are eliminated + */ + +retrosearch_module({ + products_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', + search_tag: '#tagsInput', + visualize_search_results_url: '/product_list', + max_list: 24, + tolerance: 42, + debug: ((location.hostname == 'localhost') || (location.hostname == '127.0.0.1')) +}); + + + +function retrosearch_module(options) { + + /** CONTENTS + * + * +1: Variables + * + * +2: Purify string functions + * + keyboardize + * + sanitize_GR + * + clean + * + * +3: Supplementary function (vanilla js) + * + ajax_get(url, callback) + * + createNgram (fuzzy) + * + checkSimilarity (fuzzy) + * + check_match + * + is_exact_match + * + match_one + * + * +4: Actual data loading (async) + * + * +5: Suggestions Engine (jQuery) + * + suggestions_engine + * + */ + + if (options.debug) console.log(`preparing retrosearch... (${+ new Date()})`); + + + /** 1. VARIABLES + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + var _kwlinks; // keyword links (word-connections; imported via ajax-get) + var _products = []; // all products (imported via ajax-get) + + // setup options + var _maxResults = options.max_list; // limit suggestions + var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions + var _timeout_ms = 100; // time (in ms) for the search engine to find matches (before rendering) + var _allowFuzzy = true; // enable|disable fuzzy search + var _fuzzyLimit = .5; // minimum bigram score for being considered a match + var _Ngram_base = 2; // number of N in Ngram spliting algorithm + var _isReady = false; // whether the searchbox is ready to be used + + // product keywords + var keywordsURL = options.keywords_json; + + var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } + + + + /** 2.PURIFY STRING FUNCTIONS + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + /** keyboardize + * ------------------------------------------------------------------------- + * translates string to keyboard-latin keys + * (the ones that used whan typing each letter of the word) + * + * @param str (string): original string (utf8 of latin or greek subgroups) + * @return (string): latin/ascii equivalent string + */ + + // cache (keeo in global) any-character to keyboard-latin mapping + // --- -- -- - - - + var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + const map = new Map(); + for (var i=0; i { + ap = pair.split(' '); + accented_vowels.push({ + a: ap[0], // accented + p: ap[1] // pure = non accended + }); + }) + + // the actual `sanitize_GR` function code + // --- -- -- - - - + function sanitize_GR(str) { + + str = str.toLowerCase(); + + // replace accended vowels with pure ones + accented_vowels.forEach( v => { + str = str.replaceAll(v.a, v.p); + }); + + // replace sigma on the end of words + str = str + ' '; + str = str.replaceAll('σ-', 'ς-'); + str = str.replaceAll('σ ', 'ς '); + + return str; + } + + + /** clean text + * --- -- -- - - - + * removes non keyword characters [+ . , !] and internal multiple-spaces + * @param txt (string): product description + */ + function clean_text(txt) { + return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space + .replace('!','').replace('\"', '') // remove character + .replace(' ',' ').replace(' ',' '); // remove multiple spaces + } + + + + /** 3. SUPPLEMENTARY FUNCTIONS + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + // TODO: + // exclude some generic non-critial words when proccessing user's query + // ** example code to work with: + // var ignoredKeys_kb = []; // keywords to ignore (in kb-format) + // 'μας με σε για του της των από στο στον &'.split(' ').forEach(w => { ignoredKeys_kb.push(keyboardize(w)); }); + + // pure JS ajax GET request; return data as JSON + // no fancy things like UTF8; if needed use base64 + function ajax_get(url, callback) { + var xmlhttp = new XMLHttpRequest(); + xmlhttp.onreadystatechange = function() { + if (xmlhttp.readyState == 4 && xmlhttp.status == 200) { + try { + var data = JSON.parse(xmlhttp.responseText); + } catch(err) { + if (options.debug) console.log(err.message + " in " + xmlhttp.responseText); + return; + } + callback(data); + } + }; + xmlhttp.open("GET", url, true); + xmlhttp.send(); + } + + + // callback function for sorting resulrs per r (=rating) property + function compare_rate(a,b) { + return (a.r < b.r); + } + + + /** vwOverUnder + * vertical-wise over/under + * + * @param {document.element} el : element to check + * @param {document.element} container : container element (compairing) + * @return {int} .. [-1|0|+1] + * ..: 0 if element's heignt is completely into container's viewport + * ..: -1 if element's heignt is over container's viewport + * ..: +1 if element's heignt is under container's veiwport + */ + function vwOverUnder(el, container) { + let eR = el.getBoundingClientRect(); + let cR = container.getBoundingClientRect(); + console.log('er|cr', eR, cR); + + let result = -1; // default: element is under container's viewport + + if (eR.y >= cR.y && eR.y + eR.height <= cR.y + cR.height - options.tolerance) { + result = 0 // fully inside + + } else if (eR.y < cR.y) { + result = -1 + } + + return result; + } + + + + + /** 4. ACTUAL DATA LOADING + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + + // TODO: + // control completion of async loads via .then() rather by this custom structrure + // Need to rewrite the folllowing code ............................. from here + // ........................................................................... + // ........................................................................... + + + /** workline + * ------------------------------------------------------------------------- + * custom object/data-structure + * to track status/completion of async svents + * (it does the job using a special set method) + */ + var workline = { + + trackerJL : 0, + set jsonLoaded(x) { + this.trackerJL = x; + + // fire event on certain values + if (x == 2) { + if (options.debug) console.info('suggestion-engine requirements fulfilled'); + _isReady = true; + // code to execute + // ... + } + }, + get jsonLoaded() { return this.trackerJL; } + }; + + + /** LOAD DATA (from endoints) + * --------------------------------------------------------------------------- + */ + + + function load_store_products(storeID) { + let prData = getSessionObj('prd'); + if (prData !== null) { + _products = prData; + if (options.debug) console.log(`...products fetched from cache (${+ new Date()})`); + workline.jsonLoaded++; + + } else { + ajax_get(options.products_json, function(data) { // get stor's _products + _products = data; + _products.forEach(p => {p.kb = keyboardize(p.w).toLowerCase()} ); + setSessionObj('prd', _products); + if (options.debug) console.log(`...products loaded; (${+ new Date()})`); + workline.jsonLoaded++; + CURRENT_STOREs_CATALOG = storeID; + }); + } + } + + var STORE = { id: 904 }; + + if (STORE.id != 0) load_store_products(STORE.id); + + // ........................................................................... + // ........................................................................... + // ................................................................ up to here + + + + + + /** 5. SEARCH ENGINE + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + workline.jsonLoaded++; // notify workline that jQuery is ready!! + + // suggestions engine ////////////////////////////////////////////////// + // --- + function suggestions_engine(qOrig) { + var results = []; // suggestions to respond + var pot = []; pot.length = 0; + + var space_ended = (qOrig.slice(-1) == ' ') ? true : false; + + // clean and sanitize and mark links onto q(uery) string + var q = keyboardize( sanitize_GR( clean_text(qOrig.trim()) ) ).trim(); + + // TODO: + // construct direct-linked words + // = do unequivocally replaces + // steps: + // 1. replace accented vowels with non accented ones + // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'` + + var qAr = q.split(' '); // split to words + /// if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array + /// + /// if (qAr.slice(-1) == "") { + /// qAr.pop(); + /// } + + if (options.debug) console.log('*** init SEARCH QUERY:', qOrig, 'search:', qAr); + + pot = _products; // potential results // NOTE: CRITICAL: BY REFERENCE + + var wi = 0; // word index (from list) + var wc = qAr.length; + + qAr.forEach( w => { + if (options.debug) console.log('...testing', w); + + let sf = []; // (matches) so far + let mi; // position of match + wi++; + + pot.forEach( it => { + let matched = false; + let tester = ' '+ it.kb + ' '; + + // reset previous history and ratings + if (wi == 1) { + it.r = 0; + it.history = []; + } + + + // rate word-match > start-match > simple-match + // ... up to 8 points + + if (tester.indexOf(' '+ w +' ') != -1) { + it.r += 9; + it.history.push({ w: w, rate: 9 }); + matched = true; + } + else if (tester.indexOf(' '+ w) != -1) { + it.r += 5; + it.history.push({ w: w, rate: 5 }); + matched = true; + } + else if (tester.indexOf(w) != -1) { + it.r += 2; + it.history.push({ w: w, rate: 2 }); + matched = true; + } + + // rate `near-to-start` matching .. up to 7p + // rate `earlyness` of word in query .. up to 7p + + if ((mi = tester.indexOf(' '+w)) != -1) { + let fc1 = 100 - ((mi < 99) ? mi : 99); // near-to-start factor + let fc2 = wc - wi + 1; // query earlyness factor + let r1 = Math.floor(7*fc1/100); + let r2 = Math.floor(7*fc2/wc); + + it.r += (r1 + r2); + it.history.push({ w: w, left: [fc1, r1], early: [fc2, r2] }); + } + if (matched) sf.push(it); + }); + + if ((sf.length > (_maxResults + Math.floor(_maxResults/2))) + || (wi == 1) ) { + // ..if pot has a fair amount (= max + 50%) of results + // ..or these are results of '1st-query-word' + // set sf as new source + pot.lenght = 0; pot = []; + pot = JSON.parse(JSON.stringify(sf)); // copy by value + + } else { + // else.. keep the source list and increase of 'so-far rating' + // console.log('found small list', sf, pot) + pot.forEach( it => { + sf.forEach( si => { + if (it.id == si.id) { + it.r += 10; + it.history.push({ w: w, plus: '+10'}); + } + }); + }); + } + }); + + // sort results, get max-list of best rated + results = (pot.length > _maxResults) + ? pot.sort(compare_rate).slice(0, _maxResults) + : pot.sort(compare_rate) + + if (options.debug) console.log(results); + + return results; + } + + // suggestions caller (router function) + // arguments: + // * qOrig : original query string + // ** list : callback array structure to host results + // --- + var isuggest = function(qOrig, list) { + var results; + + qSanit = sanitize_GR(qOrig); // sanitize greek accended chars + + // TODO: + // check if all source-lists are ready + // if not you need to wait ... + // via async promishes or synced timouts + + /// // DEPRECATED: Suppport for eys-code search + /// // if query seems to be some king of 'code/id' + /// if (qSanit.length>2 && qSanit.match(/^[0-9]+$/) != null) { + /// results = search_by_code(qOrig); + /// + /// } else { + /// // string match procedure with suggestions engine + results = suggestions_engine(qOrig); + /// } + + list(results); + } + + // var rsfoot = document.createElement('div'); + // rsfoot.setAttribute("id", "retrosearch-footer"); + // $('#retrosearch-footer').html('Multi Search'); + + /* + jQuery(function() { // on document ready code ////////////////////////// + + // UI-dependent code + // uses reference to specific document element (passed via options) + // ----------------------------------------------------------------------- + const searchBox = $(options.search_tag); + searchBox.typeahead( + { + hint: true, + highlight: true, + minLength: 1 + }, + { + limit: _maxResults, // +1 for extra button + // name: 'kwlinks', + displayKey: 'w', + source: isuggest, + templates: { + suggestion: function(data) { + return `
${data.w}Alfa, Beta and Gamma
`; + }, + + footer: '
Multi Searh
', + + empty: '
Δεν υπάρχει στο κωδικολόγιο του καταστήματος
' + } + } + ) + .bind("typeahead:selected", function(obj, datum, name) { + if (datum.hasOwnProperty('id')) { // ** selected: PRODUCT + + // $('#js-add-product-to-order').attr('disabled', false); + // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img); + if (options.debug) console.log('selected: ', datum.id, datum.w); + + // GOTO product page + window.location.href = 'product_list?product=id-' + datum.id; + + // $("#product-quantity").trigger('focus'); + + } + else { // selected: SUGGESTION + + $('.typeahead').typeahead('val','').trigger('blur'); + $('.typeahead').typeahead('val', datum.w +' ') + .trigger("query"); + // give some time to the engine to calculate results + // then fire focus again... + setTimeout(() => { $('.typeahead').trigger('focus'); }, _timeout_ms); + } + + }) + .bind("typeahead:cursorchange", function( event, obj) { + // track cursor-chane to handle special keys after a final product is selected + + if (typeof obj === 'object' + && !Array.isArray(obj) && obj !== null) { + + + // console.log(obj); + // var liObj = $(`.tt-menu .tt-dataset .tt-suggestion[data-id="${obj.id}"]`) + // var c = liObj.html(); + // liObj.html = c + '\n\n_'; + // liObj.html = c; + + var y = $('.tt-menu').scrollTop(); + let ou = vwOverUnder( + document.querySelector(`.tt-menu .tt-suggestion[data-id="${obj.id}"]`), + document.querySelector('.tt-menu') + ); + console.log('y:',y, 'obj:', obj, 'o/u:', ou); + $('.tt-menu').scrollTop( y - ou * options.tolerance); + + // TODO: + // need to park the cursorChanged-div into tt-menu veiwport (??) + // check: https://stackoverflow.com/questions/75002332/check-if-child-element-is-100-visible-inside-a-parent-div-that-has-overflow-hid + + } + + + // TODO: + // handle exception where obj is undefined; + // this occures... + // when cusror returns from suggestions list back to the search field + + if (typeof obj === 'undefined') cursor_on = { none: true }; + else if (obj.hasOwnProperty('id')) cursor_on = obj; + else cursor_on = { none: true }; + }) + .bind('typeahead:opened', function(e) { + console.log('attach multisearch option if not exist'); + }); + + // TRACK user search attempt /////////////////////////////////////////////// + + + // $('.typeahead').on('typeahead:opened', function(e) { + // console.log('attach multisearch option if not exist'); + // }); + + $('.typeahead').on('keyup', function(e) { + if (_isReady) { + // console.log('on:', cursor_on); + // console.log('key:', e.key); + if ((e.key == ' ') && cursor_on.hasOwnProperty('id')) { + + // product is actually selected + var datum = cursor_on; + + // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img); + $("#product-quantity").trigger('focus'); + e.preventDefault(); + + } else { + if (e.key === "Enter") { + + // if curson is not on some option + if ((typeof curson_on === 'undefined') || (curson_on.none == true)) { + + // do a common search + if (options.debug) console.log('Do a Non-Suggestions search', searchBox.val()); + + // DEPRICATED: var location = encodeURI('/product_list?productSearch=%'+ searchBox.val() +'%'); + + // search_results = common_search(searchBox.val()); + // TODO: + console.log('DO A COMMON_SEARCH'); + + + } else { + // launch a product page + var location = '/product_list?product=id-'+ cursor_on.id; + if (options.debug) console.log('product', cursor_on.id, cursor_on.w); + + // TODO: + console.log('GO TO PRODUCT PAGE', location) + } + // window.location.href = location; + } + } + } + }); + + }); + */ + + // sub-module (start) + //////////////////////////////////////////////////////////////////////////// + + function update_common_search_results(q, results) { + let queries = getSessionObj('sr'); + let newSRlist = []; + let isnewQ = true; + if (queries === null) { + setSessionObj('sr', [{ + q: q, + result: result, + t: + new Date() + }]); + return true; + + } else { + + queries.forEach(it => { + if (it.q == q) { + newSRlist.push({ + q:q, + result: result, + t: + new Date() + }); + isnewQ = false; + } else { newSRlist.push(it); } + }); + + if (isnewQ) { + newSRlist.push({ + q:q, + result: result, + t: + new Date() + }); + } + + return true; + } + } + + + + //////////////////////////////////////////////////////////////////////////// + // sub-module (end) + + + function common_search(query) { + // clear ; sanitize ; split + var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' '); + + // if last item is empty, remove it + if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop() + + var results = _products; + + // for each key fitler results + qAr.forEach( key => { + results = key_sublist(key, results) + }); + + // echo products (and prepare list to POST) + var list_ = []; + results.forEach( item => { + if (options.debug) console.log(item.id, ':', item.w); + list_.push(item.id) + }) + + // *** TODO: keep results in local storage (or on session storage) + + // update_common_search_results(query, list_); + + let l = list_.join(','); + var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`); + + console.log('common search: search query > location = search') + window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`); + + } + + /** return from list only items that include 'key' + */ + function key_sublist(key, list) { + var result = []; + list.forEach( item => { + if (item.kb.includes(key)) { + result.push(item); + } + }); + + return result; + } + +} + + +/** test script + * --- + * pl = getSessionObj('prd'); wl = ['coca', 'xvris', 'zaxarh', '2x1']; + * pot = pl; + * console.log(+ new Date()); + * wl.forEach( w => { + * let sf = []; + * for(i=0 ; i < pot.length; i++) { if (pot[i].kb.includes(w)) sf.push(pot[i]); } + * pot.lenght = 0; pot = sf; + * }); + * console.log(+ new Date(), pot); + * + * + ** test performance for vs foreach + * --- + * pl = getSessionObj('prd'); console.log('started', + new Date()); + * for(j=0; j<50; j++) { for(i=0; i { let x = it.kb; }); } + * console.log('ended', + new Date()); +**/ \ No newline at end of file -- cgit v1.2.3