diff options
| -rw-r--r-- | code-examples.py | 24 | ||||
| -rw-r--r-- | products-dict-v3.py | 270 | ||||
| -rw-r--r-- | products-dictionary.py | 228 | ||||
| -rw-r--r-- | search-v2.html | 336 | ||||
| -rw-r--r-- | search-v3.html | 422 | ||||
| -rw-r--r-- | search.html | 308 |
6 files changed, 1588 insertions, 0 deletions
diff --git a/code-examples.py b/code-examples.py new file mode 100644 index 0000000..d8d0042 --- /dev/null +++ b/code-examples.py @@ -0,0 +1,24 @@ +# test +a = 1 +b = 4 + +def addto(x, l) : + l.append({ + "n" : x, + "c": [] + }) + for it in l : + if it["n"] == 4 : + subl = it["c"] + subl.append(x) + it['c'] = subl + + +malist = [] + +malist.append({ "n" : a }) +print(malist) + +addto(b, malist) +addto(b, malist) +print(malist) diff --git a/products-dict-v3.py b/products-dict-v3.py new file mode 100644 index 0000000..6f37de9 --- /dev/null +++ b/products-dict-v3.py @@ -0,0 +1,270 @@ +## LIBRARIES +# ////////////////////////////////////////////////////////////////////////////// + +import pandas as pd # pandas for excel reading +import re # regex +import json # json +import os.path # ... + + +## LOCAL FUNCTIONS +# ////////////////////////////////////////////////////////////////////////////// + +# do-me-INTeger +# --- +def domeInt(x) : + if isinstance(x, str) : # if string + return int(x.strip()) + if isinstance(x, float) : # if float + return round(x) + return x # otherwise is int already + +# do-me-Float +# --- +def domeFloat(x) : + if isinstance(x, str) : + return float(x.strip()) + else : + return x + 0.00 # make sure that result is float + + +def cleanText(x) : + removeList = [ ' με ', ' σε ', ' για ', ' του ', ' της ', ' των ', ' από ', ' ΜΕ ', ' ΣΕ ', ' ΓΙΑ ', ' ΑΠΟ ', '&', '.', ',', '!', '(', ')', '[', ']', '\'', '\"' ] + + for r in removeList : + x = x.replace(r, ' ') + + x.replace(' ', ' ') # remove spare spaces + x.replace(' ', ' ') + x.replace(' ', ' ') + + return x + + +# function isSignificant +# decides if the term is significant to be indexed; +# a term is significant if does not contain digit-chars +# --- +def isSignificant(x) : + # fisrts exclude some notable exceptions + if x in ['7UP', '3ΑΛΦΑ'] : + return True + + return not bool(re.match("\S*\d+\S*", x)) + + +def kbLatinString( txt ) : + maTable = txt.maketrans( + "ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊϋΆΈΉΊΌΎΏΪΫ", + "sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviyaehioyviy" + ) + return txt.translate(maTable).lower() + + +# set root-keyQ: w (if not exist) +# update frequency: f +# into list: l +# NOTE: in this version, +# comparison is based on the *keyboard* format +## --- +def rootKey ( w, f, l ) : + keyExists = False + kbW = kbLatinString(w) + + for it in l : + if it['kb'] == kbW : + keyExists = True + it['f'] += f + if w not in it['alt'] : + it['alt'].append(w) + + if keyExists == False : + l.append({ + 'w' : w, + 'f' : f, + 'alt' : [ w ], + 'kb' : kbW, + 'c' : [] + }) + + + +# connect keys: a , b +# of product with id: i +# with frequency: f +# into list: l +## --- +def connectKeys( a, b, i, f, l ) : + if a == b : + return False ## exclude just-in-case + + for it in l : + if it['w'] == a : + + # found: a; + # lets update the connection to: b + bExists = False + + for jt in it['c'] : + if jt['kb'] == kbLatinString(b) : + bExists = True + # update the connection's data + jt['f'] += f + jt['p'].append(i) + + if bExists == False : + # create connection with word: b + it['c'].append({ + 'w': b, + 'kb': kbLatinString(b), + 'f': f, + 'p': [ i ] + }) + + + +## LOCAL CONSTANTS +# ////////////////////////////////////////////////////////////////////////////// + +_COL = { + # -- main info + 'freq' : 0, # frequency (based on recent orders) + 'pid' : 1, # product id + 'brand' : 2, # brand + 'barcd' : 3, # barcode + 'sklcd' : 4, + 'eyscd' : 5, + 'descr' : 6, # product description + 'sap2' : 7 # SAP category level-2 id +} + + + + +## SET SOURCE and EXPORT FileNames +# ------------------------------------------------------------------------------ +# location of excel file +loc = "./data/PRODucts2search-wBrands.xlsx" + +print("default filename:", loc) +newXLfile = input("input other Excel filename [enter to keep default]: ") + +if newXLfile != "" and os.path.exists(newXLfile): + loc = newXLfile +else : + print(newXLfile, "is not a file; default is kept;") + +## baseEXPORTname = input("Base export name: ") + + + + +## Read data +# ////////////////////////////////////////////////////////////////////////////// + +df = pd.read_excel(loc) # read data from excel file + +rows = df.iterrows() # set rows list + + +# --- Lists to fill +keywords_ = [] # all data +minilist_ = [] +products_ = [] + +## keywords format: +## [ +## { +## w : 'fresh', +## alt : [ 'Fresh', 'FRESH', 'fresh' ] +## kb : +## f : 150, +## c : [ +## { w : 'milk', f : 150 , p : [122, 254, 907] }, +## { w : 'juice', f : 50 , p : [254, 351] } +## ] +## }, +## {...}, +## ... +## ] +## --- index: +## w : word (str/utf-8) +## f : frequency (int) +## c : combos / connections (list of objects) +## p : list of product-ids found in specific words-combination (list of int) +## alt : list of alternative writtings (list of str/utf-8) +## kb: *keyboard* writting (str/latin-ascii) + + +# --- temporary variables (initialize) + +## LOOP through the rows to pre-proccess all products +## --- +for idx, row in rows : + + description = row[_COL['descr']].strip() # product description + pid = domeInt( row[_COL['pid']] ) # product-id + fq = domeInt( row[_COL['freq']] ) # frequency + + # setup product + # --- + products_.append({ + 'w' : description, + 'id' : pid, + 'f' : fq + }) + + # TODO: + # identify brands + # then ... + + description = cleanText(description) # clean sescription string + words = description.strip().split() # split to words + + # identify significant words + keys = [] + for w in words : + if isSignificant(w) : + keys.append(w) + + print(pid, keys) + # append words (and their combos) to the list + for w in keys : + rootKey( w, fq, keywords_ ) + for w2 in keys : + if w2 != w and isSignificant(w2) : + connectKeys( w, w2, pid, fq, keywords_ ) + + + +## SORT keywords +# ////////////////////////////////////////////////////////////////////////////// + +# --- sort childs of each key (per frequency, desc) +for it in keywords_ : + it['c'].sort(key=lambda x: x['f'], reverse=True) + + +# --- sort root keys +keywords_.sort(key=lambda x: x['f'], reverse=True) + +# --- create mini list based on the sorted keywords_ +for it in keywords_ : + minilist_.append({ + 'w' : it['w'], + 'f' : it['f'], + 'kb': it['kb'] + }) + + +## OUTPUT final data to a json-format file +# ////////////////////////////////////////////////////////////////////////////// + +with open("results/keywords-v3.json", "w", encoding="utf-8") as outfile : + data = json.dump(keywords_, outfile, sort_keys=False, indent=3, ensure_ascii=False) + +with open("results/minilist-v3.json", "w", encoding="utf-8") as outfile : + data = json.dump(minilist_, outfile, sort_keys=False, indent=3, ensure_ascii=False) + +with open("results/products.json", "w", encoding="utf-8") as outfile : + data = json.dump(products_, outfile, sort_keys=False, indent=3, ensure_ascii=False) diff --git a/products-dictionary.py b/products-dictionary.py new file mode 100644 index 0000000..1f35a67 --- /dev/null +++ b/products-dictionary.py @@ -0,0 +1,228 @@ +## LIBRARIES +# ////////////////////////////////////////////////////////////////////////////// + +import pandas as pd # pandas for excel reading +import re # regex +import json # json +import os.path # ... + + +## LOCAL FUNCTIONS +# ////////////////////////////////////////////////////////////////////////////// + +# do-me-INTeger +# --- +def domeInt(x) : + if isinstance(x, str) : # if string + return int(x.strip()) + if isinstance(x, float) : # if float + return round(x) + return x # otherwise is int already + +# do-me-Float +# --- +def domeFloat(x) : + if isinstance(x, str) : + return float(x.strip()) + else : + return x + 0.00 # make sure that result is float + + +def cleanText(x) : + removeList = [ ' με ', ' σε ', ' για ', ' του ', ' της ', ' των ', ' από ', '&', '.', ',', '!', '(', ')', '[', ']', '\'', '\"' ] + + for r in removeList : + x = x.replace(r, ' ') + + x.replace(' ', ' ') # remove spare spaces + x.replace(' ', ' ') + x.replace(' ', ' ') + + return x + + +def isSignificant(x) : + # is significant if words has no digit-characters + return not bool(re.match("\S*\d+\S*", x)) + + + +# set root-keyQ: w (if not exist) +# update frequency: f +# into list: l +## --- +def rootKey ( w, f, l ) : + keyExists = False + for it in l : + if it['w'] == w : + keyExists = True + it['f'] += f + + if keyExists == False : + l.append({ + 'w' : w, + 'f' : f, + 'c' : [] + }) + + + +# connect keys: a , b +# of product with id: i +# with frequency: f +# into list: l +## --- +def connectKeys( a, b, i, f, l ) : + if a == b : + return False ## exclude just-in-case + + for it in l : + if it['w'] == a : + + # word a found; + # lets update the connection to: b + bExists = False + + for jt in it['c'] : + if jt['w'] == b : + bExists = True + # update the connection's data + jt['f'] += f + jt['p'].append(i) + + if bExists == False : + # create connection with word: b + it['c'].append({ + 'w': b, + 'f': f, + 'p': [ i ] + }) + + + + + +## LOCAL CONSTANTS +# ////////////////////////////////////////////////////////////////////////////// + +_COL = { + # -- main info + 'freq' : 0, # frequency (based on recent orders) + 'pid' : 1, # product id + 'brand' : 2, # brand + 'barcd' : 3, # barcode + 'sklcd' : 4, + 'eyscd' : 5, + 'descr' : 6, # product description + 'sap2' : 7 # SAP category level-2 id +} + + + + +## SET SOURCE and EXPORT FileNames +# ------------------------------------------------------------------------------ +# location of excel file +loc = "./data/PRODucts2search-wBrands.xlsx" + +print("default filename:", loc) +newXLfile = input("input other Excel filename [enter to keep default]: ") + +if newXLfile != "" and os.path.exists(newXLfile): + loc = newXLfile +else : + print(newXLfile, "is not a file; default is kept;") + +## baseEXPORTname = input("Base export name: ") + + + +## Read data +# ////////////////////////////////////////////////////////////////////////////// + +df = pd.read_excel(loc) # read data from excel file + +rows = df.iterrows() # set rows list + + +# --- Lists to fill +keywords_ = [] # all data +minilist_ = [] + +## keywords format: +## [ +## { +## w : 'fresh', +## f : 150, +## c : [ +## { w : 'milk', f : 150 , p : [122, 254, 907] }, +## { w : 'juice', f : 50 , p : [254, 351] } +## ] +## }, +## {...}, +## ... +## ] +## --- index: +## w : word +## f : frequency +## c : combos / connections +## p : list of product-ids with this combo + +# --- temporary variables (initialize) + + +## LOOP through the rows to pre-proccess all products +## --- +for idx, row in rows : + + description = row[_COL['descr']].strip() # product description + pid = domeInt( row[_COL['pid']] ) # product-id + fq = domeInt( row[_COL['freq']] ) # frequency + + # TODO: + # identify brands + # then ... + + description = cleanText(description) # clean sescription string + words = description.strip().upper().split() # split to words + ## words = [w.strip('.,!;()[]') for w in words] # clean strings + + # identify significant words + keys = [] + for w in words : + if isSignificant(w) : + keys.append(w) + + print(pid, keys) + # append words (and their combos) to the list + for w in keys : + rootKey( w, fq, keywords_ ) + for w2 in keys : + if w2 != w : + connectKeys( w, w2, pid, fq, keywords_ ) + + +## SORT keywords +# ////////////////////////////////////////////////////////////////////////////// + +# --- sort childs of each key (per frequency, desc) +for it in keywords_ : + it['c'].sort(key=lambda x: x['f'], reverse=True) + + +# --- sort root keys +keywords_.sort(key=lambda x: x['f'], reverse=True) + +# --- create mini list based on the sorted keywords_ +for it in keywords_ : + minilist_.append( it['w'] ) + + +## OUTPUT final data to a json-format file +# ////////////////////////////////////////////////////////////////////////////// + +with open("results/keywords-v2.json", "w", encoding="utf-8") as outfile : + data = json.dump(keywords_, outfile, sort_keys=False, indent=3, ensure_ascii=False) + +with open("results/minilist.json", "w", encoding="utf-8") as outfile : + data = json.dump(minilist_, outfile, sort_keys=False, indent=3, ensure_ascii=False) diff --git a/search-v2.html b/search-v2.html new file mode 100644 index 0000000..100565b --- /dev/null +++ b/search-v2.html @@ -0,0 +1,336 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + + <style> +body { font-family: 'Cantarell', Helvetica, Arial, sans-serif; } + +.twitter-typeahead { width: 87% ;} +.typeahead, .tt-query, .tt-hint { + width: 100%; height: 30px; + padding: 8px 12px; outline: none; + font-size: 20px; line-height: 30px; + border: 2px solid #ccc; border-radius: 8px; + -webkit-border-radius: 8px; + -moz-border-radius: 8px; +} +.tt-menu { + width: 100%; margin: 12px 0; padding: 8px 0; + background-color: #fff; + border: 1px solid #ccc; border-radius: 8px; + -webkit-border-radius: 8px; + -moz-border-radius: 8px; + -webkit-box-shadow: 0 5px 10px rgba(0,0,0,.2); + -moz-box-shadow: 0 5px 10px rgba(0,0,0,.2); + box-shadow: 0 5px 10px rgba(0,0,0,.2); +} +.tt-suggestion { padding: 3px 20px; line-height: 24px; font-size: 18px; } +.tt-suggestion span { padding-left: 16px; font-size: 14px; color: #777; float: right; } +.tt-cursor { background: #ddd; } +.tt-highlight { font-weight: normal; color: #777; } + +#selections { width: 87%; padding-top: 40px; } +#selections div { padding: 4px 40px; line-height: 24px; font-size: 18px; color: #666; } +#selections div span { padding-left: 16px; font-size: 14px; color: #999; float: right; } + </style> + + <!-- js labraries --> + <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script> + <script src="https://cdnjs.cloudflare.com/ajax/libs/corejs-typeahead/1.2.1/bloodhound.min.js"></script> + <script src="https://cdnjs.cloudflare.com/ajax/libs/corejs-typeahead/1.2.1/typeahead.jquery.min.js"></script> + </head> + <body> + + <div id="the-basics"> + <input class="typeahead" id="tagsInput" type="text" placeholder="try me!"> + </div> + + <div id="selections"> + </div> + + </body> + <script> + +// arrays for kb-format (utf/EL-Gr to ascii translation) +var ORiGiN = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNM'.split(''); +var kbKeyZ = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviyaehioyviyqwertyuiopasdfghjklzxcvbnm'.split(''); + +// convert string to kb-format +function kb_trans(s) { + var charArr = s.split('') + var i + var out = '' + charArr.forEach( el => { + i = 0 + exist = -1 + ORiGiN.forEach( ori => { + if (ori == el) { + exist = i; + } + i++; + }) + out += (exist == -1) ? el : kbKeyZ[exist]; + }); + return out; +} + +// public data objects +var products; +var everyProduct; + +function loadData() { + const xhttp = new XMLHttpRequest(); + xhttp.onload = function() { + products = JSON.parse(this.responseText); + } + xhttp.open("GET", "results/keywords-v3.json"); + xhttp.send(); +} +loadData(); + +function loadData2() { + const xhttp = new XMLHttpRequest(); + xhttp.onload = function() { + everyProduct = JSON.parse(this.responseText); + } + xhttp.open("GET", "results/products.json"); + xhttp.send(); +} +loadData2(); + +// bigram fuzzy match +// --- credit: https://dirask.com/posts/JavaScript-check-words-similarity-fuzzy-compare-with-bigrams-paola1 +const createBigram = word => { + const input = word.toLowerCase(); + const vector = []; + for (let i = 0; i < input.length; ++i) { + vector.push(input.slice(i, i + 2)); + } + return vector; +}; +const checkSimilarity = (a, b) => { + if (a.length > 0 && b.length > 0) { + const aBigram = createBigram(a); + const bBigram = createBigram(b); + let hits = 0; + for (let x = 0; x < aBigram.length; ++x) { + for (let y = 0; y < bBigram.length; ++y) { + if (aBigram[x] === bBigram[y]) { + hits += 1; + } + } + } + if (hits > 0) { + const union = aBigram.length + bBigram.length; + return (2.0 * hits) / union; + } + } + return 0; +}; +var bi_1st = .6; // bigram minimum match score for 1st word +var bi_2nd = .8; // bigram minimum match score for 2nd word + +// on document ready code ///////////////////////////////////////////////////// +$(document).ready(function() { + + // suggestions engine //////////////////////////////////////////////////// + // --- + function suggestions_engine(qOrig) { + var results = []; // suggestions to respond + var proList = []; // list of products (for all suggestions) + var commonL = []; // list of common products (for multiple suggestions) + var possibleNext = []; // list of possible next suggestions + + var root, last; + + // clean q(uery) string from symbols and multiple spaces + var q = qOrig.replace('+',' ').replace('.',' ') + .replace(' ',' ') + .replace(' ',' '); + + var qAr = q.split(' '); // split to words + + if (qAr.length == 1) { // suggest 1st word //////////////////////// + var kbq = kb_trans(q) + // regex match all possible suggestions; (in kb-format) + substrRegex = new RegExp( kbq, 'i'); // match q anywhere + products.forEach( it => { + if ( (substrRegex.test(it.kb)) + || (checkSimilarity(it.kb, kbq) > bi_1st) ) { + results.push(it); + } + }); + } + + if (qAr.length == 2) { // suggest 2nd word /////////////////////////// + root = qAr[0].trim(); + kbroot = kb_trans(root); + + substrRegex = new RegExp( kb_trans(qAr[1]), 'i'); + + products.forEach( it => { // loop through suggestions + if (it.kb == kbroot ) { // match 1st suggestion + it.c.forEach ( wo => { // regex match linked words + if ( (substrRegex.test(wo.kb)) + || (checkSimilarity(wo.kb, kbroot) > bi_2nd) ) { + results.push({ + w: root +' '+ wo.w, + f: 100 + }); + proList = proList.concat(wo.p) + } + }); + } + }); + } + + if (qAr.length > 2) { + root = qAr.shift(); // get out the first item of qAr + last = qAr.pop(); // get out the lase item of qAr + // now qAr includes only the items after root and before last; + // so qAr includes all already selected suggestions (but root) + + var kbqAr = []; // array of selected suggestions in kb-format + qAr.forEach( w => { kbqAr.push(kb_trans(w)); }) + + // kb-translate the root/last keys + kbroot = kb_trans(root); + kblast = kb_trans(last); + + substrRegex = new RegExp( kblast, 'i'); // construct regex for mathing + + products.forEach( it => { + if (it.kb == kbroot ) { // find root + + // calculate list of common items/products (commonL) + // for selected suggestions + // --- + is1stOcc = true; // 1st occurance flag + it.c.forEach ( swo => { + if (kbqAr.includes( swo.kb )) { + // swo is one of the already selected suggestions + // so... + // update the commonL(ist) + if (is1stOcc) { + commonL = swo.p; + is1stOcc = false; + } + else { + // list ot common products + // = intersection of (so-far) commonL and swo.p + commonL = commonL.filter(value => swo.p.includes(value)); + } + } + else { // if swo is not already selected + // then it is a possible next suggestion + possibleNext.push(swo); + } + }); + // console.log('commonL:', commonL) + + possibleNext.forEach( poss => { // for tthe possible next suggestions + // if word matches regex + // and list of word's products has commons with commonL + // then it is a valid next suggestion + if (substrRegex.test(poss.kb)) { + // check intersection of commonL and suggestion's product-lists + tempL = commonL.filter(value => poss.p.includes(value)); + if (tempL.length) { + results.push({ + w: root +' '+ qAr.join(' ') +' '+ poss.w, + f: 100 + }); + // update proList too + proList = proList.concat(tempL) + } + } + }); + } + + }); + } + + + if ((qAr.length != 1) && (proList.length < 13)) { + // get unique product ids + let unique = proList.filter((item, i, ar) => ar.indexOf(item) === i); + // credit: https://stackoverflow.com/questions/11246758/ + + results = []; + unique.forEach( pr => { + everyProduct.forEach( pi => { + if (pi.id == pr) + results.push(pi); + }) + }); + } + return results; + } + + var isuggest = function(qOrig, list) { + var results = suggestions_engine(qOrig); + if (results.length == 0) { + var qAr = qOrig.trim().split(' '); + qAr.pop(); // remove last word + results = suggestions_engine(qAr.join(' ')); + } + list(results); + } + + // setup suggestions search/input control + // --- + const $tagsInput = $('#tagsInput') + $tagsInput.typeahead( + { + hint: true, + highlight: true, + minLength: 1 + }, + { + limit: 12, + name: 'products', + displayKey: 'w', + source: isuggest, + templates: { + suggestion: function(data) { + // console.log(data.w); + if (data.id) + return '<div>'+ data.w + '<span>' + data.id + '</span></div>'; + return '<div>'+ data.w +'</div>'; + } + } + } + ) + .bind("typeahead:selected", function(obj, datum, name) { + console.log(datum); + if (datum.hasOwnProperty('id')) { + // final product selected; do whatever ... + // ex. add to selection list + $('#selections').append('<div>'+ datum.w + '<span>' + datum.id + '</span></div>'); + + // then reset search control + $('.typeahead').typeahead('val','').trigger('blur') + .trigger("query"); + setTimeout(() => { $('.typeahead').focus(); }, 100); + } + else { + $('.typeahead').typeahead('val','').trigger('blur'); + $('.typeahead').typeahead('val', datum.w +' ') + .trigger("query"); + // give some time to the engine to calculate results + // then fire focus again... + setTimeout(() => { $('.typeahead').focus(); }, 100); + } + }) + .bind("typeahead:cursorchange", function(obj, data) { + // console.log(obj, data); + // var dt = new Date(); + // console.log('triggered cursorchange /'+dt); + }); + + + +}); + </script> +</html> diff --git a/search-v3.html b/search-v3.html new file mode 100644 index 0000000..35e29ca --- /dev/null +++ b/search-v3.html @@ -0,0 +1,422 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + + <style> +body { font-family: 'Cantarell', Helvetica, Arial, sans-serif; margin: 2em; } + +.twitter-typeahead { width: 87%; } +.typeahead, .tt-query, .tt-hint { + width: 100%; height: 30px; + padding: 8px 12px; outline: none; + font-size: 20px; line-height: 30px; + border: 2px solid #ccc; border-radius: 8px; + -webkit-border-radius: 8px; + -moz-border-radius: 8px; +} +.tt-menu { + width: 100%; margin: 12px 0; padding: 8px 0; + background-color: #fff; + border: 1px solid #ccc; border-radius: 8px; + -webkit-border-radius: 8px; + -moz-border-radius: 8px; + -webkit-box-shadow: 0 5px 10px rgba(0,0,0,.2); + -moz-box-shadow: 0 5px 10px rgba(0,0,0,.2); + box-shadow: 0 5px 10px rgba(0,0,0,.2); +} +.tt-suggestion { padding: 3px 20px; line-height: 24px; font-size: 18px; } +.tt-suggestion:hover { cursor: pointer; } +.tt-suggestion span { padding-left: 16px; font-size: 14px; color: #777; float: right; } +.tt-cursor { background: #ddd; } +.tt-highlight { font-weight: normal; color: #777; } +.tt-hint { color: #9598; } + +#selections { width: 87%; padding-top: 40px; } +#selections div { padding: 4px 40px; line-height: 24px; font-size: 18px; color: #666; } +#selections div span { padding-left: 16px; font-size: 14px; color: #999; float: right; } +.-info- { font-size: 12px !important; color: #959 !important; line-height: 14px !important; font-family: 'JetBrains Mono NL', Consolas, Monaco, monospace, fixed !important; } +.-info- b { font-weight: 900;} + </style> + + <!-- js labraries --> + <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script> + <script src="https://cdnjs.cloudflare.com/ajax/libs/corejs-typeahead/1.2.1/bloodhound.min.js"></script> + <script src="https://cdnjs.cloudflare.com/ajax/libs/corejs-typeahead/1.2.1/typeahead.jquery.min.js"></script> + </head> + <body> + + <div id="the-basics"> + <input class="typeahead" id="tagsInput" type="text" placeholder="try me!"> + </div> + + <div id="selections"> + </div> + + </body> + <script> + + +// PUBLIC VARIABLES //////////////////////////////////////////////////////////// + +var kwlinks; // keyword links (word-connections) +var products; // all products + +var trackSearch = []; // searching analytics + +// setup options +var sgLimit = 12; // limit suggestions +var bi_1st = .65; // bigram minimum match score for 1st word +var bi_2nd = .85; // bigram minimum match score for 2nd word + + +// SUPLAMENARY FUNCTIONS /////////////////////////////////////////////////////// + +// arrays for kb-format (utf/EL-Gr to ascii translation) +var ORiGiN = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNM'.split(''); +var kbKeyZ = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviyaehioyviyqwertyuiopasdfghjklzxcvbnm'.split(''); + +// convert string to kb-format +// --- +function kb_trans(s) { + var charArr = s.split('') + var i + var out = '' + charArr.forEach( el => { + i = 0 + exist = -1 + ORiGiN.forEach( ori => { + if (ori == el) { + exist = i; + } + i++; + }) + out += (exist == -1) ? el : kbKeyZ[exist]; + }); + return out; +} + + +function loadData() { + const xhttp = new XMLHttpRequest(); + xhttp.onload = function() { + kwlinks = JSON.parse(this.responseText); + } + xhttp.open("GET", "results/keywords-v3.json"); + xhttp.send(); +} +loadData(); + +function loadData2() { + const xhttp = new XMLHttpRequest(); + xhttp.onload = function() { + products = JSON.parse(this.responseText); + } + xhttp.open("GET", "results/products.json"); + xhttp.send(); +} +loadData2(); + +// bigram fuzzy match +// --- credit: https://dirask.com/posts/JavaScript-check-words-similarity-fuzzy-compare-with-bigrams-paola1 +const createBigram = word => { + const input = word.toLowerCase(); + const vector = []; + for (let i = 0; i < input.length; ++i) { + vector.push(input.slice(i, i + 2)); + } + return vector; +}; +const checkSimilarity = (a, b) => { + if (a.length > 0 && b.length > 0) { + const aBigram = createBigram(a); + const bBigram = createBigram(b); + let hits = 0; + for (let x = 0; x < aBigram.length; ++x) { + for (let y = 0; y < bBigram.length; ++y) { + if (aBigram[x] === bBigram[y]) { + hits += 1; + } + } + } + if (hits > 0) { + const union = aBigram.length + bBigram.length; + return (2.0 * hits) / union; + } + } + return 0; +}; + + + +function echo_tracking() { + var actions = []; + var c; + var chs = 0; // number of characters pressed; + var uis = 0; // number of UI actions used (arrows, enters etc.) + var countingStarted = false; // flag + trackSearch.forEach(e => { + switch(e.v) { + // use of ui actions + case 'ArrowDown' : c = '↓'; uis++; break; + case 'ArrowUp' : c = '↑'; uis++; break; + case 'Enter' : c = '↲ '; uis++; break; + case 'ArrowLeft' : c = '←'; uis++; break; + case 'ArrowRight': c = '→'; uis++; break; + case ' ' : c = '· '; uis++; break; + // ignored keys + case 'Alt' : c = 'Alt'; break; + case 'Control' : c = 'Ctrl'; break; + case 'Escape' : c = 'Esc'; break; + case 'Shift' : c = 'Shft'; break; + case 'Home' : c = 'Home'; break; + case 'End' : c = 'End'; break; + // backspace (user's typing errors) + case 'Backspace' : c = 'BkSp'; break; + case 'Delete' : c = 'Del'; break; + // actual typed characters + default: + if (e.v.length == 1) { + c = '<b><u>'+ e.v +'</u></b>'; + chs++; + } + else { // some non important key; no counter increased + c = e.v; // just record the key + } + } + actions.push(c); + }); + + return actions.join(',') +' (<u>'+ chs +' chs</u>, '+ uis +' uis)'; +} + + +// on document ready code /////////////////////////////////////////////////////// +$(document).ready(function() { + + // suggestions engine ////////////////////////////////////////////////////// + // --- + function suggestions_engine(qOrig) { + var results = []; // suggestions to respond + var proList = []; // list of products (for all suggestions) + var commonL = []; // list of common products (for multiple suggestions) + var possibleNext = []; // list of possible next suggestions + + var root, last; + + // clean q(uery) string from symbols and multiple spaces + var q = qOrig.replace('+',' ').replace('.',' ') + .replace(' ',' ') + .replace(' ',' '); + + var qAr = q.split(' '); // split to words + + if (qAr.length == 1) { // suggest 1st word //////////////////////// + var kbq = kb_trans(q) + // regex match all possible suggestions; (in kb-format) + substrRegex = new RegExp( kbq, 'i'); // match q anywhere + kwlinks.forEach( it => { + if ( (substrRegex.test(it.kb)) + || (checkSimilarity(it.kb, kbq) > bi_1st) ) { + results.push(it); + } + }); + } + + if (qAr.length == 2) { // suggest 2nd word /////////////////////////// + root = qAr[0].trim(); + kbroot = kb_trans(root); + + substrRegex = new RegExp( kb_trans(qAr[1]), 'i'); + + kwlinks.forEach( it => { // loop through suggestions + if (it.kb == kbroot ) { // match 1st suggestion + it.c.forEach ( wo => { // regex match linked words + if ( (substrRegex.test(wo.kb)) + || (checkSimilarity(wo.kb, kbroot) > bi_2nd) ) { + results.push({ + w: root +' '+ wo.w, + f: 100 + }); + proList = proList.concat(wo.p) + } + }); + } + }); + } + + if (qAr.length > 2) { + root = qAr.shift(); // get out the first item of qAr + last = qAr.pop(); // get out the lase item of qAr + // now qAr includes only the items after root and before last; + // so qAr includes all already selected suggestions (but root) + + var kbqAr = []; // array of selected suggestions in kb-format + qAr.forEach( w => { kbqAr.push(kb_trans(w)); }) + + // kb-translate the root/last keys + kbroot = kb_trans(root); + kblast = kb_trans(last); + + substrRegex = new RegExp( kblast, 'i'); // construct regex for mathing + + kwlinks.forEach( it => { + if (it.kb == kbroot ) { // find root + + // calculate list of common items/products (commonL) + // for selected suggestions + // --- + is1stOcc = true; // 1st occurance flag + it.c.forEach ( swo => { + if (kbqAr.includes( swo.kb )) { + // swo is one of the already selected suggestions + // so... + // update the commonL(ist) + if (is1stOcc) { + commonL = swo.p; + is1stOcc = false; + } + else { + // list ot common products + // = intersection of (so-far) commonL and swo.p + commonL = commonL.filter(value => swo.p.includes(value)); + } + } + else { // if swo is not already selected + // then it is a possible next suggestion + possibleNext.push(swo); + } + }); + // console.log('commonL:', commonL) + + possibleNext.forEach( poss => { // for tthe possible next suggestions + // if word matches regex + // and list of word's products has commons with commonL + // then it is a valid next suggestion + if (substrRegex.test(poss.kb)) { + // check intersection of commonL and suggestion's product-lists + tempL = commonL.filter(value => poss.p.includes(value)); + if (tempL.length) { + results.push({ + w: root +' '+ qAr.join(' ') +' '+ poss.w, + f: 100 + }); + // update proList too + proList = proList.concat(tempL) + } + } + }); + } + + }); + } + + + if ( (qAr.length != 1) && (proList.length < (sgLimit +1)) ) { + // get unique product ids + let unique = proList.filter((item, i, ar) => ar.indexOf(item) === i); + // credit: https://stackoverflow.com/questions/11246758/ + + results = []; + unique.forEach( pr => { + products.forEach( pi => { + if (pi.id == pr) + results.push(pi); + }) + }); + } + return results; + } + + // request suggestions procedure + // args... + // qOrig: original query string + // list: artay structure to host results + // --- + var isuggest = function(qOrig, list) { + var results = suggestions_engine(qOrig); + + // if no results... + // request again after removing last (key)word + if (results.length == 0) { + var qAr = qOrig.trim().split(' '); + qAr.pop(); // remove last word + results = suggestions_engine(qAr.join(' ')); + } + + list(results); + } + + // setup suggestions search/input control + // --- + const $tagsInput = $('#tagsInput') + $tagsInput.typeahead( + { + hint: true, + highlight: true, + minLength: 1 + }, + { + limit: sgLimit, + name: 'kwlinks', + displayKey: 'w', + source: isuggest, + templates: { + suggestion: function(data) { + // console.log(data.w); + if (data.id) + return '<div>'+ data.w + '<span>' + data.id + '</span></div>'; + return '<div>'+ data.w +'<span>+</span></div>'; + } + } + } + ) + .bind("typeahead:selected", function(obj, datum, name) { + // console.log(datum); + if (datum.hasOwnProperty('id')) { + // final product selected; do whatever ... + // ex. add to selection list + $('#selections').append('<div>'+ datum.w + '<span>' + datum.id + '</span></div>'); + + // then reset search control + $('.typeahead').typeahead('val','').trigger('blur') + .trigger("query"); + setTimeout(() => { $('.typeahead').focus(); }, 100); + + // finaly save tracking info; + // $('#selections').append('<div class="-info-">'+ JSON.stringify(trackSearch) +'</div>'); + $('#selections').append('<div class="-info-">'+ echo_tracking() +'</div>'); + trackSearch.length = 0; // ... and reset info to be ready for nextsearch + } + else { + $('.typeahead').typeahead('val','').trigger('blur'); + $('.typeahead').typeahead('val', datum.w +' ') + .trigger("query"); + // give some time to the engine to calculate results + // then fire focus again... + setTimeout(() => { $('.typeahead').focus(); }, 100); + } + trackSearch.push({ + e: 'key', + v: 'Enter', + i: $('#tagsInput').val() + }); + }) + .bind("typeahead:cursorchange", function(obj, data) { + // console.log(obj, data); + // var dt = new Date(); + // console.log('triggered cursorchange /'+dt); + }); + + // TRACK user search attempt /////////////////////////////////////////////// + $('.typeahead').on('keyup', function(e) { + trackSearch.push({ + e: 'key', + v: e.key, + i: $('#tagsInput').val() + }); + }); + +}); + </script> +</html> diff --git a/search.html b/search.html new file mode 100644 index 0000000..2a12941 --- /dev/null +++ b/search.html @@ -0,0 +1,308 @@ +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8"> + + <style> +body { font-family: 'Cantarell', Helvetica, Arial, sans-serif; } + +.twitter-typeahead { width: 87% ;} +.typeahead, .tt-query, .tt-hint { + width: 100%; height: 30px; + padding: 8px 12px; outline: none; + font-size: 20px; line-height: 30px; + border: 2px solid #ccc; border-radius: 8px; + -webkit-border-radius: 8px; + -moz-border-radius: 8px; +} +.tt-menu { + width: 100%; margin: 12px 0; padding: 8px 0; + background-color: #fff; + border: 1px solid #ccc; border-radius: 8px; + -webkit-border-radius: 8px; + -moz-border-radius: 8px; + -webkit-box-shadow: 0 5px 10px rgba(0,0,0,.2); + -moz-box-shadow: 0 5px 10px rgba(0,0,0,.2); + box-shadow: 0 5px 10px rgba(0,0,0,.2); +} +.tt-suggestion { padding: 3px 20px; line-height: 24px; font-size: 18px; } +.tt-suggestion span { padding-left: 16px; font-size: 14px; color: #777; float: right; } +.tt-cursor { background: #ddd; } +.tt-highlight { font-weight: normal; color: #777; } + </style> + + <!-- js labraries --> + <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script> + <script src="https://cdnjs.cloudflare.com/ajax/libs/corejs-typeahead/1.2.1/bloodhound.min.js"></script> + <script src="https://cdnjs.cloudflare.com/ajax/libs/corejs-typeahead/1.2.1/typeahead.jquery.min.js"></script> + </head> + <body> + + <div id="the-basics"> + <input class="typeahead" id="tagsInput" type="text" placeholder="try me!"> + </div> + + </body> + <script> + +// arrays for kb-format (utf/EL-Gr to ascii translation) +var ORiGiN = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNM'.split(''); +var kbKeyZ = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviyaehioyviyqwertyuiopasdfghjklzxcvbnm'.split(''); + +// convert string to kb-format +function kb_trans(s) { + var charArr = s.split('') + var i + var out = '' + charArr.forEach( el => { + i = 0 + exist = -1 + ORiGiN.forEach( ori => { + if (ori == el) { + exist = i; + } + i++; + }) + out += (exist == -1) ? el : kbKeyZ[exist]; + }); + return out; +} + +// public data objects +var products; +var everyProduct; + +function loadData() { + const xhttp = new XMLHttpRequest(); + xhttp.onload = function() { + products = JSON.parse(this.responseText); + } + xhttp.open("GET", "results/keywords-v3.json"); + xhttp.send(); +} +loadData(); + +function loadData2() { + const xhttp = new XMLHttpRequest(); + xhttp.onload = function() { + everyProduct = JSON.parse(this.responseText); + } + xhttp.open("GET", "results/products.json"); + xhttp.send(); +} +loadData2(); + +// bigram fuzzy match +// --- credit: https://dirask.com/posts/JavaScript-check-words-similarity-fuzzy-compare-with-bigrams-paola1 +const createBigram = word => { + const input = word.toLowerCase(); + const vector = []; + for (let i = 0; i < input.length; ++i) { + vector.push(input.slice(i, i + 2)); + } + return vector; +}; +const checkSimilarity = (a, b) => { + if (a.length > 0 && b.length > 0) { + const aBigram = createBigram(a); + const bBigram = createBigram(b); + let hits = 0; + for (let x = 0; x < aBigram.length; ++x) { + for (let y = 0; y < bBigram.length; ++y) { + if (aBigram[x] === bBigram[y]) { + hits += 1; + } + } + } + if (hits > 0) { + const union = aBigram.length + bBigram.length; + return (2.0 * hits) / union; + } + } + return 0; +}; +var biMMS = .6; // bigram minimum match score + +// on document ready code ///////////////////////////////////////////////////// +$(document).ready(function() { + + // suggestions engine //////////////////////////////////////////////////// + // --- + var isuggest = function(qOrig, list) { + var results = []; // suggestions to respond + var proList = []; // list of products (for all suggestions) + var commonL = []; // list of common products (for multiple suggestions) + var possibleNext = []; // list of possible next suggestions + + var root, last; + + // clean q(uery) string from symbols and multiple spaces + var q = qOrig.replace('+',' ').replace('.',' ') + .replace(' ',' ') + .replace(' ',' '); + // split to words + var qAr = q.split(' '); + // console.log(qAr); + + if (qAr.length == 1) { // suggest 1st word //////////////////////// + var kbq = kb_trans(q) + // regex match all possible suggestions; (in kb-format) + substrRegex = new RegExp( kbq, 'i'); // match q anywhere + products.forEach( it => { + if ( (substrRegex.test(it.kb)) + || (checkSimilarity(it.kb, kbq) > biMMS) ) { + // console.log(it.kb, kbq, checkSimilarity(it.kb, kbq)); + results.push(it); + } + }); + } + + if (qAr.length == 2) { // suggest 2nd word /////////////////////////// + root = qAr[0].trim(); + kbroot = kb_trans(root); + + substrRegex = new RegExp( kb_trans(qAr[1]), 'i'); + + products.forEach( it => { // loop through suggestions + if (it.kb == kbroot ) { // match 1st suggestion + it.c.forEach ( wo => { // regex match linked words + if ( (substrRegex.test(wo.kb)) + || (checkSimilarity(wo.kb, kbroot) > biMMS) ) { + results.push({ + w: root +' '+ wo.w, + f: 100 + }); + proList = proList.concat(wo.p) + } + }); + } + }); + } + + if (qAr.length > 2) { + root = qAr.shift(); // get out the first item of qAr + last = qAr.pop(); // get out the lase item of qAr + // now qAr includes only the items after root and before last; + // so qAr includes all already selected suggestions (but root) + + var kbqAr = []; // array of selected suggestions in kb-format + qAr.forEach( w => { kbqAr.push(kb_trans(w)); }) + + // kb-translate the root/last keys + kbroot = kb_trans(root); + kblast = kb_trans(last); + + substrRegex = new RegExp( kblast, 'i'); // construct regex for mathing + + products.forEach( it => { + if (it.kb == kbroot ) { // find root + + // calculate list of common items/products (commonL) + // for selected suggestions + // --- + is1stOcc = true; // 1st occurance flag + it.c.forEach ( swo => { + if (kbqAr.includes( swo.kb )) { + // swo is one of the already selected suggestions + // so... + // update the commonL(ist) + if (is1stOcc) { + commonL = swo.p; + is1stOcc = false; + } + else { + // list ot common products + // = intersection of (so-far) commonL and swo.p + commonL = commonL.filter(value => swo.p.includes(value)); + } + } + else { // if swo is not already selected + // then it is a possible next suggestion + possibleNext.push(swo); + } + }); + // console.log('commonL:', commonL) + + possibleNext.forEach( poss => { // for tthe possible next suggestions + // if word matches regex + // and list of word's products has commons with commonL + // then it is a valid next suggestion + if (substrRegex.test(poss.kb)) { + // check intersection of commonL and suggestion's product-lists + tempL = commonL.filter(value => poss.p.includes(value)); + if (tempL.length) { + results.push({ + w: root +' '+ qAr.join(' ') +' '+ poss.w, + f: 100 + }); + // update proList too + proList = proList.concat(tempL) + } + } + }); + } + + }); + } + + + if ((qAr.length != 1) && (proList.length < 13)) { + // get unique product ids + let unique = proList.filter((item, i, ar) => ar.indexOf(item) === i); + // credit: https://stackoverflow.com/questions/11246758/ + + results = []; + unique.forEach( pr => { + everyProduct.forEach( pi => { + if (pi.id == pr) + results.push(pi); + }) + }); + } + + list(results); + } + + // setup suggestions search/input control + // --- + const $tagsInput = $('#tagsInput') + $tagsInput.typeahead( + { + hint: true, + highlight: true, + minLength: 0 + }, + { + limit: 12, + name: 'products', + displayKey: 'w', + source: isuggest, + templates: { + suggestion: function(data) { + console.log(data.w); + if (data.id) + return '<div>'+ data.w + '<span>' + data.id + '</span></div>'; + return '<div>'+ data.w +'</div>'; + } + } + } + ) + .bind("typeahead:selected", function(obj, datum, name) { + $('.typeahead').typeahead('val','').trigger('blur'); + $('.typeahead').typeahead('val', datum.w +' ') + .trigger("query"); + // give some time to the engine to calculate results + // then fire focus again... + setTimeout(() => { $('.typeahead').focus(); }, 100); + }) + .bind("typeahead:cursorchange", function(obj, data) { + // console.log(obj, data); + // var dt = new Date(); + // console.log('triggered cursorchange /'+dt); + }); + + + +}); + </script> +</html> |