diff options
| -rw-r--r-- | javascript/keygen.js | 432 |
1 files changed, 281 insertions, 151 deletions
diff --git a/javascript/keygen.js b/javascript/keygen.js index fef20e7..2349901 100644 --- a/javascript/keygen.js +++ b/javascript/keygen.js @@ -6,15 +6,19 @@ * Contents: * #1 Requirements * #2.1 Personalized constants and parametres - * #2.2 Preloaded Data + * #2.2 Purify string functions + * #2.3 Preloaded Data * #3 Supporting functions * #4 Output functions * #5 Entry-point function main() */ -// #1 -// REQUIREMENTS -//////////////////////////////////////////////////////////////////////////////// + +/** #1 + * --- + * REQUIREMENTS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// const fs = require('fs'); const os = require('os'); @@ -23,12 +27,16 @@ const {Storage} = require('@google-cloud/storage'); // Google Cloud Storage -// #2.1 -// SETUP PERSONALIZED CONSTANTS AND PARAMETRES -//////////////////////////////////////////////////////////////////////////////// + +/** #2.1 + * --- + * SETUP PERSONALIZED CONSTANTS AND PARAMETRES + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// // get products FROM api/endpoint parametres +// --- -- -- - - - const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json'; const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch"; const request_settings = { method: "Get" }; @@ -69,19 +77,97 @@ const FgGreen = "\x1b[32m"; -// #2.2 -// PRELOADED DATA -//////////////////////////////////////////////////////////////////////////////// -// any-character to keyboard-latin mapping -// ----------------------------------------------------------------------------- +/** #2.2 + * --- + * PURIFY STRING FUNCTIONS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + + +/** keyboardize + * ----------------------------------------------------------------------------- + * translates string to keyboard-latin keys + * (the ones that used whan typing each letter of the word) + * + * @param str (string): original string (utf8 of latin or greek subgroups) + * @return (string): latin/ascii equivalent string + */ + +// cache (keeo in global) any-character to keyboard-latin mapping +// --- -- -- - - - var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); const map = new Map(); for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]); +// "keyboardize" function +// --- -- -- - - - +function keyboardize(str) { + str = str.replace('\'',''); + var out = ''; + for (var i=0 ; i< str.length; i++) out += map.get(str[i]); + return out; +} + + + + +/** sanitize_EL + * ----------------------------------------------------------------------------- + * replaces greek accended vowels with non accended ones + * takes care of sigma on the end of words + * + * @param str (string) + * @return sanitized string + */ + +// first cache (=create a global array) +// of accended to non-accended vowels mapping +// --- -- -- - - - +accented_vowels = []; +[ + 'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω', + 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω' +].forEach( pair => { + ap = pair.split(' '); + accented_vowels.push({ + a: ap[0], // accented + p: ap[1] // pure = non accended + }); +}) + +// the sanitize_GR function +// --- -- -- - - - +function sanitize_GR(str) { + + str = str.toLowerCase(); + + // replace accended vowels with pure ones + accented_vowels.forEach( v => { + str = str.replaceAll(v.a, v.p); + }); + + // replace sigma on the end of words + str = str + ' '; + str = str.replaceAll('σ-', 'ς-'); + str = str.replaceAll('σ ', 'ς '); + + return str.trim(); +} + + + +/** #2.3 + * --- + * PRELOADED DATA + * in the future these data may be loaded from a database / api-endpoint + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + -// synonyms +// synonyms //////////////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- var synonyms = []; // groups of synonyms @@ -128,18 +214,30 @@ var synonyms_Originals = [ 'Καρτέλλα Καρτέλα Καρτέλλες' ] synonyms_Originals.forEach( grp => { + // remove accends so any replaces can be done flawlessly + grp = sanitize_GR(grp); + + // create array of synonyms; + // push it to sanitized and keyboardized arrays synonyms.push( grp.split(' ') ); - synonym_kbs.push( kb_trans(grp).split(' ') ); + synonym_kbs.push( keyboardize(grp).split(' ') ); }) -// significant terms + +// significant terms /////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- -significantExceptios = '7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R'.split(' ') +significantExceptios = sanitize_GR('7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ') // replaces (correcting descriptions) -// = construct unequivocally liked words +// = construct unequivocally liked words /////////////////////////////////////// // ----------------------------------------------------------------------------- + +// NOTE: TODO: +// in future implementations multi-word keywords +// may use the non-breaking space as conecting character (\u00A0) instead of dush (-) +// (or maybe both of them) +// --- -- -- - - - replaces = []; replaceSource = [ '3 ΑΛΦΑ ;3-ΑΛΦΑ ', @@ -199,12 +297,14 @@ replaceSource = [ 'Χωρίς προσθήκη ζάχαρης ;Χωρίς-ζάχαρη' ]; replaceSource.forEach( it => { - st = it.split(';'); + st = sanitize_GR( + it + ).split(';'); replaces.push({ src: st[0], trg: st[1] }); }); -// words that shall not be searched first +// words that shall not be searched first ////////////////////////////////////// // ----------------------------------------------------------------------------- var noRootKeywords = []; noRoot = [ @@ -281,63 +381,70 @@ noRoot = [ 'Καθαρισμού', 'Ρούχων' ]; -noRoot.forEach( w => { noRootKeywords.push(kb_trans(w)); }); +noRoot.forEach( w => { noRootKeywords.push(keyboardize(w)); }); -// list of linked-words +// list of linked-words //////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- linkedWords = [ - 'Χωρίς-Γλουτένη', - 'Χωρίς-Ζάχαρη', - 'Χωρίς-Αλάτι', - 'Χωρίς-Λακτόζη', - 'Χωρίς-Συντηρητικά', - 'Χωρίς-Αλκοόλ', - 'Χωρίς-Kαφεϊνη', - 'Χωρίς-Γλυκάνισο', - 'Χωρίς-Ανθρακικό', - 'Υψηλής-Παστερίωσης', - 'Ολικής-Άλεσης', - 'Ολικής-Aλέσεως', - 'Χαρτί-Υγείας', - 'ρολό-υγείας', - 'χαρτί-τουαλέτας', - 'Χαρτί-Κουζίνας', - 'Μπάρες-Δημητριακών', - 'Μπαρμπα-Στάθης', - 'COCA-COLA', - 'Aς-Μαγειρέψουμε', - 'ΚΡΙΣ-ΚΡΙΣ', - 'ΚΡΙ-ΚΡΙ', - 'ΕΛ-ΓΚΡΕΚΟ', - 'FREE-STEP', - 'EL-SABOR', - 'LE-PETIT-MARSEILLAIS', - 'DOUWE-EGBERTS', - 'ΕΝ-ΕΛΛΑΔΙ', - 'SPIN-SPAN', - 'CRETA-FARMS', - 'CRETA-FARM', - 'NES-CAFE', - 'Ολες-τις-Χρήσεις', - 'Το-Μάννα', - 'Χωρίς-προσθήκη-ζάχαρης' + sanitize_GR('Χωρίς-Γλουτένη'), + sanitize_GR('Χωρίς-Ζάχαρη'), + sanitize_GR('Χωρίς-Αλάτι'), + sanitize_GR('Χωρίς-Λακτόζη'), + sanitize_GR('Χωρίς-Συντηρητικά'), + sanitize_GR('Χωρίς-Αλκοόλ'), + sanitize_GR('Χωρίς-Kαφεϊνη'), + sanitize_GR('Χωρίς-Γλυκάνισο'), + sanitize_GR('Χωρίς-Ανθρακικό'), + sanitize_GR('Υψηλής-Παστερίωσης'), + sanitize_GR('Ολικής-Άλεσης'), + sanitize_GR('Ολικής-Aλέσεως'), + sanitize_GR('Χαρτί-Υγείας'), + sanitize_GR('ρολό-υγείας'), + sanitize_GR('χαρτί-τουαλέτας'), + sanitize_GR('Χαρτί-Κουζίνας'), + sanitize_GR('Μπάρες-Δημητριακών'), + sanitize_GR('Μπαρμπα-Στάθης'), + sanitize_GR('COCA-COLA'), + sanitize_GR('Aς-Μαγειρέψουμε'), + sanitize_GR('ΚΡΙΣ-ΚΡΙΣ'), + sanitize_GR('ΚΡΙ-ΚΡΙ'), + sanitize_GR('ΕΛ-ΓΚΡΕΚΟ'), + sanitize_GR('FREE-STEP'), + sanitize_GR('EL-SABOR'), + sanitize_GR('LE-PETIT-MARSEILLAIS'), + sanitize_GR('DOUWE-EGBERTS'), + sanitize_GR('ΕΝ-ΕΛΛΑΔΙ'), + sanitize_GR('SPIN-SPAN'), + sanitize_GR('CRETA-FARMS'), + sanitize_GR('CRETA-FARM'), + sanitize_GR('NES-CAFE'), + sanitize_GR('Ολες-τις-Χρήσεις'), + sanitize_GR('Το-Μάννα'), + sanitize_GR('Χωρίς-προσθήκη-ζάχαρης') ]; -// list of words to exclude from keywords +// list of words to exclude from keywords ////////////////////////////////////// // ----------------------------------------------------------------------------- // NOTE: APPLIED in PER-WORD base -> after spliting description to words removeList = [] -removeOriginals = 'κατά παρά υπό από μετά προς μας με σε για του της των από στο το στον & r s ft l τ e g h k m n o p s x'.split(' ') -removeOriginals.forEach( w => { removeList.push( kb_trans(w)); }); +removeOriginals = sanitize_GR( + 'κατά παρά υπό από μετά προς μας με σε για του της των από στο το στον & r s ft l τ e g h k m n o p s x' +) +.split(' ') +removeOriginals.forEach( w => { removeList.push( keyboardize(w)); }); -// #3 -// SUPPORTING FUNCTIONS -//////////////////////////////////////////////////////////////////////////////// + + +/** #3 + * --- + * SUPPORTING FUNCTIONS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// /** freq @@ -354,10 +461,12 @@ function freq_of(id) { } - -// proccess data -// The MAIN proccess -// ----------------------------------------------------------------------------- +/** proccess data + * --- -- -- - - - + * + * central proccessing procedure + * ----------------------------------------------------------------------------- + */ function extract_linked_keywords(obj) { // console.log(JSON.stringify(obj, null, 2)); @@ -379,7 +488,7 @@ function extract_linked_keywords(obj) { // filter words; keep only significant words.forEach( w => { - if (removeList.indexOf(kb_trans(w)) == -1) // if not excluded + if (removeList.indexOf(keyboardize(w)) == -1) // if not excluded if (is_significant(w)) // and significant keys.push(w); // add it to keys }); @@ -389,7 +498,7 @@ function extract_linked_keywords(obj) { var wl = synonym_keys(w); // if key CAN be a root word (not a no-Root-keyword) update root-node - if (noRootKeywords.indexOf(kb_trans(w)) == -1) { + if (noRootKeywords.indexOf(keyboardize(w)) == -1) { root_key(wl, fq); // update root keyword stats @@ -431,13 +540,19 @@ function extract_linked_keywords(obj) { // functions for linking words in keywords dictionary //////////////////////////////////////////////////////////////////////////////// -// set root-keyword: wl (if not exist) -// update frequency: f -// * wl is a list of synonym-words -// ** comparison is based on the *keyboard* format -// --- + +/** root_key( wl, f ) + * --- -- -- - - - + * set root keyword: wl (if not exist) + * update frequency: f + * + * @param wl (list) :a list of synonym-words + * @param f (int): frequency of product + * + * ** comparison is based on the *keyboard* format + */ function root_key ( wl, f ) { - var wkb = kb_trans(wl[0]) // cache kb format + var wkb = keyboardize(wl[0]) // cache kb format // check if exists in root keys already // NOTE: you only need to check the 1st word of synonyms-list @@ -457,14 +572,17 @@ function root_key ( wl, f ) { return true; } - -// connect keys: a , b (each one is a list of synonmyms) -// of product with id: i -// with frequency: f -// --- +/** connect_keys + * --- -- -- - - - + * @param (list) : connect keyword a (synonyms list) + * @param (list) : with keyword b (synonyms list) + * @param (int) : id (eys code) of product where keywords a and b are linked + * @param (int) : frequency of product (now many times has been purchased) + * @returns true + */ function connect_keys( a, b, id, f ) { - var kbA = kb_trans(a[0]); - var kbB = kb_trans(b[0]); + var kbA = keyboardize(a[0]); + var kbB = keyboardize(b[0]); var bExists = false; if (kbA == kbB) return false; // exclude just-in-case @@ -502,28 +620,28 @@ function connect_keys( a, b, id, f ) { //////////////////////////////////////////////////////////////////////////////// -// kb_trans translates string to keyboard-latin keys; -// --- -function kb_trans(str) { - str = str.replace('\'',''); - var out = ''; - for (var i=0 ; i< str.length; i++) out += map.get(str[i]); - return out; -} -// clean text trims some characters (+.') and internal multiple-spaces -// --- +/** clean text + * --- -- -- - - - + * removes non keyword characters [\+, \., \'] and internal multiple-spaces + * @param txt (string): product description + */ function clean_text(txt) { - return txt.replace('+',' ').replace('.',' ') + return sanitize_GR(txt).replace('+',' ').replace('.',' ') .replace(' ',' ') .replace(' ',' '); } -// check if term is significant -// (if not, the term will be excluded from keywords dicionary) -// --- +/** is significant + * --- -- -- - - - + * check if term is significant (and needs to be indexed) + * (if not, the term will be excluded from keywords dicionary) + * + * @param str + * @return (boolean) + */ function is_significant(str) { if (str == '') return false; if (significantExceptios.indexOf(str) !== -1) return true; @@ -531,11 +649,18 @@ function is_significant(str) { } -// check if word: w -// ...has synonyms; return list of synonyms -// --- +/** synonym_keys + * --- -- -- - - - + * check if word: w + * has synonyms; return list of synonyms + * + * @param w (string) + * @return list of synonyms + * + * NOTE: in no synonyms exist, retutn the word as a list + */ function synonym_keys(w) { - w_kb = kb_trans(w); + w_kb = keyboardize(w); for (i=0 ; i < synonym_kbs.length ; i++) { if (synonym_kbs.indexOf(w_kb) !== -1) return synonyms[i]; @@ -544,9 +669,20 @@ function synonym_keys(w) { } + +/** mark_explicit_links + * + * mark linked words + * shall be handled as one-(key)word + * + * @param str + * @return + */ + // edit common mistakes // with suggested replaces -function do_replaces(str) { +function mark_explicit_links(str) { + replaces.forEach( it => { str = str.replace(it.src, it.trg); }); return str; } @@ -555,8 +691,11 @@ function do_replaces(str) { // preproccess description // --- function preproccess_text(str) { - str = clean_text(str); - str = do_replaces(str); + + str = clean_text(str); // remove non-keyword characters + + str = mark_explicit_links(sanitize_GR(str)); // mark explicit links + replaces.forEach( lw => { reg = new RegExp( lw.src, "gi"); // prepare regex for case insensitive (i) replace all (g) str = str.toLowerCase().replace(reg, lw.trg); @@ -566,61 +705,47 @@ function preproccess_text(str) { } -// DEPRICATED: -// mark linked words (connect them with a dash) -// return new text after "all-links" are marked -// --- -function mark_linked_words(txt) { - linkedWords.forEach( lw => { - txt = mark_link( lw, txt ); - }); - return txt -} +/* --- + DEPRICATED: + // mark linked words (connect them with a dash) + // return new text after "all-links" are marked + // --- + function mark_linked_words(txt) { + linkedWords.forEach( lw => { - -// TODO: -// update function -// new algorithm steps: -// -// str.toLowerCase().replace(/αλφα βητα/gi, 'α-β'); console.log(replacedOnce); -// -// mark a link (lws) to a text (source) -// conecting them with a dash/minus character -// --- -function mark_link(lws, source) { - var src_kb = kb_trans(source.replace(' ', '-')); - var lws_kb = kb_trans(lws.replace(' ', '-')); - var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase()) - if (_left !== -1 ) { - return source.slice(0, _left) + lws + source.slice(_left + lws.length); + txt = mark_link( lw, txt ); + }); + return txt } - else return source; -} +*/ -accented_vowels = []; -[ - 'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω', - 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω' -].forEach( pair => { - ab = pair.split(' '); - accented_vowels.push({ - a: ab[0], // accented - p: ab[1] // pure - }); -}) -// accendless -// replaces accended vowels by non accended ones -function accendless(str) { +/* --- + DEPRICATED: + // mark a link (lws) to a text (source) + // conecting them with a dash/minus character + // --- + function mark_link(lws, source) { + var src_kb = keyboardize(source.replace(' ', '-')); + var lws_kb = keyboardize(lws.replace(' ', '-')); + var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase()) + if (_left !== -1 ) { + return source.slice(0, _left) + lws + source.slice(_left + lws.length); + } + else return source; + } +*/ -} -// #4 -// OUTPUT FUNCTIONS -//////////////////////////////////////////////////////////////////////////////// + +/** #4 + * --- + * OUTPUT FUNCTIONS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// @@ -667,9 +792,14 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) { -// #5 -// MAIN function (exposed function to run the whole proccess) -//////////////////////////////////////////////////////////////////////////////// + + +/** #5 + * --- + * MAIN function (exposed function to run the whole proccess) + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + /** main() |