keygen keyboardize and sanitize_GR functions for supporting greek linked-words (work in progress)

1 files changed, 281 insertions, 151 deletions

diff --git a/javascript/keygen.js b/javascript/keygen.js
index fef20e7..2349901 100644
--- a/javascript/keygen.js
+++ b/javascript/keygen.js
@@ -6,15 +6,19 @@
  * Contents:
  * #1 Requirements
  * #2.1 Personalized constants and parametres
- * #2.2 Preloaded Data
+ * #2.2 Purify string functions
+ * #2.3 Preloaded Data
  * #3 Supporting functions
  * #4 Output functions
  * #5 Entry-point function main()
  */
 
-// #1
-// REQUIREMENTS
-////////////////////////////////////////////////////////////////////////////////
+
+/**  #1
+ * ---
+ * REQUIREMENTS
+ * -----------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////////
 
 const fs = require('fs');
 const os = require('os');
@@ -23,12 +27,16 @@ const {Storage} = require('@google-cloud/storage');     // Google Cloud Storage
 
 
 
-// #2.1
-// SETUP PERSONALIZED CONSTANTS AND PARAMETRES
-////////////////////////////////////////////////////////////////////////////////
+
+/** #2.1
+ * ---
+ * SETUP PERSONALIZED CONSTANTS AND PARAMETRES
+ * -----------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////////
 
 
 // get products FROM api/endpoint parametres
+// --- -- -- - - -
 const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json';
 const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch";
 const request_settings = { method: "Get" };
@@ -69,19 +77,97 @@ const FgGreen = "\x1b[32m";
 
 
 
-// #2.2
-// PRELOADED DATA
-////////////////////////////////////////////////////////////////////////////////
 
-// any-character to keyboard-latin mapping
-// -----------------------------------------------------------------------------
+/** #2.2
+ * ---
+ * PURIFY STRING FUNCTIONS
+ * -----------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////////
+
+
+
+/** keyboardize
+ * -----------------------------------------------------------------------------
+ * translates string to keyboard-latin keys
+ * (the ones that used whan typing each letter of the word)
+ * 
+ * @param str (string): original string (utf8 of latin or greek subgroups)
+ * @return (string): latin/ascii equivalent string
+ */
+
+// cache (keeo in global) any-character to keyboard-latin mapping
+// --- -- -- - - -
 var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
 var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
 const map = new Map();
 for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
 
+// "keyboardize" function
+// --- -- -- - - -
+function keyboardize(str) {
+    str = str.replace('\'','');
+    var out = '';
+    for (var i=0 ; i< str.length; i++)  out += map.get(str[i]);
+    return out;
+}
+
+
+
+
+/**  sanitize_EL
+ * -----------------------------------------------------------------------------
+ * replaces greek accended vowels with non accended ones
+ * takes care of sigma on the end of words
+ * 
+ * @param str (string)
+ * @return sanitized string
+ */
+
+// first cache (=create a global array)
+// of accended to non-accended vowels mapping
+// --- -- -- - - -
+accented_vowels = [];   
+[
+    'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω',
+    'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω'
+].forEach( pair => {
+    ap = pair.split(' ');
+    accented_vowels.push({
+        a: ap[0],   // accented
+        p: ap[1]    // pure = non accended
+    });
+})
+
+// the sanitize_GR function
+// --- -- -- - - -
+function sanitize_GR(str) {
+
+    str = str.toLowerCase();
+
+    // replace accended vowels with pure ones
+    accented_vowels.forEach( v => {
+        str = str.replaceAll(v.a, v.p);
+    });
+
+    // replace sigma on the end of words
+    str = str + ' ';
+    str = str.replaceAll('σ-', 'ς-');
+    str = str.replaceAll('σ ', 'ς ');
+    
+    return str.trim();
+}
+
+
+
+/** #2.3
+ * ---
+ * PRELOADED DATA
+ * in the future these data may be loaded from a database / api-endpoint
+ * -----------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////////
+
 
-// synonyms
+// synonyms ////////////////////////////////////////////////////////////////////
 // -----------------------------------------------------------------------------
 
 var synonyms = [];      // groups of synonyms
@@ -128,18 +214,30 @@ var synonyms_Originals = [
     'Καρτέλλα Καρτέλα Καρτέλλες'
 ]
 synonyms_Originals.forEach( grp => {
+    // remove accends so any replaces can be done flawlessly
+    grp = sanitize_GR(grp);     
+
+    // create array of synonyms;
+    // push it to sanitized and keyboardized arrays
     synonyms.push( grp.split(' ') );
-    synonym_kbs.push( kb_trans(grp).split(' ') );
+    synonym_kbs.push( keyboardize(grp).split(' ') );
 })
 
-// significant terms
+
+// significant terms ///////////////////////////////////////////////////////////
 // -----------------------------------------------------------------------------
-significantExceptios = '7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R'.split(' ')
+significantExceptios = sanitize_GR('7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ')
 
 
 // replaces (correcting descriptions)
-// = construct unequivocally liked words
+// = construct unequivocally liked words ///////////////////////////////////////
 // -----------------------------------------------------------------------------
+
+// NOTE: TODO:
+// in future implementations multi-word keywords
+// may use the non-breaking space as conecting character (\u00A0) instead of dush (-)
+// (or maybe both of them)
+// --- -- -- - - -
 replaces = [];
 replaceSource = [
     '3 ΑΛΦΑ ;3-ΑΛΦΑ ',
@@ -199,12 +297,14 @@ replaceSource = [
     'Χωρίς προσθήκη ζάχαρης ;Χωρίς-ζάχαρη'
 ];
 replaceSource.forEach( it => {
-    st = it.split(';');
+    st = sanitize_GR(
+            it
+        ).split(';');
     replaces.push({ src: st[0], trg: st[1] });
 });
 
 
-// words that shall not be searched first
+// words that shall not be searched first //////////////////////////////////////
 // -----------------------------------------------------------------------------
 var noRootKeywords = [];
 noRoot = [
@@ -281,63 +381,70 @@ noRoot = [
     'Καθαρισμού',
     'Ρούχων'
 ];
-noRoot.forEach( w => { noRootKeywords.push(kb_trans(w)); });
+noRoot.forEach( w => { noRootKeywords.push(keyboardize(w)); });
 
 
-// list of linked-words
+// list of linked-words ////////////////////////////////////////////////////////
 // -----------------------------------------------------------------------------
 linkedWords = [
-    'Χωρίς-Γλουτένη',
-    'Χωρίς-Ζάχαρη',
-    'Χωρίς-Αλάτι',
-    'Χωρίς-Λακτόζη',
-    'Χωρίς-Συντηρητικά',
-    'Χωρίς-Αλκοόλ',
-    'Χωρίς-Kαφεϊνη',
-    'Χωρίς-Γλυκάνισο',
-    'Χωρίς-Ανθρακικό',
-    'Υψηλής-Παστερίωσης',
-    'Ολικής-Άλεσης',
-    'Ολικής-Aλέσεως',
-    'Χαρτί-Υγείας',
-    'ρολό-υγείας',
-    'χαρτί-τουαλέτας',
-    'Χαρτί-Κουζίνας',
-    'Μπάρες-Δημητριακών',
-    'Μπαρμπα-Στάθης',
-    'COCA-COLA',
-    'Aς-Μαγειρέψουμε',
-    'ΚΡΙΣ-ΚΡΙΣ',
-    'ΚΡΙ-ΚΡΙ',
-    'ΕΛ-ΓΚΡΕΚΟ',
-    'FREE-STEP',
-    'EL-SABOR',
-    'LE-PETIT-MARSEILLAIS',
-    'DOUWE-EGBERTS',
-    'ΕΝ-ΕΛΛΑΔΙ',
-    'SPIN-SPAN',
-    'CRETA-FARMS',
-    'CRETA-FARM',
-    'NES-CAFE',
-    'Ολες-τις-Χρήσεις',
-    'Το-Μάννα',
-    'Χωρίς-προσθήκη-ζάχαρης'
+    sanitize_GR('Χωρίς-Γλουτένη'),
+    sanitize_GR('Χωρίς-Ζάχαρη'),
+    sanitize_GR('Χωρίς-Αλάτι'),
+    sanitize_GR('Χωρίς-Λακτόζη'),
+    sanitize_GR('Χωρίς-Συντηρητικά'),
+    sanitize_GR('Χωρίς-Αλκοόλ'),
+    sanitize_GR('Χωρίς-Kαφεϊνη'),
+    sanitize_GR('Χωρίς-Γλυκάνισο'),
+    sanitize_GR('Χωρίς-Ανθρακικό'),
+    sanitize_GR('Υψηλής-Παστερίωσης'),
+    sanitize_GR('Ολικής-Άλεσης'),
+    sanitize_GR('Ολικής-Aλέσεως'),
+    sanitize_GR('Χαρτί-Υγείας'),
+    sanitize_GR('ρολό-υγείας'),
+    sanitize_GR('χαρτί-τουαλέτας'),
+    sanitize_GR('Χαρτί-Κουζίνας'),
+    sanitize_GR('Μπάρες-Δημητριακών'),
+    sanitize_GR('Μπαρμπα-Στάθης'),
+    sanitize_GR('COCA-COLA'),
+    sanitize_GR('Aς-Μαγειρέψουμε'),
+    sanitize_GR('ΚΡΙΣ-ΚΡΙΣ'),
+    sanitize_GR('ΚΡΙ-ΚΡΙ'),
+    sanitize_GR('ΕΛ-ΓΚΡΕΚΟ'),
+    sanitize_GR('FREE-STEP'),
+    sanitize_GR('EL-SABOR'),
+    sanitize_GR('LE-PETIT-MARSEILLAIS'),
+    sanitize_GR('DOUWE-EGBERTS'),
+    sanitize_GR('ΕΝ-ΕΛΛΑΔΙ'),
+    sanitize_GR('SPIN-SPAN'),
+    sanitize_GR('CRETA-FARMS'),
+    sanitize_GR('CRETA-FARM'),
+    sanitize_GR('NES-CAFE'),
+    sanitize_GR('Ολες-τις-Χρήσεις'),
+    sanitize_GR('Το-Μάννα'),
+    sanitize_GR('Χωρίς-προσθήκη-ζάχαρης')
 ];
 
 
-// list of words to exclude from keywords
+// list of words to exclude from keywords //////////////////////////////////////
 // -----------------------------------------------------------------------------
 // NOTE: APPLIED in PER-WORD base -> after spliting description to words
 removeList = []
-removeOriginals = 'κατά παρά υπό από μετά προς μας με σε για του της των από στο το στον & r s ft l τ e g h k m n o p s x'.split(' ')
-removeOriginals.forEach( w => { removeList.push( kb_trans(w)); });
+removeOriginals = sanitize_GR(
+    'κατά παρά υπό από μετά προς μας με σε για του της των από στο το στον & r s ft l τ e g h k m n o p s x'
+)
+.split(' ')
+removeOriginals.forEach( w => { removeList.push( keyboardize(w)); });
 
 
 
 
-// #3
-// SUPPORTING FUNCTIONS
-////////////////////////////////////////////////////////////////////////////////
+
+
+/** #3
+ * ---
+ * SUPPORTING FUNCTIONS
+ * -----------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////////
 
 
 /** freq
@@ -354,10 +461,12 @@ function freq_of(id) {
 }
 
 
-
-// proccess data
-// The MAIN proccess 
-// -----------------------------------------------------------------------------
+/**  proccess data
+ * --- -- -- - - -
+ * 
+ * central proccessing procedure
+ * -----------------------------------------------------------------------------
+ */
 function extract_linked_keywords(obj) {   
     // console.log(JSON.stringify(obj, null, 2));
 
@@ -379,7 +488,7 @@ function extract_linked_keywords(obj) {
 
             // filter words; keep only significant
             words.forEach( w => {
-                if (removeList.indexOf(kb_trans(w)) == -1)      // if not excluded
+                if (removeList.indexOf(keyboardize(w)) == -1)      // if not excluded
                     if (is_significant(w))                  // and significant
                         keys.push(w);               // add it to keys
             });
@@ -389,7 +498,7 @@ function extract_linked_keywords(obj) {
                 var wl = synonym_keys(w);
 
                 // if key CAN be a root word (not a no-Root-keyword) update root-node
-                if (noRootKeywords.indexOf(kb_trans(w)) == -1) {
+                if (noRootKeywords.indexOf(keyboardize(w)) == -1) {
 
                     root_key(wl, fq);       // update root keyword stats
 
@@ -431,13 +540,19 @@ function extract_linked_keywords(obj) {
 // functions for linking words in keywords dictionary
 ////////////////////////////////////////////////////////////////////////////////
 
-// set root-keyword: wl (if not exist)
-// update frequency: f
-// * wl is a list of synonym-words
-// ** comparison is based on the *keyboard* format
-//  ---
+
+/** root_key( wl, f )
+ * --- -- -- - - -
+ * set root keyword: wl (if not exist)
+ * update frequency: f
+ * 
+ * @param wl (list) :a list of synonym-words
+ * @param f (int): frequency of product
+ * 
+ * ** comparison is based on the *keyboard* format
+ */
 function root_key ( wl, f ) {
-    var wkb = kb_trans(wl[0])     // cache kb format
+    var wkb = keyboardize(wl[0])     // cache kb format
 
     // check if exists in root keys already
     // NOTE: you only need to check the 1st word of synonyms-list
@@ -457,14 +572,17 @@ function root_key ( wl, f ) {
     return true;
 }
 
-
-// connect keys: a , b (each one is a list of synonmyms)
-// of product with id: i
-// with frequency: f
-// ---
+/** connect_keys
+ * --- -- -- - - -
+ * @param (list) : connect keyword a (synonyms list)
+ * @param (list) : with keyword b (synonyms list)
+ * @param (int)  : id (eys code) of product where keywords a and b are linked
+ * @param (int)  : frequency of product (now many times has been purchased)
+ * @returns true
+ */
 function connect_keys( a, b, id, f ) {
-    var kbA = kb_trans(a[0]);
-    var kbB = kb_trans(b[0]);
+    var kbA = keyboardize(a[0]);
+    var kbB = keyboardize(b[0]);
     var bExists = false;
 
     if (kbA == kbB)     return false;       // exclude just-in-case
@@ -502,28 +620,28 @@ function connect_keys( a, b, id, f ) {
 ////////////////////////////////////////////////////////////////////////////////
 
 
-// kb_trans translates string to keyboard-latin keys;
-// ---
-function kb_trans(str) {
-    str = str.replace('\'','');
-    var out = '';
-    for (var i=0 ; i< str.length; i++)  out += map.get(str[i]);
-    return out;
-}
 
 
-// clean text trims some characters (+.') and internal multiple-spaces
-// ---
+/** clean text
+ * --- -- -- - - -
+ * removes non keyword characters [\+, \., \'] and internal multiple-spaces
+ * @param txt (string): product description
+ */
 function clean_text(txt) {
-    return txt.replace('+',' ').replace('.',' ')
+    return sanitize_GR(txt).replace('+',' ').replace('.',' ')
                 .replace('   ',' ')
                 .replace('  ',' ');
 }
 
 
-// check if term is significant
-// (if not, the term will be excluded from keywords dicionary)
-// ---
+/** is significant
+ * --- -- -- - - -
+ * check if term is significant (and needs to be indexed)
+ * (if not, the term will be excluded from keywords dicionary)
+ * 
+ * @param str
+ * @return (boolean)
+ */
 function is_significant(str) {
     if (str == '')     return false;
     if (significantExceptios.indexOf(str) !== -1)   return true;
@@ -531,11 +649,18 @@ function is_significant(str) {
 }
 
 
-// check if word: w
-// ...has synonyms; return list of synonyms
-// ---
+/** synonym_keys
+ * --- -- -- - - -
+ * check if word: w
+ * has synonyms; return list of synonyms
+ * 
+ * @param w (string)
+ * @return list of synonyms
+ * 
+ * NOTE: in no synonyms exist, retutn the word as a list
+ */
 function synonym_keys(w) {
-    w_kb = kb_trans(w);
+    w_kb = keyboardize(w);
     for (i=0 ; i < synonym_kbs.length ; i++) {
         if (synonym_kbs.indexOf(w_kb) !== -1)
             return synonyms[i];
@@ -544,9 +669,20 @@ function synonym_keys(w) {
 }
 
 
+
+/** mark_explicit_links
+ * 
+ * mark linked words
+ * shall be handled as one-(key)word
+ * 
+ * @param str 
+ * @return 
+ */
+
 // edit common mistakes
 // with suggested replaces
-function do_replaces(str) {
+function mark_explicit_links(str) {
+
     replaces.forEach( it => { str = str.replace(it.src, it.trg); });
     return str;
 }
@@ -555,8 +691,11 @@ function do_replaces(str) {
 // preproccess description
 // ---
 function preproccess_text(str) {
-    str = clean_text(str);
-    str = do_replaces(str);
+
+    str = clean_text(str);  // remove non-keyword characters
+
+    str = mark_explicit_links(sanitize_GR(str));    // mark explicit links
+    
     replaces.forEach( lw => {
         reg = new RegExp( lw.src, "gi");    // prepare regex for case insensitive (i) replace all (g)
         str = str.toLowerCase().replace(reg, lw.trg);
@@ -566,61 +705,47 @@ function preproccess_text(str) {
 }
 
 
-// DEPRICATED:
-// mark linked words (connect them with a dash)
-// return new text after "all-links" are marked
-// ---
-function mark_linked_words(txt) {
-    linkedWords.forEach( lw => {
 
-        txt = mark_link( lw, txt );
-    });
-    return txt
-}
+/* ---
+ DEPRICATED:
+    // mark linked words (connect them with a dash)
+    // return new text after "all-links" are marked
+    // ---
+    function mark_linked_words(txt) {
+        linkedWords.forEach( lw => {
 
-
-// TODO:
-// update function
-// new algorithm steps:
-// 
-// str.toLowerCase().replace(/αλφα βητα/gi, 'α-β'); console.log(replacedOnce);
-// 
-// mark a link (lws) to a text (source)
-// conecting them with a dash/minus character
-// ---
-function mark_link(lws, source) {
-    var src_kb = kb_trans(source.replace(' ', '-'));
-    var lws_kb = kb_trans(lws.replace(' ', '-'));
-    var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase())
-    if (_left !== -1 ) {
-        return source.slice(0, _left) + lws + source.slice(_left + lws.length);
+            txt = mark_link( lw, txt );
+        });
+        return txt
     }
-    else return source;
-}
+*/
 
 
-accented_vowels = [];
-[
-    'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω',
-    'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω'
-].forEach( pair => {
-    ab = pair.split(' ');
-    accented_vowels.push({
-        a: ab[0],   // accented
-        p: ab[1]    // pure
-    });
-})
 
-// accendless
-// replaces accended vowels by non accended ones
-function accendless(str) {
+/* ---
+ DEPRICATED:
+    // mark a link (lws) to a text (source)
+    // conecting them with a dash/minus character
+    // ---
+    function mark_link(lws, source) {
+        var src_kb = keyboardize(source.replace(' ', '-'));
+        var lws_kb = keyboardize(lws.replace(' ', '-'));
+        var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase())
+        if (_left !== -1 ) {
+            return source.slice(0, _left) + lws + source.slice(_left + lws.length);
+        }
+        else return source;
+    }
+*/
 
-}
 
 
-// #4
-// OUTPUT FUNCTIONS
-////////////////////////////////////////////////////////////////////////////////
+
+/** #4
+ * ---
+ * OUTPUT FUNCTIONS
+ * -----------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////////
 
 
 
@@ -667,9 +792,14 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) {
 
 
 
-// #5
-// MAIN function (exposed function to run the whole proccess)
-////////////////////////////////////////////////////////////////////////////////
+
+
+/** #5
+ * ---
+ * MAIN function (exposed function to run the whole proccess)
+ * -----------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////////
+
 
 
 /** main()