path: root/javascript/keygen.js

/** keygen
 * 
 * this script constucts a linked-wordkeys structure
 * -----------------------------------------------------------------------------
 * 
 * Contents:
 * ---
 * #1 Requirements
 *  -* fs
 *  -* os
 *  -* node-fetch
 *  -* @google-cloud/storage
 * 
 * #2.1 Personalized constants and parametres
 *  -* frequency endpoin
 *  -* products endpoint
 * 
 * #2.2 Purify string functions
 *  -* keyboardize()
 *  -* sanitize()
 *  -* clean()
 * 
 * #2.3 Preloaded Data  (TODO: load via endpoint)
 *  -* synonyms
 *  -* significant exceptions
 *  -* replaces (linked words)
 *  -* noRootKeywords
 *  -* removeList
 * 
 * #3 Supporting functions
 *  -* freq_of( product's eys-code)
 *  -* extract_linked_keywords -> NOTE: MAIN PROCEDURE to do the job
 *  -* root_key( word_list, frequency )
 *  -* connect_keys( wordlist_#1, wordlist_#2 , product_id, frequency )
 *  -* is_significant
 *  -* synonym_keys( word )
 *  -* mark_explicit_links()
 *  -* preproccess_text()
 * 
 * #4 Output functions
 *  -* save_local()
 *  -* upload_file
 * 
 * #5 Entry-point function main()
 * -* main()    NOTE: entry-point function
 */


/**  #1
 * ---
 * REQUIREMENTS
 * -----------------------------------------------------------------------------
 *//////////////////////////////////////////////////////////////////////////////

const fs = require('fs');
const os = require('os');
const fetch = require('node-fetch');
const {Storage} = require('@google-cloud/storage');     // Google Cloud Storage




/** #2.1
 * ---
 * SETUP PERSONALIZED CONSTANTS AND PARAMETRES
 * -----------------------------------------------------------------------------
 *//////////////////////////////////////////////////////////////////////////////


// get products FROM api/endpoint parametres
// --- -- -- - - -
const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json';
const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch";
const request_settings = { method: "Get" };


// google-storage parametres
// --- -- -- - - -
const projectId = 'pythia-251711';
const keyFilename = '../auth/pythia-251711-047e3d5e6608.json';


// temp files
// need to be created prior to save-to-google-storage proccess
// because upload_file() uploads an existing local file to G-Storage
// --- -- -- - - -
const temp_kw_file = '../results/keywords.json';
const temp_prod_file = '../results/products.json';


// Global Variables
// -----------------------------------------------------------------------------

// Global arrays
// keyword links (word-links dictionary; array of objects)
// -----------------------------------------------------------------------------
var kwlinks_ = [];  // keyword-links structure (structure to construct)
var prods_ = [];    // products (array to construct)
var _fr =[];        // product frequencies array (as read from endpiont) 
var _pr =[];        // product array (as read from endpiont) 


// Console Colors
// -----------------------------------------------------------------------------
const Reset = "\x1b[0m";
const FgRed = "\x1b[31m";
const FgGreen = "\x1b[32m";





/** #2.2
 * ---
 * PURIFY STRING FUNCTIONS
 * -----------------------------------------------------------------------------
 *//////////////////////////////////////////////////////////////////////////////



/** keyboardize
 * -----------------------------------------------------------------------------
 * translates string to keyboard-latin keys
 * (the ones that used whan typing each letter of the word)
 * 
 * @param str (string): original string (utf8 of latin or greek subgroups)
 * @return (string): latin/ascii equivalent string
 */

// cache (keeo in global) any-character to keyboard-latin mapping
// --- -- -- - - -
var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
const map = new Map();
for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);

// "keyboardize" function
// --- -- -- - - -
function keyboardize(str) {
    str = str.replace('\'','');
    var out = '';
    // [map]'s implementation is 40x faster than [for]'s
    for (var i=0 ; i< str.length; i++)  out += map.get(str[i]);
    return out;
}




/**  sanitize_GR
 * -----------------------------------------------------------------------------
 * replaces greek accended vowels with non accended ones
 * takes care of sigma on the end of words
 * 
 * @param str (string)
 * @return sanitized string
 */

// first cache (=create a global array)
// of accended to non-accended vowels mapping
// --- -- -- - - -
accented_vowels = [];   
[
    'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ΐ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω',
    'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω'
].forEach( pair => {
    ap = pair.split(' ');
    accented_vowels.push({
        a: ap[0],   // accented
        p: ap[1]    // pure = non accended
    });
})

// the sanitize_GR function
// --- -- -- - - -
function sanitize_GR(str) {

    str = str.toLowerCase();

    // replace accended vowels with pure ones
    accented_vowels.forEach( v => {
        str = str.replaceAll(v.a, v.p);
    });

    // replace sigma on the end of words
    str = str + ' ';
    str = str.replaceAll('σ-', 'ς-');
    str = str.replaceAll('σ ', 'ς ');
    
    return str;
}


/** clean text
 * --- -- -- - - -
 * removes non keyword characters [+ . , !] and internal multiple-spaces
 * @param txt (string): product description
 */
function clean_text(txt) {
    return txt.replace('+',' ').replace('.',' ').replace(',',' ')   // change to space
        .replace('!','').replace('\"', '')                  // remove character
        .replace('   ',' ').replace('  ',' ');          // remove multiple spaces
}



/** #2.3
 * ---
 * PRELOADED DATA
 * in the future these data may be loaded from a database / api-endpoint
 * -----------------------------------------------------------------------------
 * /////////////////////////////////////////////////////////////////////////////


/** synonyms ///////////////////////////////////////////////////////////////////
 * -----------------------------------------------------------------------------
 */

var synonyms = [];      // groups of synonyms
var synonym_kbs = [];   // cache kb-formats for performance
var synonyms_Originals = [
    'μπίρα μπύρα μπίρες μπύρες',
    'αυγά αβγά αυγό',
    'σίκαλης σικάλεως',
    'ξηρά ξερά',
    'ρολό ρολλό',
    'coca-cola cocacola coke',
    'max-factor, maxfactor',
    'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας',
    'χαρτί-κουζίνας ρολό-κουζίνας',
    'οινος κρασι',
    'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ',
    'DR-OETKER OETKER',
    'DR.BECKMANN BECKMANN',
    'NES-CAFE NESCAFE',
    'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής',
    'τσίπουρο ρακή',
    'Βρώμη Βρώμης',
    'Φράουλα Φράουλες Φράουλας',
    'Μαλλιά Μαλλιών',
    'Κέικ Cake',
    'CRETA-FARMS CRETA-FARM',
    'MARSEILLAIS LE-PETIT-MARSEILLAIS PETIT-MARSEILLAIS',
    'Γαϊδούρας Γαϊδάρου',
    'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ',
    'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ',
    'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ',
    'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ',
    'Ντομάτα Ντομάτας Τομάτα',
    'Ελαφρύ Ελαφρά Light',
    'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά Ελληνικός',
    'τριμμένη τριμμένο τριμμένα',
    'Τόνος Τόνου',
    'Κριθαρένια κρίθινα',
    'Χωρίς-Kαφεϊνη Decaffeine',
    'Το-Μάννα Μάννα',
    'Κράνμπερι Κράνμπερις',
    'Κρήτης Κρητικό Κρητική',
    'Πέννες Πένες',
    'σοκολατένιος σοκολατένια σοκολατένιο',
    'Μακαρόνια Ζυμαρικά Σπαγγέτι Σπαγγετίνι Σπαγγετόνι',
    'Λαζάνια Λιγκουίνι Ματσάτα Linguine',
    'Καρτέλλα Καρτέλα Καρτέλλες',
    'κλασική κλασικός κλασικό',
    'tea τσάι',
    'ωτοκαθαριστές μπατονέτες'
]
synonyms_Originals.forEach( grp => {
    // remove accends so any replaces can be done flawlessly
    grp = sanitize_GR(grp).trim();     

    // push it to array of synonyms (in lowerCase);
    synonyms.push( grp.toLowerCase().split(' ') );

    // push it the keyboardized array of synonyms
    synonym_kbs.push( keyboardize(grp).toLowerCase().split(' ') );
})

// console.log(synonyms); process.exit();
// console.log(synonym_kbs); process.exit();



/** significant terms //////////////////////////////////////////////////////////
 * -----------------------------------------------------------------------------
 */
significantExceptios = sanitize_GR('7-UP 7UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ')


/** replaces (correcting descriptions)
 * = construct unequivocally liked words ///////////////////////////////////////
 * -----------------------------------------------------------------------------
 * 
 * NOTE: TODO:
 * in future implementations multi-word keywords
 * may use the non-breaking space as conecting character (\u00A0) instead of dush (-)
 * (or maybe both of them)
 * --- -- -- - - -
 */
replaces = [];
replaceSource = [
    '3Α;3-ΑΛΦΑ',
    '3 ΑΛΦΑ;3-ΑΛΦΑ',
    'HEAD & SHOULDERS;HEAD&SHOULDERS',
    'HEAD N SHOULDERS;HEAD&SHOULDERS',
    'W.K Kellogg; W-K-Kellogg',
    'W.K Kellogg;',
    '7 DAYS;7-DAYS',
    '7 UP;7UP',
    '7-UP;7UP',
    'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ',
    'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ',
    'COCA COLA;COCA-COLA',
    'J P. CHENET; J.P.CHENET',
    'J.P. CHENET; J.P.CHENET',
    'COCACOLA;COCA-COLA',
    'NES CAFE;NESCAFE',
    'NES-CAFE;NESCAFE',
    'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS',
    'PETIT MARSEILLAIS;PETIT-MARSEILLAIS',
    'Το Μάννα;Το-Μάννα',
    'Χωρίς Γλουτένη;Χωρίς-Γλουτένη',
    'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη',
    'Χωρίς Αλάτι;Χωρίς-Αλάτι',
    'Χωρίς Λακτόζη;Χωρίς-Λακτόζη',
    'Χωρίς Συντηρητικά;Χωρίς-Συντηρητικά',
    'Χωρίς Αλκοόλ;Χωρίς-Αλκοόλ',
    'Χωρίς Καφεϊνη;Χωρίς-καφεϊνη',
    'Χωρίς Καφεΐνη;Χωρίς-καφεϊνη',
    'Χωρίς Γλυκάνισο;Χωρίς-Γλυκάνισο',
    'Χωρίς Ανθρακικό;Χωρίς-Ανθρακικό',
    'Υψηλής Παστερίωσης;Υψηλής-Παστερίωσης',
    'Ολικής Άλεσης;Ολικής-Άλεσης',
    'Ολικής Aλέσεως;Ολικής-Aλέσεως',
    'Χαρτί Υγείας;Χαρτί-Υγείας',
    'ρολό υγείας;ρολό-υγείας',
    'χαρτί τουαλέτας;χαρτί-τουαλέτας',
    'Χαρτί Κουζίνας;Χαρτί-Κουζίνας',
    'ρολό κουζίνας;ρολό-κουζίνας',
    'Μπάρες Δημητριακών;Μπάρες-Δημητριακών',
    'Ας Μαγειρέψουμε;Ας-Μαγειρέψουμε',
    'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ',
    'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ',
    'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ',
    'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ',
    'ΕΛ ΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ',
    'ΕΛΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ',
    'FREE STEP;FREE-STEP',
    'EL SABOR;EL-SABOR',
    'ELSABOR;EL-SABOR',
    'DOUWE EGBERTS;DOUWE-EGBERTS',
    'DOUWEEGBERTS;DOUWE-EGBERTS',
    'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ',
    'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ',
    'SPIN SPAN;SPIN-SPAN',
    'SPINSPAN;SPIN-SPAN',
    'CRETA-FARMS;CRETA-FARM',
    'CRETA-FARM;CRETA-FARM',
    'CRETAFARM;CRETA-FARM',
    'Ολες-τις-Χρήσεις;Ολες-τις-Χρήσεις',
    'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη',
    'φρουι ζελε, φρουί-ζελε',
    'DR BECKMANN, DR-BECKMANN',

    'Χωρίς Kαφεϊνη;Χωρίς-καφεϊνη',
    'Χωρίς Kαφεΐνη;Χωρίς-καφεϊνη',
];
replaceSource.forEach( it => {
    st = sanitize_GR(
            it.toLowerCase()
        ).split(';');
    replaces.push({     
        src: ' '+ st[0] +' ',   // encolse between spaces
        trg: ' '+ st[1] +' '    // to separate from before/after words
    });
});

// console.log(replaces); process.exit();


// words that shall not be searched first //////////////////////////////////////
// -----------------------------------------------------------------------------
var noRootKeywords = [];
noRoot = [
    'χωρίς',
    'εισαγωγής',
    'δώρο',
    'γεύση',
    'γεύσεις',
    'φέτες',
    'Χωρίς-Γλουτένη',
    'Γλουτένη',
    'Λακτόζη',
    'Παστερίωσης',
    'Ανθρακικό',
    'Συντηρητικά',
    'Χωρίς-Ζάχαρη',
    'Άλεσης',
    'Χωρίς-Αλάτι',
    'Χωρίς-Λακτόζη',
    'Χωρίς-Συντηρητικά',
    'Χωρίς-Αλκοόλ',
    'Χωρίς-Kαφεϊνη',
    'Χωρίς-Γλυκάνισο',
    'Χωρίς-Ανθρακικό',
    'Υψηλής-Παστερίωσης',
    'Ολες-τις-Χρήσεις',
    'Ολικής-Άλεσης',
    'Ολικής-Aλέσεως',
    'Ολικής',
    'Σοκολάτας',
    'Υγείας',
    'Γαϊδούρας',
    'Δημητριακών',
    'Γαϊδάρου',
    'Ρούχων',
    'Πιάτων',
    'πλύσεις',
    'Πλυντηρίου',
    'Φύλλων',
    'Γάλακτος',
    'Χρήσης',
    'Τύπου',
    'Ολλανδίας',
    'Απορριμμάτων',
    'Medium',
    'Μαλλιά',
    'Μαλλιών',
    'Γενικής',
    'Plus',
    'Classic',
    'Έκπληξη',
    'Μάνης',
    'Ελάτου',
    'Άγριων',
    'Βοτάνων',
    'Κατσαρίδες',
    'Λακωνίας',
    'ΠΑΡΑΓΓΕΛΙΩΝ',
    'Λήμνου',
    'Αργινίνης',
    'Καλαθάκι',
    'Σκύλου',
    'Γάτας',
    'ΠΓΕ',
    'ΤΟ',
    'Σώματος',
    'Αδυνατίσματος',
    'Προστασίας',
    'Ακράτειας',
    'Προσώπου',
    'προσθήκη',
    'Βρεφικής',
    'Καθαρισμού',
    'Ρούχων',
    'birthday',
    'δημιουργικοτητας'
];
noRoot.forEach( w => { noRootKeywords.push(keyboardize(w)); });


// list of linked-words ////////////////////////////////////////////////////////
// -----------------------------------------------------------------------------
linkedWords = [
    sanitize_GR('Χωρίς-Γλουτένη'),
    sanitize_GR('Χωρίς-Ζάχαρη'),
    sanitize_GR('Χωρίς-Αλάτι'),
    sanitize_GR('Χωρίς-Λακτόζη'),
    sanitize_GR('Χωρίς-Συντηρητικά'),
    sanitize_GR('Χωρίς-Αλκοόλ'),
    sanitize_GR('Χωρίς-Kαφεϊνη'),
    sanitize_GR('Χωρίς-Γλυκάνισο'),
    sanitize_GR('Χωρίς-Ανθρακικό'),
    sanitize_GR('Υψηλής-Παστερίωσης'),
    sanitize_GR('Ολικής-Άλεσης'),
    sanitize_GR('Ολικής-Aλέσεως'),
    sanitize_GR('Χαρτί-Υγείας'),
    sanitize_GR('ρολό-υγείας'),
    sanitize_GR('χαρτί-τουαλέτας'),
    sanitize_GR('Χαρτί-Κουζίνας'),
    sanitize_GR('Μπάρες-Δημητριακών'),
    sanitize_GR('Μπαρμπα-Στάθης'),
    sanitize_GR('COCA-COLA'),
    sanitize_GR('Aς-Μαγειρέψουμε'),
    sanitize_GR('ΚΡΙΣ-ΚΡΙΣ'),
    sanitize_GR('ΚΡΙ-ΚΡΙ'),
    sanitize_GR('ΕΛ-ΓΚΡΕΚΟ'),
    sanitize_GR('FREE-STEP'),
    sanitize_GR('EL-SABOR'),
    sanitize_GR('LE-PETIT-MARSEILLAIS'),
    sanitize_GR('DOUWE-EGBERTS'),
    sanitize_GR('ΕΝ-ΕΛΛΑΔΙ'),
    sanitize_GR('SPIN-SPAN'),
    sanitize_GR('CRETA-FARMS'),
    sanitize_GR('CRETA-FARM'),
    sanitize_GR('NES-CAFE'),
    sanitize_GR('Ολες-τις-Χρήσεις'),
    sanitize_GR('Το-Μάννα'),
    sanitize_GR('Χωρίς-προσθήκη-ζάχαρης'),
    sanitize_GR('φρουί-ζελέ')
];


// list of words to exclude from keywords //////////////////////////////////////
// -----------------------------------------------------------------------------
// NOTE: APPLIED in PER-WORD base -> after spliting description to words
removeList = []
removeOriginals = sanitize_GR([
    'κατά παρά υπό από μετά προς',
    'μας με σε για του της τον τη την των από στο το στον',
    '& r s ft l τ e g h k m n o p s x c ω',
    'cc cv gr lt'
].join(' ')).split(' ')
removeOriginals.forEach( w => { removeList.push( keyboardize(w)); });






/** #3
 * ---
 * SUPPORTING FUNCTIONS
 * -----------------------------------------------------------------------------
 *//////////////////////////////////////////////////////////////////////////////


/** freq
 * 
 * @param id (int): product's eys_code
 * @return frequency of product
 */
function freq_of(id) {
    _fr.forEach( rec => {
        if (rec.id == id) return rec.fq
    });

    return 0;
}


/**  proccess data /////////////////////////////////////////////////////////////
 * --- -- -- - - -
 * 
 * central proccessing procedure
 * -----------------------------------------------------------------------------
 * 
 * Operation:
 * ---
 * for each title {
 *   + clean text, get frequency
 *   + preproccess and split into individual words
 *   + for each significant word {
 *       + update root keywords
 *       + update linked words
 *   }
 * }
 * post-process results
 * 
 * /////////////////////////////////////////////////////////////////////////////
 */
function extract_linked_keywords(obj) {   
    // console.log(JSON.stringify(obj, null, 2));

    obj.forEach( rec => {

        if (rec.img != 0) {

            //var description = preproccess_text(rec.txt);

            var description = clean_text(rec.txt);

            var pid = rec.eys;
            var fq = freq_of(pid)+1;

            prods_.push({
                id: pid,
                w: description.trim()
            })

            var keys = [];
            var words = preproccess_text(description).split(' ');

            // filter words; keep only significant
            words.forEach( w => {
                if (removeList.indexOf(keyboardize(w)) == -1)      // if not excluded
                    if (is_significant(w))                  // and significant
                        keys.push(w);               // add it to keys
            });
            // console.log(pid, description, keys);

            keys.forEach( w => {
                var wl = synonym_keys(w);

                // if key CAN be a root word (not a no-Root-keyword) update root-node
                if (noRootKeywords.indexOf(keyboardize(w)) == -1) {

                    root_key(wl, fq);       // update root keyword stats

                    // if key is the only in the list of product's keywords
                    // connect it with a dummy key (to preserve the reference to the product)
                    if (keys.length == 1)  connect_keys(wl, ['*'], pid, fq);

                    // connect w with all the other product's keywords
                    keys.forEach( w2 => {
                        if (w2 != w) {
                            var w2syns = synonym_keys(w2);
                            connect_keys( wl, w2syns, pid, fq);
                        }
                    });
                }

            });
        }
    });    // main proccessing finished;
    
    // post proccess 
    // -------------------------------------------------------------------------

    // remove cached keys from final array
    kwlinks_.forEach( ro => { 
        delete ro.kb;
        ro.c.forEach( ch => { delete ch.kb; });
    });

    // sort root and child nodes by frequency descanding
    kwlinks_.forEach( it => {
        it.c = it.c.sort((a, b) => b.f - a.f );
    });
    kwlinks_ = kwlinks_.sort((a, b) => b.f - a.f );

}


// functions for linking words in keywords dictionary
////////////////////////////////////////////////////////////////////////////////


/** root_key( wl, f )
 * --- -- -- - - -
 * set root keyword: wl (if not exist)
 * update frequency: f
 * 
 * @param wl (list) :a list of synonym-words
 * @param f (int): frequency of product
 * 
 * ** comparison is based on the *keyboard* format
 */
function root_key ( wl, f ) {
    var wkb = keyboardize(wl[0])     // cache kb format

    // check if exists in root keys already
    // NOTE: you only need to check the 1st word of synonyms-list
    for (i=0; i< kwlinks_.length ; i++) {
        if (kwlinks_[i].kb == wkb) {
            kwlinks_[i].f += f;
            return true;
        }
    }
    // if not exists, append keyword
    kwlinks_.push({
        w  : wl,
        kb : wkb,
        f  : f,
        c  : []
    });
    return true;
}

/** connect_keys
 * --- -- -- - - -
 * @param (list) : connect keyword a (synonyms list)
 * @param (list) : with keyword b (synonyms list)
 * @param (int)  : id (eys code) of product where keywords a and b are linked
 * @param (int)  : frequency of product (now many times has been purchased)
 * @returns true
 */
function connect_keys( a, b, id, f ) {
    var kbA = keyboardize(a[0]);
    var kbB = keyboardize(b[0]);
    var bExists = false;

    if (kbA == kbB)     return false;       // exclude just-in-case
    
    for (i=0; i< kwlinks_.length ; i++) {
        if (kwlinks_[i].kb == kbA) {        // found: a;
            // update connection to: b
            bExists = false;
            for (j=0 ; j < kwlinks_[i].c.length ; j++) {
                if (kwlinks_[i].c[j].kb == kbB) {
                    bExists = true;
                    // update the connection's data
                    kwlinks_[i].c[j].f += f;
                    kwlinks_[i].c[j].p.push(id)
                    break;
                }
            }
            // if connection not exist, init a new one
            if (bExists == false) {
                // create connection with: b
                kwlinks_[i].c.push({
                    w  : b,
                    kb : kbB,
                    f  : f,
                    p  : [ id ]
                });
            }
            return true;
        }
    }
}


// other supplementary functions
////////////////////////////////////////////////////////////////////////////////




/** is significant
 * --- -- -- - - -
 * check if term is significant (and needs to be indexed)
 * (if not, the term will be excluded from keywords dicionary)
 * 
 * @param str
 * @return (boolean)
 */
function is_significant(str) {
    if (str == '')     return false;
    if (significantExceptios.indexOf(str) !== -1)   return true;
    return !(/\d/.test(str));
}


/** synonym_keys
 * --- -- -- - - -
 * check if word: w
 * has synonyms; return list of synonyms
 * 
 * @param w (string)
 * @return list of synonyms
 * 
 * NOTE: in no synonyms exist, retutn the word as a list
 */
function synonym_keys(w) {
    w_kb = keyboardize(w);
    for (i=0 ; i < synonym_kbs.length ; i++) {
        if (synonym_kbs[i].indexOf(w_kb) !== -1)
            return synonyms[i];
    }
    return [ w ];
}


/** mark_explicit_links
 * 
 * mark linked words shall be handled as one-(key)word
 * (also edit common mistakes with suggested replaces)
 * 
 * @param str 
 * @return 
 */
function mark_explicit_links(str) {
    str = ' '+ str +' ';
    replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); });
    return str.replaceAll('  ', ' ').trim();
}


/** preproccess description
 * == do...
 * + clean
 * + sanitize
 * + mark lined words
 */ 
function preproccess_text(description) {

    str = clean_text(description);  // remove non-keyword characters

    str = mark_explicit_links(sanitize_GR( str.toLowerCase() ));    // mark explicit links
    
    // replaces.forEach( lw => {
    //     reg = new RegExp( lw.src, "gi");    // prepare regex for case insensitive (i) replace all (g)
    //     str = str.toLowerCase().replace(reg, lw.trg);
    // })
    // depricated: str = mark_linked_words(str);
    return str;
}


/* ---
 DEPRICATED:
    // mark linked words (connect them with a dash)
    // return new text after "all-links" are marked
    // ---
    function mark_linked_words(txt) {
        linkedWords.forEach( lw => {

            txt = mark_link( lw, txt );
        });
        return txt
    }
*/


/* ---
 DEPRICATED:
    // mark a link (lws) to a text (source)
    // conecting them with a dash/minus character
    // ---
    function mark_link(lws, source) {
        var src_kb = keyboardize(source.replace(' ', '-'));
        var lws_kb = keyboardize(lws.replace(' ', '-'));
        var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase())
        if (_left !== -1 ) {
            return source.slice(0, _left) + lws + source.slice(_left + lws.length);
        }
        else return source;
    }
*/


/** #4
 * ---
 * OUTPUT FUNCTIONS
 * -----------------------------------------------------------------------------
 *//////////////////////////////////////////////////////////////////////////////



// Save Local
// -----------------------------------------------------------------------------
async function save_local(jsonArray, filePath) {
    let fileStr = JSON.stringify(jsonArray);    // convert json to string

    // write string to file
    fs.writeFileSync(filePath, fileStr, 'utf8', (err) => {
        if (err) {
            return err;
        } else {
            return true;
        }
    });
}



// Save to Google Cloud Storage
// -----------------------------------------------------------------------------
async function upload_file( bucketName, srcFilePath, trgFilePath ) {
    // Creates a client
    const storage = new Storage({projectId, keyFilename});

    try {
        await storage.bucket(bucketName).upload(srcFilePath, {
            destination: trgFilePath, 
            gzip: true,     // serve compressed
            metadata: {     // cache for 8 hours
                cacheControl: 'public, max-age=60'      // production set: 28800
            }
        });
        console.log(`${srcFilePath} uploaded to ${bucketName}`);
    }

    catch(err) {
        console.error('ERROR:', err);
    }

}




/** #5
 * ---
 * MAIN function (exposed function to run the whole proccess)
 * -----------------------------------------------------------------------------
 *//////////////////////////////////////////////////////////////////////////////



/** main()
 * 
 * nodejs's exported function
 * used as entry-point function (in case of Google Cloud function)
 * 
 * Operation
 * + 1: load products and frequencies
 * + 2: forward to extract_linked_keywords()
 * + 3: Save results (localy, then to cloud)
 */

// keep only 1 of next 2 lines
// exports.main = () => {       // entry point functioon when google cloud function
var main = () => {              // concise function when direct script

    // PROMISE ...
    // fetch all jsons from endpints ...........................................
    Promise.all([

        fetch(products_endpoint, request_settings)
            .then(res => res.json())
            .then((json) => {
                _pr = json;
                console.log('products data loaded from emarket api/endpoint');
            }),

        fetch(frequency_endpoint, request_settings)
            .then(res => res.json())
            .then((json) => {
                _fr = json;
                console.log('frequencies loaded from CDN')
            })

    ])
    .then (() => {  // then...
        // proccess data and extract ketwords

        console.log(`... All data from endpoints: ${FgGreen}LOADED!${Reset}`);

        // proccess data
        extract_linked_keywords(_pr.data);

        console.log('keywords extracted;\ntry to save results localy...');

        // PROMISE ...
        // save local files ....................................................
        Promise.all([
            save_local(kwlinks_, temp_kw_file)
            .then(() => { console.log('keywords saced localy');}),

            save_local(prods_, temp_prod_file)
            .then(() => { console.log('products saved localy');})
        ])
        .then(() => {   // then ...

            console.log(`... All local saves: ${FgGreen}DONE!${Reset}`);

            // PROMISE ...
            // upload keytword and product jsons into Cloud Storage ............
            Promise.all([
                upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'),
                upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json')
            ])
            .then(() => {   // then ...

                console.log(`... Uploads to cloud storage: ${FgGreen}DONE!${Reset}`);

                // echo usage stats; then END

                // check
                // echo 100 most frequent
                let i = 0;
                kwlinks_.forEach(rec => {
                    if (i<100) {
                        console.log(i, JSON.stringify(rec.w))
                    }
                    i++;
                });

                // echo memory stats
                const used = process.memoryUsage();
                for (let key in used) {
                    console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
                }

                console.log(`${FgGreen}THE END!${Reset}`);

            });

        });

    });

}


// NOTE:
// if running the script directry you need to call the main function
// if running through google cloud-functions you do NOT need to call main()
// (you need define main() as the entry-point function instead)
main();