path: root/javascript/keygen_GCF.js

/** keygen
 * 
 * this script constucts a linked-wordkeys structure
 * -----------------------------------------------------------------------------
 * 
 * Contents:
 * #1 Requirements
 * #2.1 Personalized constants and parametres
 * #2.2 Preloaded Data
 * #3 Supporting functions
 * #4 Output functions
 * #5 Entry-point function main()
 */

// #1
// REQUIREMENTS
////////////////////////////////////////////////////////////////////////////////

const fs = require('fs');
const os = require('os');
const fetch = require('node-fetch');
const {Storage} = require('@google-cloud/storage');     // Google Cloud Storage


// temp files
// need to be created prior to save-to-google-storage proccess
// because upload_file() uploats an existing local file to G-Storage
// --- -- -- - - -
const temp_kw_file = os.tmpdir() + '/keywords.json';
const temp_prod_file = os.tmpdir() + '/products.json';



// #2.1
// SETUP PERSONALIZED CONSTANTS AND PARAMETRES
////////////////////////////////////////////////////////////////////////////////


// get products FROM api/endpoint parametres
const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json';
const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch";
const request_settings = { method: "Get" };



// Global Variables
// -----------------------------------------------------------------------------

// Global arrays
// keyword links (word-links dictionary; array of objects)
// -----------------------------------------------------------------------------
var kwlinks_ = [];  // keyword-links structure (structure to construct)
var prods_ = [];    // products (array to construct)
var _fr =[];        // product frequencies array (as read from endpiont) 
var _pr =[];        // product array (as read from endpiont) 



// #2.2
// PRELOADED DATA
////////////////////////////////////////////////////////////////////////////////

// any-character to keyboard-latin mapping
// -----------------------------------------------------------------------------
var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
const map = new Map();
for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);


// synonyms
// -----------------------------------------------------------------------------

var synonyms = [];      // groups of synonyms
var synonym_kbs = [];   // cache kb-formats for performance
var synonyms_Originals = [
    'μπίρα μπύρα μπίρες μπύρες',
    'αυγά αβγά αυγό',
    'σίκαλης σικάλεως',
    'ξηρά ξερά',
    'ρολό ρολλό',
    'coca-cola cocacola coke',
    'max-factor, maxfactor',
    'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας',
    'χαρτί-κουζίνας ρολό-κουζίνας',
    'οινος κρασι',
    'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ',
    'DR-OETKER OETKER',
    'DR.BECKMANN BECKMANN',
    'NES-CAFE NESCAFE',
    'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής',
    'τσίπουρο ρακή',
    'Βρώμη Βρώμης',
    'Φράουλα Φράουλες Φράουλας',
    'Μαλλιά Μαλλιών',
    'Κέικ Cake',
    'CRETA-FARMS CRETA-FARM',
    'MARSEILLAIS LE-PETIT-MARSEILLAIS PETIT-MARSEILLAIS',
    'Γαϊδούρας Γαϊδάρου',
    'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ',
    'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ',
    'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ',
    'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ',
    'Ντομάτα Ντομάτας',
    'Ελαφρύ Ελαφρά Light',
    'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά',
    'τριμμένη τριμμένο',
    'Τόνος Τόνου',
    'Κριθαρένια κρίθινα',
    'Χωρίς-Kαφεϊνη Decaffeine',
    'Το-Μάννα Μάννα',
    'Κράνμπερι Κράνμπερις',
    'Κρήτης Κρητικό',
    'Πέννες Πένες',
    'Μακαρόνια Σπαγγέτι Σπαγγετίνι Σπαγγετόνι',
    'Καρτέλλα Καρτέλα Καρτέλλες'
]
synonyms_Originals.forEach( grp => {
    synonyms.push( grp.split(' ') );
    synonym_kbs.push( kb_trans(grp).split(' ') );
})

// significant terms
// -----------------------------------------------------------------------------
significantExceptios = '7UP 3ΑΛΦΑ 17 3Π 7DAYS K2R'.split(' ')


// replaces (correcting descriptions)
// -----------------------------------------------------------------------------
replaces = [];
replaceSource = [
    '3 ΑΛΦΑ ;3ΑΛΦΑ ',
    'HEAD & SHOULDERS ;HEAD&SHOULDERS ',
    'W.K Kellogg ; ',
    'ΦΙΛΕΤ ;Φιλέτο ',
    'ΕΝΕΛΛΑΔ ;Εν-Ελλάδι ',
    'ΓΑΛΟΠΟΥΛ ;Γαλοπούλα ',
    '7 DAYS ;7DAYS ',
    'ΜΠΑΡΜΠΑ ΣΤΑΘΗ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ ',
    'Zero% ;Zero ',
    
];
replaceSource.forEach( it => {
    st = it.split(';');
    replaces.push({ src: st[0], trg: st[1] });
});


// words that shall not be searched first
// -----------------------------------------------------------------------------
var noRootKeywords = [];
noRoot = [
    'χωρίς',
    'εισαγωγής',
    'δώρο',
    'γεύση',
    'γεύσεις',
    'φέτες',
    'Χωρίς-Γλουτένη',
    'Γλουτένη',
    'Λακτόζη',
    'Παστερίωσης',
    'Ανθρακικό',
    'Συντηρητικά',
    'Χωρίς-Ζάχαρη',
    'Άλεσης',
    'Χωρίς-Αλάτι',
    'Χωρίς-Λακτόζη',
    'Χωρίς-Συντηρητικά',
    'Χωρίς-Αλκοόλ',
    'Χωρίς-Kαφεϊνη',
    'Χωρίς-Γλυκάνισο',
    'Χωρίς-Ανθρακικό',
    'Υψηλής-Παστερίωσης',
    'Ολες-τις-Χρήσεις',
    'Ολικής-Άλεσης',
    'Ολικής-Aλέσεως',
    'Ολικής',
    'Σοκολάτας',
    'Υγείας',
    'Γαϊδούρας',
    'Δημητριακών',
    'Γαϊδάρου',
    'Ρούχων',
    'Πιάτων',
    'πλύσεις',
    'Πλυντηρίου',
    'Φύλλων',
    'Γάλακτος',
    'Χρήσης',
    'Τύπου',
    'Ολλανδίας',
    'Απορριμμάτων',
    'Medium',
    'Μαλλιά',
    'Μαλλιών',
    'Γενικής',
    'Plus',
    'Classic',
    'Έκπληξη',
    'Μάνης',
    'Ελάτου',
    'Άγριων',
    'Βοτάνων',
    'Κατσαρίδες',
    'Λακωνίας',
    'ΠΑΡΑΓΓΕΛΙΩΝ',
    'Λήμνου',
    'Αργινίνης',
    'Καλαθάκι',
    'Σκύλου',
    'Γάτας',
    'ΠΓΕ',
    'ΤΟ',
    'Μελί',
    'Σώματος',
    'Αδυνατίσματος',
    'Προστασίας',
    'Ακράτειας',
    'Προσώπου',
    'προσθήκη',
    'Βρεφικής',
    'Καθαρισμού',
    'Ρούχων'
];
noRoot.forEach( w => { noRootKeywords.push(kb_trans(w)); });


// list of linked-words
// -----------------------------------------------------------------------------
linkedWords = [
    'Χωρίς-Γλουτένη',
    'Χωρίς-Ζάχαρη',
    'Χωρίς-Αλάτι',
    'Χωρίς-Λακτόζη',
    'Χωρίς-Συντηρητικά',
    'Χωρίς-Αλκοόλ',
    'Χωρίς-Kαφεϊνη',
    'Χωρίς-Γλυκάνισο',
    'Χωρίς-Ανθρακικό',
    'Υψηλής-Παστερίωσης',
    'Ολικής-Άλεσης',
    'Ολικής-Aλέσεως',
    'Χαρτί-Υγείας',
    'ρολό-υγείας',
    'χαρτί-τουαλέτας',
    'Χαρτί-Κουζίνας',
    'Μπάρες-Δημητριακών',
    'Μπαρμπα-Στάθης',
    'COCA-COLA',
    'Aς-Μαγειρέψουμε',
    'ΚΡΙΣ-ΚΡΙΣ',
    'ΚΡΙ-ΚΡΙ',
    'ΕΛ-ΓΚΡΕΚΟ',
    'FREE-STEP',
    'EL-SABOR',
    'LE-PETIT-MARSEILLAIS',
    'DOUWE-EGBERTS',
    'ΕΝ-ΕΛΛΑΔΙ',
    'SPIN-SPAN',
    'CRETA-FARMS',
    'CRETA-FARM',
    'NES-CAFE',
    'Ολες-τις-Χρήσεις',
    'Το-Μάννα',
    'Χωρίς-προσθήκη-ζάχαρης'
];


// list of words to exclude from keywords
// -----------------------------------------------------------------------------
// NOTE: APPLIED in PER-WORD base -> after spliting description to words
removeList = []
removeOriginals = 'κατά παρά υπό από μετά προς μας με σε για του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ')
removeOriginals.forEach( w => { removeList.push( kb_trans(w)); });

// #3
// SUPPORTING FUNCTIONS
////////////////////////////////////////////////////////////////////////////////


/** freq
 * 
 * @param id (int): product's eys_code
 * @return frequency of product
 */
function freq_of(id) {
    _fr.forEach( rec => {
        if (rec.id == id) return rec.fq
    });

    return 0;
}


// proccess data
// The MAIN proccess 
// -----------------------------------------------------------------------------
function extract_linked_keywords(obj) {   
    // console.log(JSON.stringify(obj, null, 2));

    obj.forEach( rec => {

        if (rec.img != 0) {

            var description = preproccess_text(rec.txt);
            var pid = rec.eys;
            var fq = freq_of(pid)+1;

            prods_.push({
                id: pid,
                w: description
            })

            var keys = [];
            var words = description.split(' ');

            // filter words; keep only significant
            words.forEach( w => {
                if (removeList.indexOf(kb_trans(w)) == -1)      // if not excluded
                    if (is_significant(w))                  // and significant
                        keys.push(w);               // add it to keys
            });
            // console.log(pid, description, keys);

            keys.forEach( w => {
                var wl = synonym_keys(w);

                // if key CAN be a root word (not a no-Root-keyword) update root-node
                if (noRootKeywords.indexOf(kb_trans(w)) == -1) {

                    root_key(wl, fq);       // update root keyword stats

                    // if key is the only in the list of product's keywords
                    // connect it with a dummy key (to preserve the reference to the product)
                    if (keys.length == 1)  connect_keys(wl, ['*'], pid, fq);

                    // connect w with all the other product's keywords
                    keys.forEach( w2 => {
                        if (w2 != w) {
                            var w2syns = synonym_keys(w2);
                            connect_keys( wl, w2syns, pid, fq);
                        }
                    });
                }

            });
        }
    });    // main proccessing finished;
    
    // post proccess 
    // -------------------------------------------------------------------------

    // remove cached keys from final array
    kwlinks_.forEach( ro => { 
        delete ro.kb;
        ro.c.forEach( ch => { delete ch.kb; });
    });

    // sort root and child nodes by frequency descanding
    kwlinks_.forEach( it => {
        it.c = it.c.sort((a, b) => b.f - a.f );
    });
    kwlinks_ = kwlinks_.sort((a, b) => b.f - a.f );

}


// functions for linking words in keywords dictionary
////////////////////////////////////////////////////////////////////////////////

// set root-keyword: wl (if not exist)
// update frequency: f
// * wl is a list of synonym-words
// ** comparison is based on the *keyboard* format
//  ---
function root_key ( wl, f ) {
    var wkb = kb_trans(wl[0])     // cache kb format

    // check if exists in root keys already
    // NOTE: you only need to check the 1st word of synonyms-list
    for (i=0; i< kwlinks_.length ; i++) {
        if (kwlinks_[i].kb == wkb) {
            kwlinks_[i].f += f;
            return true;
        }
    }
    // if not exists, append keyword
    kwlinks_.push({
        w  : wl,
        kb : wkb,
        f  : f,
        c  : []
    });
    return true;
}


// connect keys: a , b (each one is a list of synonmyms)
// of product with id: i
// with frequency: f
// ---
function connect_keys( a, b, id, f ) {
    var kbA = kb_trans(a[0]);
    var kbB = kb_trans(b[0]);
    var bExists = false;

    if (kbA == kbB)     return false;       // exclude just-in-case
    
    for (i=0; i< kwlinks_.length ; i++) {
        if (kwlinks_[i].kb == kbA) {        // found: a;
            // update connection to: b
            bExists = false;
            for (j=0 ; j < kwlinks_[i].c.length ; j++) {
                if (kwlinks_[i].c[j].kb == kbB) {
                    bExists = true;
                    // update the connection's data
                    kwlinks_[i].c[j].f += f;
                    kwlinks_[i].c[j].p.push(id)
                    break;
                }
            }
            // if connection not exist, init a new one
            if (bExists == false) {
                // create connection with: b
                kwlinks_[i].c.push({
                    w  : b,
                    kb : kbB,
                    f  : f,
                    p  : [ i ]
                });
            }
            return true;
        }
    }
}


// other supplementary functions
////////////////////////////////////////////////////////////////////////////////


// kb_trans translates string to keyboard-latin keys;
// ---
function kb_trans(str) {
    str = str.replace('\'','');
    var out = '';
    for (var i=0 ; i< str.length; i++)  out += map.get(str[i]);
    return out;
}


// clean text trims some characters (+.') and internal multiple-spaces
// ---
function clean_text(txt) {
    return txt.replace('+',' ').replace('.',' ')
                .replace('   ',' ')
                .replace('  ',' ');
}


// check if term is significant
// (if not, the term will be excluded from keywords dicionary)
// ---
function is_significant(str) {
    if (str == '')     return false;
    if (significantExceptios.indexOf(str) !== -1)   return true;
    return !(/\d/.test(str));
}


// check if word: w
// ...has synonyms; return list of synonyms
// ---
function synonym_keys(w) {
    w_kb = kb_trans(w);
    for (i=0 ; i < synonym_kbs.length ; i++) {
        if (synonym_kbs.indexOf(w_kb) !== -1)
            return synonyms[i];
    }
    return [ w ];
}


// edit common mistakes
// with suggested replaces
function do_replaces(str) {
    replaces.forEach( it => { str = str.replace(it.src, it.trg); });
    return str;
}


// preproccess description
// ---
function preproccess_text(str) {
    str = clean_text(str);
    str = do_replaces(str);
    str = mark_linked_words(str);
    return str;
}


// mark linked words (connect them with a dash)
// return new text after "all-links" are marked
// ---
function mark_linked_words(txt) {
    linkedWords.forEach( lw => { txt = mark_link( lw, txt ); });
    return txt
}


// mark a link (lws) to a text (source)
// conecting them with a dash/minus character
// ---
function mark_link(lws, source) {
    var src_kb = kb_trans(source.replace(' ', '-'));
    var lws_kb = kb_trans(lws.replace(' ', '-'));
    var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase())
    if (_left !== -1 ) {
        return source.slice(0, _left) + lws + source.slice(_left + lws.length);
    }
    else return source;
}



// #4
// OUTPUT FUNCTIONS
////////////////////////////////////////////////////////////////////////////////


// Save Local
// -----------------------------------------------------------------------------
async function save_local(jsonArray, fileName) {
    let fileStr = JSON.stringify(jsonArray);    // convert json to string
    
    // write string to file
    fs.writeFileSync(fileName, fileStr, 'utf8', (err) => {
        if (err) {
            return err;
        }
        return true;

    });

    return true;
}


// Save to Google Cloud Storage
// -----------------------------------------------------------------------------
async function upload_file( bucketName, srcFilePath, trgFilePath ) {
    // Creates a client
    const storage = new Storage();

    try {
        await storage.bucket(bucketName).upload(srcFilePath, {
            destination: trgFilePath, 
            gzip: true,     // serve compressed
            metadata: {     // cache for 8 hours
                cacheControl: 'public, max-age=60'      // production set: 28800
            }
        });
        console.log(`${srcFilePath} uploaded to ${bucketName}`);
    }

    catch(err) {
        console.error('ERROR:', err);
    }

}




// #5
// MAIN function (exposed function to run the whole proccess)
////////////////////////////////////////////////////////////////////////////////


/** main()
 * 
 * nodejs's exported function
 * used as entry-point function (in case of Google Cloud function)
 */

// keep only 1 of next 2 lines
exports.main = () => {       // entry point functioon when google cloud function

    // PROMISE ...
    // fetch all jsons from endpints ...........................................
    Promise.all([

        fetch(products_endpoint, request_settings)
            .then(res => res.json())
            .then((json) => {
                _pr = json;
                console.log('products data loaded from emarket api/endpoint');
            }),

        fetch(frequency_endpoint, request_settings)
            .then(res => res.json())
            .then((json) => {
                _fr = json;
                console.log('frequencies loaded from CDN')
            })

    ])
    .then (() => {  // then...
        // proccess data and extract ketwords

        console.log('All data from endpoints: LOADED!');

        // proccess data
        extract_linked_keywords(_pr.data);

        console.log('keywords extracted; try to save results localy...');

        // PROMISE ...
        // save local files ....................................................
        Promise.all([
            save_local(kwlinks_, temp_kw_file)
            .then(() => { console.log('keywords saved localy');}),

            save_local(prods_, temp_prod_file)
            .then(() => { console.log('products saved localy');})
        ])
        .then(() => {   // then ...

            console.log('Local saves: DONE!');

            // PROMISE ...
            // upload keytword and product jsons into Cloud Storage ............
            Promise.all([
                upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'),
                upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json')
            ])
            .then(() => {   // then ...

                console.log('Everything seems ok; END._')

            });

        });

    });

}