/** keygen
 * 
 * this script constucts a linked-wordkeys structure
 * -----------------------------------------------------------------------------
 * 
 * Contents:
 * #1 Requirements
 * #2.1 Personalized constants and parametres
 * #2.2 Purify string functions
 * #2.3 Preloaded Data
 * #3 Supporting functions
 * #4 Output functions
 * #5 Entry-point function main()
 */


/**  #1
 * ---
 * REQUIREMENTS
 * -----------------------------------------------------------------------------
 *//////////////////////////////////////////////////////////////////////////////

 const fs = require('fs');
 const os = require('os');
 const fetch = require('node-fetch');
 const {Storage} = require('@google-cloud/storage');     // Google Cloud Storage
 
 
 
 
 /** #2.1
  * ---
  * SETUP PERSONALIZED CONSTANTS AND PARAMETRES
  * -----------------------------------------------------------------------------
  *//////////////////////////////////////////////////////////////////////////////
 
 
 // get products FROM api/endpoint parametres
 // --- -- -- - - -
 const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json';
 const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch";
 const request_settings = { method: "Get" };
 
 
 // google-storage parametres
 // --- -- -- - - -
 const projectId = 'pythia-251711';
 const keyFilename = '../auth/pythia-251711-047e3d5e6608.json';
 
 
 // temp files
 // need to be created prior to save-to-google-storage proccess
 // because upload_file() uploats an existing local file to G-Storage
 // --- -- -- - - -
 const temp_kw_file = '../results/keywords.json';
 const temp_prod_file = '../results/products.json';
 
 
 // Global Variables
 // -----------------------------------------------------------------------------
 
 // Global arrays
 // keyword links (word-links dictionary; array of objects)
 // -----------------------------------------------------------------------------
 var kwlinks_ = [];  // keyword-links structure (structure to construct)
 var prods_ = [];    // products (array to construct)
 var _fr =[];        // product frequencies array (as read from endpiont) 
 var _pr =[
    "ΚΡΙ ΚΡΙ Γιαούρτι Στραγγιστό & Αγαπημένο 2% 1kg",
    "COCA COLA Zero Αναψυκτικό Χωρίς ζάχαρη 6x330ml",
//    "COCA COLA Αναψυκτικό 2x1lt",
//    "ΚΡΙΣ ΚΡΙΣ Τόστιμο Ψωμί Τοστ Σταρένιο 400gr",
//    "ΚΡΙ ΚΡΙ Heartmade Παγωτό Βανίλια Κακάο & Φράουλα 1kg (2lt) 50% Δώρο",
//    "ΚΡΙ ΚΡΙ Kids Peppa Επιδό… Παιδικό 2x140gr 1 Δώρο",
//    "ΒΙΚΟΣ Αναψυκτικό Cola 330ml"
 ];        // product array (as read from endpiont) 
 
 
 // Console Colors
 // -----------------------------------------------------------------------------
 const Reset = "\x1b[0m";
 const FgRed = "\x1b[31m";
 const FgGreen = "\x1b[32m";
 
 
 
 
 
 /** #2.2
  * ---
  * PURIFY STRING FUNCTIONS
  * -----------------------------------------------------------------------------
  *//////////////////////////////////////////////////////////////////////////////
 
 
 
 /** keyboardize
  * -----------------------------------------------------------------------------
  * translates string to keyboard-latin keys
  * (the ones that used whan typing each letter of the word)
  * 
  * @param str (string): original string (utf8 of latin or greek subgroups)
  * @return (string): latin/ascii equivalent string
  */
 
 // cache (keeo in global) any-character to keyboard-latin mapping
 // --- -- -- - - -
 var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
 var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
 const map = new Map();
 for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
 
 // "keyboardize" function
 // --- -- -- - - -
 function keyboardize(str) {
     str = str.replace('\'','');
     var out = '';
     for (var i=0 ; i< str.length; i++)  out += map.get(str[i]);
     return out;
 }
 
 
 
 
 /**  sanitize_GR
  * -----------------------------------------------------------------------------
  * replaces greek accended vowels with non accended ones
  * takes care of sigma on the end of words
  * 
  * @param str (string)
  * @return sanitized string
  */
 
 // first cache (=create a global array)
 // of accended to non-accended vowels mapping
 // --- -- -- - - -
 accented_vowels = [];   
 [
     'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω'   // ,
     // 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω'
 ].forEach( pair => {
     ap = pair.split(' ');
     accented_vowels.push({
         a: ap[0],   // accented
         p: ap[1]    // pure = non accended
     });
 })
 
 // the sanitize_GR function
 // --- -- -- - - -
 function sanitize_GR(str) {
 
     str = str.toLowerCase();
 
     // replace accended vowels with pure ones
     accented_vowels.forEach( v => {
         str = str.replaceAll(v.a, v.p);
     });
 
     // replace sigma on the end of words
     str = str + ' ';
     str = str.replaceAll('σ-', 'ς-');
     str = str.replaceAll('σ ', 'ς ');
     
     // return str.trim();
     return str;
 }
 
 
 
 /** #2.3
  * ---
  * PRELOADED DATA
  * in the future these data may be loaded from a database / api-endpoint
  * -----------------------------------------------------------------------------
  *//////////////////////////////////////////////////////////////////////////////
 
 
 // synonyms ////////////////////////////////////////////////////////////////////
 // -----------------------------------------------------------------------------
 
 var synonyms = [];      // groups of synonyms
 var synonym_kbs = [];   // cache kb-formats for performance
 var synonyms_Originals = [
     'μπίρα μπύρα μπίρες μπύρες',
     'coca-cola cocacola coke',
     'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ',
     'Ντομάτα Ντομάτας',
     'Ελαφρύ Ελαφρά Light',
     'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά',
     'τριμμένη τριμμένο',
     'Τόνος Τόνου',
     'Κριθαρένια κρίθινα',
     'Χωρίς-Kαφεϊνη Decaffeine',
     'Το-Μάννα Μάννα',
     'Κράνμπερι Κράνμπερις',
     'Κρήτης Κρητικό',
     'Πέννες Πένες',
     'Μακαρόνια Ζυμαρικά Σπαγγέτι Σπαγγετίνι Σπαγγετόνι',
     'Καρτέλλα Καρτέλα Καρτέλλες'
 ]
 synonyms_Originals.forEach( grp => {
     // remove accends so any replaces can be done flawlessly
     grp = sanitize_GR(grp);     
 
     // create array of synonyms;
     // push it to sanitized and keyboardized arrays
     synonyms.push( grp.split(' ') );
     synonym_kbs.push( keyboardize(grp).split(' ') );
 })
 
 
 // significant terms ///////////////////////////////////////////////////////////
 // -----------------------------------------------------------------------------
 significantExceptios = sanitize_GR('7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ')
 
 
 // replaces (correcting descriptions)
 // = construct unequivocally liked words ///////////////////////////////////////
 // -----------------------------------------------------------------------------
 
 // NOTE: TODO:
 // in future implementations multi-word keywords
 // may use the non-breaking space as conecting character (\u00A0) instead of dush (-)
 // (or maybe both of them)
 // --- -- -- - - -
 replaces = [];
 replaceSource = [
     'COCA COLA;COCA-COLA',
     'COCACOLA;COCA-COLA',
     'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ',
     'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ',
     'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη',
     'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ',
     'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ',
     'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη'
 ];
 replaceSource.forEach( it => {
     st = sanitize_GR(
             it.toLowerCase()
         ).split(';');
     replaces.push({     
         src: ' '+ st[0] +' ',   // encolse between spaces
         trg: ' '+ st[1] +' '    // to separate from before/after words
     });
 });
 
 // console.log(replaces); process.exit();
 
 
 // words that shall not be searched first //////////////////////////////////////
 // -----------------------------------------------------------------------------
 var noRootKeywords = [];
 noRoot = [
     'χωρίς',
     'εισαγωγής',
     'δώρο',
     'γεύση',
     'γεύσεις',
     'φέτες',
     'Χωρίς-Γλουτένη',
     'Γλουτένη',
     'Λακτόζη',
     'Παστερίωσης',
     'Ανθρακικό',
     'Συντηρητικά',
     'Χωρίς-Ζάχαρη',
     'Άλεσης',
     'Χωρίς-Αλάτι',
     'Χωρίς-Λακτόζη',
     'Χωρίς-Συντηρητικά',
     'Χωρίς-Αλκοόλ',
     'Χωρίς-Kαφεϊνη',
     'Χωρίς-Γλυκάνισο',
     'Χωρίς-Ανθρακικό',
     'Υψηλής-Παστερίωσης',
     'Ολες-τις-Χρήσεις',
     'Ολικής-Άλεσης',
     'Ολικής-Aλέσεως',
     'Ολικής',
     'Σοκολάτας',
     'Υγείας',
     'Γαϊδούρας',
     'Δημητριακών',
     'Γαϊδάρου',
     'Ρούχων',
     'Πιάτων',
     'πλύσεις',
     'Πλυντηρίου',
     'Φύλλων',
     'Γάλακτος',
     'Χρήσης',
     'Τύπου',
     'Ολλανδίας',
     'Απορριμμάτων',
     'Medium',
     'Μαλλιά',
     'Μαλλιών',
     'Γενικής',
     'Plus',
     'Classic',
     'Έκπληξη',
     'Μάνης',
     'Ελάτου',
     'Άγριων',
     'Βοτάνων',
     'Κατσαρίδες',
     'Λακωνίας',
     'ΠΑΡΑΓΓΕΛΙΩΝ',
     'Λήμνου',
     'Αργινίνης',
     'Καλαθάκι',
     'Σκύλου',
     'Γάτας',
     'ΠΓΕ',
     'ΤΟ',
     'Μελί',
     'Σώματος',
     'Αδυνατίσματος',
     'Προστασίας',
     'Ακράτειας',
     'Προσώπου',
     'προσθήκη',
     'Βρεφικής',
     'Καθαρισμού',
     'Ρούχων'
 ];
 noRoot.forEach( w => { noRootKeywords.push(keyboardize(w)); });
 
 
 // list of linked-words ////////////////////////////////////////////////////////
 // -----------------------------------------------------------------------------
 linkedWords = [
     sanitize_GR('Χωρίς-Γλουτένη'),
     sanitize_GR('Χωρίς-Ζάχαρη'),
     sanitize_GR('Χωρίς-Αλάτι'),
     sanitize_GR('Χωρίς-Λακτόζη'),
     sanitize_GR('Χωρίς-Συντηρητικά'),
     sanitize_GR('Χωρίς-Αλκοόλ'),
     sanitize_GR('Χωρίς-Kαφεϊνη'),
     sanitize_GR('Χωρίς-Γλυκάνισο'),
     sanitize_GR('Χωρίς-Ανθρακικό'),
     sanitize_GR('Υψηλής-Παστερίωσης'),
     sanitize_GR('Ολικής-Άλεσης'),
     sanitize_GR('Ολικής-Aλέσεως'),
     sanitize_GR('Χαρτί-Υγείας'),
     sanitize_GR('ρολό-υγείας'),
     sanitize_GR('χαρτί-τουαλέτας'),
     sanitize_GR('Χαρτί-Κουζίνας'),
     sanitize_GR('Μπάρες-Δημητριακών'),
     sanitize_GR('Μπαρμπα-Στάθης'),
     sanitize_GR('COCA-COLA'),
     sanitize_GR('Aς-Μαγειρέψουμε'),
     sanitize_GR('ΚΡΙΣ-ΚΡΙΣ'),
     sanitize_GR('ΚΡΙ-ΚΡΙ'),
     sanitize_GR('ΕΛ-ΓΚΡΕΚΟ'),
     sanitize_GR('FREE-STEP'),
     sanitize_GR('EL-SABOR'),
     sanitize_GR('LE-PETIT-MARSEILLAIS'),
     sanitize_GR('DOUWE-EGBERTS'),
     sanitize_GR('ΕΝ-ΕΛΛΑΔΙ'),
     sanitize_GR('SPIN-SPAN'),
     sanitize_GR('CRETA-FARMS'),
     sanitize_GR('CRETA-FARM'),
     sanitize_GR('NES-CAFE'),
     sanitize_GR('Ολες-τις-Χρήσεις'),
     sanitize_GR('Το-Μάννα'),
     sanitize_GR('Χωρίς-προσθήκη-ζάχαρης')
 ];
 
 
 // list of words to exclude from keywords //////////////////////////////////////
 // -----------------------------------------------------------------------------
 // NOTE: APPLIED in PER-WORD base -> after spliting description to words
 removeList = []
 removeOriginals = sanitize_GR(
     'κατά παρά υπό από μετά προς μας με σε για του της των από στο το στον & r s ft l τ e g h k m n o p s x'
 )
 .split(' ')
 removeOriginals.forEach( w => { removeList.push( keyboardize(w)); });
 
 
 
 
 
 
 /** #3
  * ---
  * SUPPORTING FUNCTIONS
  * -----------------------------------------------------------------------------
  *//////////////////////////////////////////////////////////////////////////////
 
 
 /** freq
  * 
  * @param id (int): product's eys_code
  * @return frequency of product
  */
 function freq_of(id) {
     _fr.forEach( rec => {
         if (rec.id == id) return rec.fq
     });
 
     return 0;
 }
 
 
 /**  proccess data
  * --- -- -- - - -
  * 
  * central proccessing procedure
  * -----------------------------------------------------------------------------
  */
 function extract_linked_keywords(obj) {   
     // console.log(JSON.stringify(obj, null, 2));

     var i = 0;
 
     obj.forEach( rec => {
 
            i++;
             //var description = preproccess_text(rec.txt);
 
             var description = clean_text(rec).trim();
 
             var pid = i;
             var fq = 1;
 
             prods_.push({
                 id: pid,
                 w: description
             })
 
             var keys = [];
             var words = preproccess_text(description).split(' ');
 
             // filter words; keep only significant
             words.forEach( w => {
                 if (removeList.indexOf(keyboardize(w)) == -1)      // if not excluded
                     if (is_significant(w))                  // and significant
                         keys.push(w);               // add it to keys
             });
             // console.log(pid, description, keys);
 
             keys.forEach( w => {
                 var wl = synonym_keys(w);
 
                 // if key CAN be a root word (not a no-Root-keyword) update root-node
                 if (noRootKeywords.indexOf(keyboardize(w)) == -1) {
 
                     root_key(wl, fq);       // update root keyword stats
 
                     // if key is the only in the list of product's keywords
                     // connect it with a dummy key (to preserve the reference to the product)
                     if (keys.length == 1)  connect_keys(wl, ['*'], pid, fq);
 
                     // connect w with all the other product's keywords
                     keys.forEach( w2 => {
                         if (w2 != w) {
                             var w2syns = synonym_keys(w2);
                             connect_keys( wl, w2syns, pid, fq);
                         }
                     });
                 }
 
             });

     });    // main proccessing finished;
     
     // post proccess 
     // -------------------------------------------------------------------------
 
     // remove cached keys from final array
     kwlinks_.forEach( ro => { 
         delete ro.kb;
         ro.c.forEach( ch => { delete ch.kb; });
     });
 
     // sort root and child nodes by frequency descanding
     kwlinks_.forEach( it => {
         it.c = it.c.sort((a, b) => b.f - a.f );
     });
     kwlinks_ = kwlinks_.sort((a, b) => b.f - a.f );
 
 }
 
 
 // functions for linking words in keywords dictionary
 ////////////////////////////////////////////////////////////////////////////////
 
 
 /** root_key( wl, f )
  * --- -- -- - - -
  * set root keyword: wl (if not exist)
  * update frequency: f
  * 
  * @param wl (list) :a list of synonym-words
  * @param f (int): frequency of product
  * 
  * ** comparison is based on the *keyboard* format
  */
 function root_key ( wl, f ) {
     var wkb = keyboardize(wl[0])     // cache kb format
 
     // check if exists in root keys already
     // NOTE: you only need to check the 1st word of synonyms-list
     for (i=0; i< kwlinks_.length ; i++) {
         if (kwlinks_[i].kb == wkb) {
             kwlinks_[i].f += f;
             return true;
         }
     }
     // if not exists, append keyword
     kwlinks_.push({
         w  : wl,
         kb : wkb,
         f  : f,
         c  : []
     });
     return true;
 }
 
 /** connect_keys
  * --- -- -- - - -
  * @param (list) : connect keyword a (synonyms list)
  * @param (list) : with keyword b (synonyms list)
  * @param (int)  : id (eys code) of product where keywords a and b are linked
  * @param (int)  : frequency of product (now many times has been purchased)
  * @returns true
  */
 function connect_keys( a, b, id, f ) {
     var kbA = keyboardize(a[0]);
     var kbB = keyboardize(b[0]);
     var bExists = false;
 
     if (kbA == kbB)     return false;       // exclude just-in-case
     
     for (i=0; i< kwlinks_.length ; i++) {
         if (kwlinks_[i].kb == kbA) {        // found: a;
             // update connection to: b
             bExists = false;
             for (j=0 ; j < kwlinks_[i].c.length ; j++) {
                 if (kwlinks_[i].c[j].kb == kbB) {
                     bExists = true;
                     // update the connection's data
                     kwlinks_[i].c[j].f += f;
                     kwlinks_[i].c[j].p.push(id)
                     break;
                 }
             }
             // if connection not exist, init a new one
             if (bExists == false) {
                 // create connection with: b
                 kwlinks_[i].c.push({
                     w  : b,
                     kb : kbB,
                     f  : f,
                     p  : [ i ]
                 });
             }
             return true;
         }
     }
 }
 
 
 // other supplementary functions
 ////////////////////////////////////////////////////////////////////////////////
 
 
 
 
 /** clean text
  * --- -- -- - - -
  * removes non keyword characters [\+, \., \'] and internal multiple-spaces
  * @param txt (string): product description
  */
 function clean_text(txt) {
     return txt.replace('+',' ').replace('.',' ')
         .replace('   ',' ').replace('  ',' ');
 }
 
 
 /** is significant
  * --- -- -- - - -
  * check if term is significant (and needs to be indexed)
  * (if not, the term will be excluded from keywords dicionary)
  * 
  * @param str
  * @return (boolean)
  */
 function is_significant(str) {
     if (str == '')     return false;
     if (significantExceptios.indexOf(str) !== -1)   return true;
     return !(/\d/.test(str));
 }
 
 
 /** synonym_keys
  * --- -- -- - - -
  * check if word: w
  * has synonyms; return list of synonyms
  * 
  * @param w (string)
  * @return list of synonyms
  * 
  * NOTE: in no synonyms exist, retutn the word as a list
  */
 function synonym_keys(w) {
     w_kb = keyboardize(w);
     for (i=0 ; i < synonym_kbs.length ; i++) {
         if (synonym_kbs.indexOf(w_kb) !== -1)
             return synonyms[i];
     }
     return [ w ];
 }
 
 
 
 /** mark_explicit_links
  * 
  * mark linked words
  * shall be handled as one-(key)word
  * 
  * @param str 
  * @return 
  */
 
 // edit common mistakes
 // with suggested replaces
 function mark_explicit_links(str) {
     str = ' '+ str +' ';
     replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); });
     return str.replaceAll('  ', ' ').trim();
 }
 
 
 // preproccess description
 // ---
 function preproccess_text(description) {
 
     str = clean_text(description);  // remove non-keyword characters
 
     str = mark_explicit_links(sanitize_GR( str.toLowerCase() ));    // mark explicit links

     console.log('>', str);
     
     // replaces.forEach( lw => {
     //     reg = new RegExp( lw.src, "gi");    // prepare regex for case insensitive (i) replace all (g)
     //     str = str.toLowerCase().replace(reg, lw.trg);
     // })
     // depricated: str = mark_linked_words(str);
     return str;
 }
 
 
 
 /* ---
  DEPRICATED:
     // mark linked words (connect them with a dash)
     // return new text after "all-links" are marked
     // ---
     function mark_linked_words(txt) {
         linkedWords.forEach( lw => {
 
             txt = mark_link( lw, txt );
         });
         return txt
     }
 */
 
 
 
 /* ---
  DEPRICATED:
     // mark a link (lws) to a text (source)
     // conecting them with a dash/minus character
     // ---
     function mark_link(lws, source) {
         var src_kb = keyboardize(source.replace(' ', '-'));
         var lws_kb = keyboardize(lws.replace(' ', '-'));
         var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase())
         if (_left !== -1 ) {
             return source.slice(0, _left) + lws + source.slice(_left + lws.length);
         }
         else return source;
     }
 */
 
 
 
 
 /** #4
  * ---
  * OUTPUT FUNCTIONS
  * -----------------------------------------------------------------------------
  *//////////////////////////////////////////////////////////////////////////////
 
 
 
 // Save Local
 // -----------------------------------------------------------------------------
 async function save_local(jsonArray, filePath) {
     let fileStr = JSON.stringify(jsonArray);    // convert json to string
 
     // write string to file
     fs.writeFileSync(filePath, fileStr, 'utf8', (err) => {
         if (err) {
             return err;
         } else {
             return true;
         }
     });
 }
 
 
 
 // Save to Google Cloud Storage
 // -----------------------------------------------------------------------------
 async function upload_file( bucketName, srcFilePath, trgFilePath ) {
     // Creates a client
     const storage = new Storage({projectId, keyFilename});
 
     try {
         await storage.bucket(bucketName).upload(srcFilePath, {
             destination: trgFilePath, 
             gzip: true,     // serve compressed
             metadata: {     // cache for 8 hours
                 cacheControl: 'public, max-age=60'      // production set: 28800
             }
         });
         console.log(`${srcFilePath} uploaded to ${bucketName}`);
     }
 
     catch(err) {
         console.error('ERROR:', err);
     }
 
 }
 
 
 
 
 
 
 /** #5
  * ---
  * MAIN function (exposed function to run the whole proccess)
  * -----------------------------------------------------------------------------
  *//////////////////////////////////////////////////////////////////////////////
 
 
 
 /** main()
  * 
  * nodejs's exported function
  * used as entry-point function (in case of Google Cloud function)
  */
 
 // keep only 1 of next 2 lines
 // exports.main = () => {       // entry point functioon when google cloud function
 var main = () => {              // concise function when direct script

    // proccess data and extract ketwords

    // proccess data
    extract_linked_keywords(_pr);


    // console.log(kwlinks_)

    // console.log(prods_)
 
 }
 
 
 // NOTE:
 // if running the script directry you need to call the main function
 // if running through google cloud-functions you do NOT need to call main()
 // (you need define main() as the entry-point function instead)
 main();