patches for keygan; patches shall be absorbed into the main proccedure (work in progress)

1 files changed, 781 insertions, 0 deletions

diff --git a/javascript/test-linkwords.js b/javascript/test-linkwords.js
new file mode 100644
index 0000000..680be70
--- /dev/null
+++ b/javascript/test-linkwords.js
@@ -0,0 +1,781 @@
+/** keygen
+ * 
+ * this script constucts a linked-wordkeys structure
+ * -----------------------------------------------------------------------------
+ * 
+ * Contents:
+ * #1 Requirements
+ * #2.1 Personalized constants and parametres
+ * #2.2 Purify string functions
+ * #2.3 Preloaded Data
+ * #3 Supporting functions
+ * #4 Output functions
+ * #5 Entry-point function main()
+ */
+
+
+/**  #1
+ * ---
+ * REQUIREMENTS
+ * -----------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////////
+
+ const fs = require('fs');
+ const os = require('os');
+ const fetch = require('node-fetch');
+ const {Storage} = require('@google-cloud/storage');     // Google Cloud Storage
+ 
+ 
+ 
+ 
+ /** #2.1
+  * ---
+  * SETUP PERSONALIZED CONSTANTS AND PARAMETRES
+  * -----------------------------------------------------------------------------
+  *//////////////////////////////////////////////////////////////////////////////
+ 
+ 
+ // get products FROM api/endpoint parametres
+ // --- -- -- - - -
+ const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json';
+ const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch";
+ const request_settings = { method: "Get" };
+ 
+ 
+ // google-storage parametres
+ // --- -- -- - - -
+ const projectId = 'pythia-251711';
+ const keyFilename = '../auth/pythia-251711-047e3d5e6608.json';
+ 
+ 
+ // temp files
+ // need to be created prior to save-to-google-storage proccess
+ // because upload_file() uploats an existing local file to G-Storage
+ // --- -- -- - - -
+ const temp_kw_file = '../results/keywords.json';
+ const temp_prod_file = '../results/products.json';
+ 
+ 
+ // Global Variables
+ // -----------------------------------------------------------------------------
+ 
+ // Global arrays
+ // keyword links (word-links dictionary; array of objects)
+ // -----------------------------------------------------------------------------
+ var kwlinks_ = [];  // keyword-links structure (structure to construct)
+ var prods_ = [];    // products (array to construct)
+ var _fr =[];        // product frequencies array (as read from endpiont) 
+ var _pr =[
+    "ΚΡΙ ΚΡΙ Γιαούρτι Στραγγιστό & Αγαπημένο 2% 1kg",
+    "COCA COLA Zero Αναψυκτικό Χωρίς ζάχαρη 6x330ml",
+//    "COCA COLA Αναψυκτικό 2x1lt",
+//    "ΚΡΙΣ ΚΡΙΣ Τόστιμο Ψωμί Τοστ Σταρένιο 400gr",
+//    "ΚΡΙ ΚΡΙ Heartmade Παγωτό Βανίλια Κακάο & Φράουλα 1kg (2lt) 50% Δώρο",
+//    "ΚΡΙ ΚΡΙ Kids Peppa Επιδό… Παιδικό 2x140gr 1 Δώρο",
+//    "ΒΙΚΟΣ Αναψυκτικό Cola 330ml"
+ ];        // product array (as read from endpiont) 
+ 
+ 
+ // Console Colors
+ // -----------------------------------------------------------------------------
+ const Reset = "\x1b[0m";
+ const FgRed = "\x1b[31m";
+ const FgGreen = "\x1b[32m";
+ 
+ 
+ 
+ 
+ 
+ /** #2.2
+  * ---
+  * PURIFY STRING FUNCTIONS
+  * -----------------------------------------------------------------------------
+  *//////////////////////////////////////////////////////////////////////////////
+ 
+ 
+ 
+ /** keyboardize
+  * -----------------------------------------------------------------------------
+  * translates string to keyboard-latin keys
+  * (the ones that used whan typing each letter of the word)
+  * 
+  * @param str (string): original string (utf8 of latin or greek subgroups)
+  * @return (string): latin/ascii equivalent string
+  */
+ 
+ // cache (keeo in global) any-character to keyboard-latin mapping
+ // --- -- -- - - -
+ var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
+ var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
+ const map = new Map();
+ for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
+ 
+ // "keyboardize" function
+ // --- -- -- - - -
+ function keyboardize(str) {
+     str = str.replace('\'','');
+     var out = '';
+     for (var i=0 ; i< str.length; i++)  out += map.get(str[i]);
+     return out;
+ }
+ 
+ 
+ 
+ 
+ /**  sanitize_GR
+  * -----------------------------------------------------------------------------
+  * replaces greek accended vowels with non accended ones
+  * takes care of sigma on the end of words
+  * 
+  * @param str (string)
+  * @return sanitized string
+  */
+ 
+ // first cache (=create a global array)
+ // of accended to non-accended vowels mapping
+ // --- -- -- - - -
+ accented_vowels = [];   
+ [
+     'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω'   // ,
+     // 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω'
+ ].forEach( pair => {
+     ap = pair.split(' ');
+     accented_vowels.push({
+         a: ap[0],   // accented
+         p: ap[1]    // pure = non accended
+     });
+ })
+ 
+ // the sanitize_GR function
+ // --- -- -- - - -
+ function sanitize_GR(str) {
+ 
+     str = str.toLowerCase();
+ 
+     // replace accended vowels with pure ones
+     accented_vowels.forEach( v => {
+         str = str.replaceAll(v.a, v.p);
+     });
+ 
+     // replace sigma on the end of words
+     str = str + ' ';
+     str = str.replaceAll('σ-', 'ς-');
+     str = str.replaceAll('σ ', 'ς ');
+     
+     // return str.trim();
+     return str;
+ }
+ 
+ 
+ 
+ /** #2.3
+  * ---
+  * PRELOADED DATA
+  * in the future these data may be loaded from a database / api-endpoint
+  * -----------------------------------------------------------------------------
+  *//////////////////////////////////////////////////////////////////////////////
+ 
+ 
+ // synonyms ////////////////////////////////////////////////////////////////////
+ // -----------------------------------------------------------------------------
+ 
+ var synonyms = [];      // groups of synonyms
+ var synonym_kbs = [];   // cache kb-formats for performance
+ var synonyms_Originals = [
+     'μπίρα μπύρα μπίρες μπύρες',
+     'coca-cola cocacola coke',
+     'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ',
+     'Ντομάτα Ντομάτας',
+     'Ελαφρύ Ελαφρά Light',
+     'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά',
+     'τριμμένη τριμμένο',
+     'Τόνος Τόνου',
+     'Κριθαρένια κρίθινα',
+     'Χωρίς-Kαφεϊνη Decaffeine',
+     'Το-Μάννα Μάννα',
+     'Κράνμπερι Κράνμπερις',
+     'Κρήτης Κρητικό',
+     'Πέννες Πένες',
+     'Μακαρόνια Ζυμαρικά Σπαγγέτι Σπαγγετίνι Σπαγγετόνι',
+     'Καρτέλλα Καρτέλα Καρτέλλες'
+ ]
+ synonyms_Originals.forEach( grp => {
+     // remove accends so any replaces can be done flawlessly
+     grp = sanitize_GR(grp);     
+ 
+     // create array of synonyms;
+     // push it to sanitized and keyboardized arrays
+     synonyms.push( grp.split(' ') );
+     synonym_kbs.push( keyboardize(grp).split(' ') );
+ })
+ 
+ 
+ // significant terms ///////////////////////////////////////////////////////////
+ // -----------------------------------------------------------------------------
+ significantExceptios = sanitize_GR('7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ')
+ 
+ 
+ // replaces (correcting descriptions)
+ // = construct unequivocally liked words ///////////////////////////////////////
+ // -----------------------------------------------------------------------------
+ 
+ // NOTE: TODO:
+ // in future implementations multi-word keywords
+ // may use the non-breaking space as conecting character (\u00A0) instead of dush (-)
+ // (or maybe both of them)
+ // --- -- -- - - -
+ replaces = [];
+ replaceSource = [
+     'COCA COLA;COCA-COLA',
+     'COCACOLA;COCA-COLA',
+     'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ',
+     'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ',
+     'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη',
+     'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ',
+     'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ',
+     'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη'
+ ];
+ replaceSource.forEach( it => {
+     st = sanitize_GR(
+             it.toLowerCase()
+         ).split(';');
+     replaces.push({     
+         src: ' '+ st[0] +' ',   // encolse between spaces
+         trg: ' '+ st[1] +' '    // to separate from before/after words
+     });
+ });
+ 
+ // console.log(replaces); process.exit();
+ 
+ 
+ // words that shall not be searched first //////////////////////////////////////
+ // -----------------------------------------------------------------------------
+ var noRootKeywords = [];
+ noRoot = [
+     'χωρίς',
+     'εισαγωγής',
+     'δώρο',
+     'γεύση',
+     'γεύσεις',
+     'φέτες',
+     'Χωρίς-Γλουτένη',
+     'Γλουτένη',
+     'Λακτόζη',
+     'Παστερίωσης',
+     'Ανθρακικό',
+     'Συντηρητικά',
+     'Χωρίς-Ζάχαρη',
+     'Άλεσης',
+     'Χωρίς-Αλάτι',
+     'Χωρίς-Λακτόζη',
+     'Χωρίς-Συντηρητικά',
+     'Χωρίς-Αλκοόλ',
+     'Χωρίς-Kαφεϊνη',
+     'Χωρίς-Γλυκάνισο',
+     'Χωρίς-Ανθρακικό',
+     'Υψηλής-Παστερίωσης',
+     'Ολες-τις-Χρήσεις',
+     'Ολικής-Άλεσης',
+     'Ολικής-Aλέσεως',
+     'Ολικής',
+     'Σοκολάτας',
+     'Υγείας',
+     'Γαϊδούρας',
+     'Δημητριακών',
+     'Γαϊδάρου',
+     'Ρούχων',
+     'Πιάτων',
+     'πλύσεις',
+     'Πλυντηρίου',
+     'Φύλλων',
+     'Γάλακτος',
+     'Χρήσης',
+     'Τύπου',
+     'Ολλανδίας',
+     'Απορριμμάτων',
+     'Medium',
+     'Μαλλιά',
+     'Μαλλιών',
+     'Γενικής',
+     'Plus',
+     'Classic',
+     'Έκπληξη',
+     'Μάνης',
+     'Ελάτου',
+     'Άγριων',
+     'Βοτάνων',
+     'Κατσαρίδες',
+     'Λακωνίας',
+     'ΠΑΡΑΓΓΕΛΙΩΝ',
+     'Λήμνου',
+     'Αργινίνης',
+     'Καλαθάκι',
+     'Σκύλου',
+     'Γάτας',
+     'ΠΓΕ',
+     'ΤΟ',
+     'Μελί',
+     'Σώματος',
+     'Αδυνατίσματος',
+     'Προστασίας',
+     'Ακράτειας',
+     'Προσώπου',
+     'προσθήκη',
+     'Βρεφικής',
+     'Καθαρισμού',
+     'Ρούχων'
+ ];
+ noRoot.forEach( w => { noRootKeywords.push(keyboardize(w)); });
+ 
+ 
+ // list of linked-words ////////////////////////////////////////////////////////
+ // -----------------------------------------------------------------------------
+ linkedWords = [
+     sanitize_GR('Χωρίς-Γλουτένη'),
+     sanitize_GR('Χωρίς-Ζάχαρη'),
+     sanitize_GR('Χωρίς-Αλάτι'),
+     sanitize_GR('Χωρίς-Λακτόζη'),
+     sanitize_GR('Χωρίς-Συντηρητικά'),
+     sanitize_GR('Χωρίς-Αλκοόλ'),
+     sanitize_GR('Χωρίς-Kαφεϊνη'),
+     sanitize_GR('Χωρίς-Γλυκάνισο'),
+     sanitize_GR('Χωρίς-Ανθρακικό'),
+     sanitize_GR('Υψηλής-Παστερίωσης'),
+     sanitize_GR('Ολικής-Άλεσης'),
+     sanitize_GR('Ολικής-Aλέσεως'),
+     sanitize_GR('Χαρτί-Υγείας'),
+     sanitize_GR('ρολό-υγείας'),
+     sanitize_GR('χαρτί-τουαλέτας'),
+     sanitize_GR('Χαρτί-Κουζίνας'),
+     sanitize_GR('Μπάρες-Δημητριακών'),
+     sanitize_GR('Μπαρμπα-Στάθης'),
+     sanitize_GR('COCA-COLA'),
+     sanitize_GR('Aς-Μαγειρέψουμε'),
+     sanitize_GR('ΚΡΙΣ-ΚΡΙΣ'),
+     sanitize_GR('ΚΡΙ-ΚΡΙ'),
+     sanitize_GR('ΕΛ-ΓΚΡΕΚΟ'),
+     sanitize_GR('FREE-STEP'),
+     sanitize_GR('EL-SABOR'),
+     sanitize_GR('LE-PETIT-MARSEILLAIS'),
+     sanitize_GR('DOUWE-EGBERTS'),
+     sanitize_GR('ΕΝ-ΕΛΛΑΔΙ'),
+     sanitize_GR('SPIN-SPAN'),
+     sanitize_GR('CRETA-FARMS'),
+     sanitize_GR('CRETA-FARM'),
+     sanitize_GR('NES-CAFE'),
+     sanitize_GR('Ολες-τις-Χρήσεις'),
+     sanitize_GR('Το-Μάννα'),
+     sanitize_GR('Χωρίς-προσθήκη-ζάχαρης')
+ ];
+ 
+ 
+ // list of words to exclude from keywords //////////////////////////////////////
+ // -----------------------------------------------------------------------------
+ // NOTE: APPLIED in PER-WORD base -> after spliting description to words
+ removeList = []
+ removeOriginals = sanitize_GR(
+     'κατά παρά υπό από μετά προς μας με σε για του της των από στο το στον & r s ft l τ e g h k m n o p s x'
+ )
+ .split(' ')
+ removeOriginals.forEach( w => { removeList.push( keyboardize(w)); });
+ 
+ 
+ 
+ 
+ 
+ 
+ /** #3
+  * ---
+  * SUPPORTING FUNCTIONS
+  * -----------------------------------------------------------------------------
+  *//////////////////////////////////////////////////////////////////////////////
+ 
+ 
+ /** freq
+  * 
+  * @param id (int): product's eys_code
+  * @return frequency of product
+  */
+ function freq_of(id) {
+     _fr.forEach( rec => {
+         if (rec.id == id) return rec.fq
+     });
+ 
+     return 0;
+ }
+ 
+ 
+ /**  proccess data
+  * --- -- -- - - -
+  * 
+  * central proccessing procedure
+  * -----------------------------------------------------------------------------
+  */
+ function extract_linked_keywords(obj) {   
+     // console.log(JSON.stringify(obj, null, 2));
+
+     var i = 0;
+ 
+     obj.forEach( rec => {
+ 
+            i++;
+             //var description = preproccess_text(rec.txt);
+ 
+             var description = clean_text(rec).trim();
+ 
+             var pid = i;
+             var fq = 1;
+ 
+             prods_.push({
+                 id: pid,
+                 w: description
+             })
+ 
+             var keys = [];
+             var words = preproccess_text(description).split(' ');
+ 
+             // filter words; keep only significant
+             words.forEach( w => {
+                 if (removeList.indexOf(keyboardize(w)) == -1)      // if not excluded
+                     if (is_significant(w))                  // and significant
+                         keys.push(w);               // add it to keys
+             });
+             // console.log(pid, description, keys);
+ 
+             keys.forEach( w => {
+                 var wl = synonym_keys(w);
+ 
+                 // if key CAN be a root word (not a no-Root-keyword) update root-node
+                 if (noRootKeywords.indexOf(keyboardize(w)) == -1) {
+ 
+                     root_key(wl, fq);       // update root keyword stats
+ 
+                     // if key is the only in the list of product's keywords
+                     // connect it with a dummy key (to preserve the reference to the product)
+                     if (keys.length == 1)  connect_keys(wl, ['*'], pid, fq);
+ 
+                     // connect w with all the other product's keywords
+                     keys.forEach( w2 => {
+                         if (w2 != w) {
+                             var w2syns = synonym_keys(w2);
+                             connect_keys( wl, w2syns, pid, fq);
+                         }
+                     });
+                 }
+ 
+             });
+
+     });    // main proccessing finished;
+     
+     // post proccess 
+     // -------------------------------------------------------------------------
+ 
+     // remove cached keys from final array
+     kwlinks_.forEach( ro => { 
+         delete ro.kb;
+         ro.c.forEach( ch => { delete ch.kb; });
+     });
+ 
+     // sort root and child nodes by frequency descanding
+     kwlinks_.forEach( it => {
+         it.c = it.c.sort((a, b) => b.f - a.f );
+     });
+     kwlinks_ = kwlinks_.sort((a, b) => b.f - a.f );
+ 
+ }
+ 
+ 
+ // functions for linking words in keywords dictionary
+ ////////////////////////////////////////////////////////////////////////////////
+ 
+ 
+ /** root_key( wl, f )
+  * --- -- -- - - -
+  * set root keyword: wl (if not exist)
+  * update frequency: f
+  * 
+  * @param wl (list) :a list of synonym-words
+  * @param f (int): frequency of product
+  * 
+  * ** comparison is based on the *keyboard* format
+  */
+ function root_key ( wl, f ) {
+     var wkb = keyboardize(wl[0])     // cache kb format
+ 
+     // check if exists in root keys already
+     // NOTE: you only need to check the 1st word of synonyms-list
+     for (i=0; i< kwlinks_.length ; i++) {
+         if (kwlinks_[i].kb == wkb) {
+             kwlinks_[i].f += f;
+             return true;
+         }
+     }
+     // if not exists, append keyword
+     kwlinks_.push({
+         w  : wl,
+         kb : wkb,
+         f  : f,
+         c  : []
+     });
+     return true;
+ }
+ 
+ /** connect_keys
+  * --- -- -- - - -
+  * @param (list) : connect keyword a (synonyms list)
+  * @param (list) : with keyword b (synonyms list)
+  * @param (int)  : id (eys code) of product where keywords a and b are linked
+  * @param (int)  : frequency of product (now many times has been purchased)
+  * @returns true
+  */
+ function connect_keys( a, b, id, f ) {
+     var kbA = keyboardize(a[0]);
+     var kbB = keyboardize(b[0]);
+     var bExists = false;
+ 
+     if (kbA == kbB)     return false;       // exclude just-in-case
+     
+     for (i=0; i< kwlinks_.length ; i++) {
+         if (kwlinks_[i].kb == kbA) {        // found: a;
+             // update connection to: b
+             bExists = false;
+             for (j=0 ; j < kwlinks_[i].c.length ; j++) {
+                 if (kwlinks_[i].c[j].kb == kbB) {
+                     bExists = true;
+                     // update the connection's data
+                     kwlinks_[i].c[j].f += f;
+                     kwlinks_[i].c[j].p.push(id)
+                     break;
+                 }
+             }
+             // if connection not exist, init a new one
+             if (bExists == false) {
+                 // create connection with: b
+                 kwlinks_[i].c.push({
+                     w  : b,
+                     kb : kbB,
+                     f  : f,
+                     p  : [ i ]
+                 });
+             }
+             return true;
+         }
+     }
+ }
+ 
+ 
+ // other supplementary functions
+ ////////////////////////////////////////////////////////////////////////////////
+ 
+ 
+ 
+ 
+ /** clean text
+  * --- -- -- - - -
+  * removes non keyword characters [\+, \., \'] and internal multiple-spaces
+  * @param txt (string): product description
+  */
+ function clean_text(txt) {
+     return txt.replace('+',' ').replace('.',' ')
+         .replace('   ',' ').replace('  ',' ');
+ }
+ 
+ 
+ /** is significant
+  * --- -- -- - - -
+  * check if term is significant (and needs to be indexed)
+  * (if not, the term will be excluded from keywords dicionary)
+  * 
+  * @param str
+  * @return (boolean)
+  */
+ function is_significant(str) {
+     if (str == '')     return false;
+     if (significantExceptios.indexOf(str) !== -1)   return true;
+     return !(/\d/.test(str));
+ }
+ 
+ 
+ /** synonym_keys
+  * --- -- -- - - -
+  * check if word: w
+  * has synonyms; return list of synonyms
+  * 
+  * @param w (string)
+  * @return list of synonyms
+  * 
+  * NOTE: in no synonyms exist, retutn the word as a list
+  */
+ function synonym_keys(w) {
+     w_kb = keyboardize(w);
+     for (i=0 ; i < synonym_kbs.length ; i++) {
+         if (synonym_kbs.indexOf(w_kb) !== -1)
+             return synonyms[i];
+     }
+     return [ w ];
+ }
+ 
+ 
+ 
+ /** mark_explicit_links
+  * 
+  * mark linked words
+  * shall be handled as one-(key)word
+  * 
+  * @param str 
+  * @return 
+  */
+ 
+ // edit common mistakes
+ // with suggested replaces
+ function mark_explicit_links(str) {
+     str = ' '+ str +' ';
+     replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); });
+     return str.replaceAll('  ', ' ').trim();
+ }
+ 
+ 
+ // preproccess description
+ // ---
+ function preproccess_text(description) {
+ 
+     str = clean_text(description);  // remove non-keyword characters
+ 
+     str = mark_explicit_links(sanitize_GR( str.toLowerCase() ));    // mark explicit links
+
+     console.log('>', str);
+     
+     // replaces.forEach( lw => {
+     //     reg = new RegExp( lw.src, "gi");    // prepare regex for case insensitive (i) replace all (g)
+     //     str = str.toLowerCase().replace(reg, lw.trg);
+     // })
+     // depricated: str = mark_linked_words(str);
+     return str;
+ }
+ 
+ 
+ 
+ /* ---
+  DEPRICATED:
+     // mark linked words (connect them with a dash)
+     // return new text after "all-links" are marked
+     // ---
+     function mark_linked_words(txt) {
+         linkedWords.forEach( lw => {
+ 
+             txt = mark_link( lw, txt );
+         });
+         return txt
+     }
+ */
+ 
+ 
+ 
+ /* ---
+  DEPRICATED:
+     // mark a link (lws) to a text (source)
+     // conecting them with a dash/minus character
+     // ---
+     function mark_link(lws, source) {
+         var src_kb = keyboardize(source.replace(' ', '-'));
+         var lws_kb = keyboardize(lws.replace(' ', '-'));
+         var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase())
+         if (_left !== -1 ) {
+             return source.slice(0, _left) + lws + source.slice(_left + lws.length);
+         }
+         else return source;
+     }
+ */
+ 
+ 
+ 
+ 
+ /** #4
+  * ---
+  * OUTPUT FUNCTIONS
+  * -----------------------------------------------------------------------------
+  *//////////////////////////////////////////////////////////////////////////////
+ 
+ 
+ 
+ // Save Local
+ // -----------------------------------------------------------------------------
+ async function save_local(jsonArray, filePath) {
+     let fileStr = JSON.stringify(jsonArray);    // convert json to string
+ 
+     // write string to file
+     fs.writeFileSync(filePath, fileStr, 'utf8', (err) => {
+         if (err) {
+             return err;
+         } else {
+             return true;
+         }
+     });
+ }
+ 
+ 
+ 
+ // Save to Google Cloud Storage
+ // -----------------------------------------------------------------------------
+ async function upload_file( bucketName, srcFilePath, trgFilePath ) {
+     // Creates a client
+     const storage = new Storage({projectId, keyFilename});
+ 
+     try {
+         await storage.bucket(bucketName).upload(srcFilePath, {
+             destination: trgFilePath, 
+             gzip: true,     // serve compressed
+             metadata: {     // cache for 8 hours
+                 cacheControl: 'public, max-age=60'      // production set: 28800
+             }
+         });
+         console.log(`${srcFilePath} uploaded to ${bucketName}`);
+     }
+ 
+     catch(err) {
+         console.error('ERROR:', err);
+     }
+ 
+ }
+ 
+ 
+ 
+ 
+ 
+ 
+ /** #5
+  * ---
+  * MAIN function (exposed function to run the whole proccess)
+  * -----------------------------------------------------------------------------
+  *//////////////////////////////////////////////////////////////////////////////
+ 
+ 
+ 
+ /** main()
+  * 
+  * nodejs's exported function
+  * used as entry-point function (in case of Google Cloud function)
+  */
+ 
+ // keep only 1 of next 2 lines
+ // exports.main = () => {       // entry point functioon when google cloud function
+ var main = () => {              // concise function when direct script
+
+    // proccess data and extract ketwords
+
+    // proccess data
+    extract_linked_keywords(_pr);
+
+
+    // console.log(kwlinks_)
+
+    // console.log(prods_)
+ 
+ }
+ 
+ 
+ // NOTE:
+ // if running the script directry you need to call the main function
+ // if running through google cloud-functions you do NOT need to call main()
+ // (you need define main() as the entry-point function instead)
+ main();
\ No newline at end of file