linkeysearch: javascript scripts keygen+freq can deploy as Google Cloud Funcctions

5 files changed, 764 insertions, 11 deletions

diff --git a/javascript/async.js b/javascript/async.js
new file mode 100644
index 0000000..561270c
--- /dev/null
+++ b/javascript/async.js
@@ -0,0 +1,118 @@
+/** example file
+ * 
+ * this script tests async chain handling;
+ * runnig async functions sequentialy
+ * or running in parallel
+ * 
+ */
+
+
+/**
+ * functions a and b are asyncohronus
+ */
+
+async function a(msg) {
+    sum = 0;
+    for (i=0 ; i< parseInt(Math.floor(Math.random() * 1000000000)); i++) {
+        sum += i;
+    }
+    console.log(msg, sum);
+    return sum
+}
+
+async function b(upto) {
+    sum = 0;
+    for (i=0 ; i< upto; i++) {
+        sum += i;
+    }
+    console.log(upto, sum);
+    return new Promise((resolve, reject) => { resolve(sum); });
+}
+
+
+/** test
+ * --- -- -- - - -
+ * is an asynchronous function
+ * that calls multiple times in parallel
+ * the a and b functions
+ */
+async function test() {
+    labels = ['one', 'two', 'three', 'four', 'five'];
+    labels.forEach( lab => { 
+        a(lab)
+    });
+    console.log('test1');
+
+    limits = [1000000, 50000, 30000000, 500, 8];
+    limits.forEach( lim => {
+        b(lim).then(console.log(lim,'is done!'));
+    });
+    console.log('test2');
+}
+
+
+test()  // run test (many async functions in parallel)
+.then(() => {   // after test is ended
+    console.log('test parts(1+2) ended');
+
+    // run async functions in sequentialy
+    a('more')
+    .then(() => {
+        b(100000)
+        .then(() => {
+            a('last')
+            .then(() => {
+                b(20)
+                .then(() =>{
+                    console.log('exiting...');
+                    console.log('\n\n--------------------\nNext please...\n--------------------\n\n');
+                })
+            });
+        });
+    });
+
+});
+
+
+
+/**
+ * xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+ * -----------------------------------------------------------------------------
+ * 
+ * * * * * * * * * * * *   S E C O N D   E X A M P L E   * * * * * * * * * * * *
+ * 
+ * -----------------------------------------------------------------------------
+ * xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+ */
+
+
+/** sleep
+ * --- -- -- - - -
+ * an async function with predictable waiting time
+ */
+async function sleep(time = 1) {
+	const sleepMilliseconds = time * 1000;
+        return new Promise((resolve, reject) => {
+            setTimeout(() => {
+                console.log(`Slept for: ${sleepMilliseconds}ms`);
+                resolve(sleepMilliseconds);
+            }, sleepMilliseconds);
+        });
+}
+
+// run all in parallel
+Promise.all([   // after each call is completed
+                // '.then(...)' make a message note
+    sleep(3).then(()=>{ console.log('instance', 3, 'completed')}),
+    sleep(2).then(()=>{ console.log('instance', 2, 'completed')}),
+    sleep(1).then(()=>{ console.log('instance', 1, 'completed')})
+
+])
+.then( () => {  // then 
+    console.log('Now, there should be all done!', 'DONE!')
+});
+
+
+// message to test that async functions keep running on the background
+console.log('--> is this the end?', 'NOPE! <--');
+
diff --git a/javascript/freq.js b/javascript/freq.js
index f62017c..86bba22 100644
--- a/javascript/freq.js
+++ b/javascript/freq.js
@@ -108,7 +108,7 @@ function save_local(jsonArray, fileName) {
     let fileStr = JSON.stringify(jsonArray);    // convert json to string
     
     // write string to file
-    fs.writeFileSync(local_dir +'/ '+ fileName, fileStr, 'utf8', (err) => {
+    fs.writeFileSync(fileName, fileStr, 'utf8', (err) => {
         if (err) {
             console.log("An error occured while writing keywords.json");
             return console.log(err);
diff --git a/javascript/freq_GCF.js b/javascript/freq_GCF.js
index 79810e6..2da3eee 100644
--- a/javascript/freq_GCF.js
+++ b/javascript/freq_GCF.js
@@ -1,6 +1,5 @@
 /** freq.js
  * 
- * Google Cloud Functions impementation of `linkeysearch:javascript/freq`
  * this script extracts products' order-frequency 
  * -----------------------------------------------------------------------------
  * 
diff --git a/javascript/keygen.js b/javascript/keygen.js
index 39174b1..59407a9 100644
--- a/javascript/keygen.js
+++ b/javascript/keygen.js
@@ -671,15 +671,22 @@ var main = () => {              // concise function when direct script
             upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json');
             upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json');
 
-            /// // check
-            /// // echo 100 most frequent
-            /// let i = 0;
-            /// kwlinks_.forEach(rec => {
-            ///     if (i<100) {
-            ///         console.log(i, JSON.stringify(rec.w))
-            ///     }
-            ///     i++;
-            /// })
+            // check
+            // echo 100 most frequent
+            let i = 0;
+            kwlinks_.forEach(rec => {
+                if (i<100) {
+                    console.log(i, JSON.stringify(rec.w))
+                }
+                i++;
+            });
+
+            // echo memory stats
+            const used = process.memoryUsage();
+            for (let key in used) {
+                console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
+            }
+
         });
 
     });
diff --git a/javascript/keygen_GCF.js b/javascript/keygen_GCF.js
new file mode 100644
index 0000000..8b13dd9
--- /dev/null
+++ b/javascript/keygen_GCF.js
@@ -0,0 +1,629 @@
+/** keygen
+ * 
+ * this script constucts a linked-wordkeys structure
+ * -----------------------------------------------------------------------------
+ * 
+ * Contents:
+ * #1 Requirements
+ * #2.1 Personalized constants and parametres
+ * #2.2 Preloaded Data
+ * #3 Supporting functions
+ * #4 Output functions
+ * #5 Entry-point function main()
+ */
+
+// #1
+// REQUIREMENTS
+////////////////////////////////////////////////////////////////////////////////
+
+const fs = require('fs');
+const os = require('os');
+const fetch = require('node-fetch');
+const { ECDH } = require('crypto');
+const {Storage} = require('@google-cloud/storage');     // Google Cloud Storage
+
+
+
+// #2.1
+// SETUP PERSONALIZED CONSTANTS AND PARAMETRES
+////////////////////////////////////////////////////////////////////////////////
+
+
+// get products FROM api/endpoint parametres
+const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json';
+const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch";
+const request_settings = { method: "Get" };
+
+
+
+// Global Variables
+// -----------------------------------------------------------------------------
+
+// keyword links (word-links dictionary; array of objects)
+// -----------------------------------------------------------------------------
+var kwlinks_ = [];      ////////////////////////////// MAIN OUTPUT OF THE SCRIPT
+var prods_ = [];
+var fr_ =[];
+
+
+// #2.2
+// PRELOADED DATA
+////////////////////////////////////////////////////////////////////////////////
+
+// any-character to keyboard-latin mapping
+// -----------------------------------------------------------------------------
+var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
+var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
+const map = new Map();
+for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
+
+
+// synonyms
+// -----------------------------------------------------------------------------
+
+var synonyms = [];      // groups of synonyms
+var synonym_kbs = [];   // cache kb-formats for performance
+var synonyms_Originals = [
+    'μπίρα μπύρα μπίρες μπύρες',
+    'αυγά αβγά αυγό',
+    'σίκαλης σικάλεως',
+    'ξηρά ξερά',
+    'ρολό ρολλό',
+    'coca-cola cocacola coke',
+    'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας',
+    'χαρτί-κουζίνας ρολό-κουζίνας',
+    'οινος κρασι',
+    'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ',
+    'DR-OETKER OETKER',
+    'DR.BECKMANN BECKMANN',
+    'NES-CAFE NESCAFE',
+    'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής',
+    'τσίπουρο ρακή',
+    'Βρώμη Βρώμης',
+    'Φράουλα Φράουλες Φράουλας',
+    'Μαλλιά Μαλλιών',
+    'Κέικ Cake',
+    'CRETA-FARMS CRETA-FARM',
+    'MARSEILLAIS LE-PETIT-MARSEILLAIS PETIT-MARSEILLAIS',
+    'Γαϊδούρας Γαϊδάρου',
+    'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ',
+    'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ',
+    'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ',
+    'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ',
+    'Ντομάτα Ντομάτας',
+    'Ελαφρύ Ελαφρά Light',
+    'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά',
+    'τριμμένη τριμμένο',
+    'Τόνος Τόνου',
+    'Κριθαρένια κρίθινα',
+    'Χωρίς-Kαφεϊνη Decaffeine',
+    'Το-Μάννα Μάννα',
+    'Κράνμπερι Κράνμπερις',
+    'Κρήτης Κρητικό',
+    'Πέννες Πένες',
+    'Μακαρόνια Σπαγγέτι Σπαγγετίνι Σπαγγετόνι',
+    'Καρτέλλα Καρτέλα Καρτέλλες'
+]
+synonyms_Originals.forEach( grp => {
+    synonyms.push( grp.split(' ') );
+    synonym_kbs.push( kb_trans(grp).split(' ') );
+})
+
+// significant terms
+// -----------------------------------------------------------------------------
+significantExceptios = '7UP 3ΑΛΦΑ 17 3Π 7DAYS K2R'.split(' ')
+
+
+// replaces (correcting descriptions)
+// -----------------------------------------------------------------------------
+replaces = [];
+replaceSource = [
+    '3 ΑΛΦΑ ;3ΑΛΦΑ ',
+    'HEAD & SHOULDERS ;HEAD&SHOULDERS ',
+    'W.K Kellogg ; ',
+    'ΦΙΛΕΤ ;Φιλέτο ',
+    'ΕΝΕΛΛΑΔ ;Εν-Ελλάδι ',
+    'ΓΑΛΟΠΟΥΛ ;Γαλοπούλα ',
+    '7 DAYS ;7DAYS ',
+    'ΜΠΑΡΜΠΑ ΣΤΑΘΗ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ ',
+    'Zero% ;Zero ',
+    
+];
+replaceSource.forEach( it => {
+    st = it.split(';');
+    replaces.push({ src: st[0], trg: st[1] });
+});
+
+
+// words that shall not be searched first
+// -----------------------------------------------------------------------------
+var noRootKeywords = [];
+noRoot = [
+    'χωρίς',
+    'εισαγωγής',
+    'δώρο',
+    'γεύση',
+    'γεύσεις',
+    'φέτες',
+    'Χωρίς-Γλουτένη',
+    'Γλουτένη',
+    'Λακτόζη',
+    'Παστερίωσης',
+    'Ανθρακικό',
+    'Συντηρητικά',
+    'Χωρίς-Ζάχαρη',
+    'Άλεσης',
+    'Χωρίς-Αλάτι',
+    'Χωρίς-Λακτόζη',
+    'Χωρίς-Συντηρητικά',
+    'Χωρίς-Αλκοόλ',
+    'Χωρίς-Kαφεϊνη',
+    'Χωρίς-Γλυκάνισο',
+    'Χωρίς-Ανθρακικό',
+    'Υψηλής-Παστερίωσης',
+    'Ολες-τις-Χρήσεις',
+    'Ολικής-Άλεσης',
+    'Ολικής-Aλέσεως',
+    'Ολικής',
+    'Γαϊδούρας',
+    'Γαϊδάρου',
+    'Ρούχων',
+    'Πιάτων',
+    'πλύσεις',
+    'Πλυντηρίου',
+    'Φύλλων',
+    'Γάλακτος',
+    'Χρήσης',
+    'Τύπου',
+    'Ολλανδίας',
+    'Απορριμμάτων',
+    'Medium',
+    'Μαλλιά',
+    'Μαλλιών',
+    'Γενικής',
+    'Plus',
+    'Classic',
+    'Έκπληξη',
+    'Μάνης',
+    'Ελάτου',
+    'Άγριων',
+    'Βοτάνων',
+    'Κατσαρίδες',
+    'Λακωνίας',
+    'ΠΑΡΑΓΓΕΛΙΩΝ',
+    'Λήμνου',
+    'Αργινίνης',
+    'Καλαθάκι',
+    'Σκύλου',
+    'Γάτας',
+    'Σώματος',
+    'Αδυνατίσματος',
+    'Προστασίας',
+    'Ακράτειας',
+    'Προσώπου',
+    'προσθήκη',
+    'Βρεφικής',
+    'Καθαρισμού',
+    'Ρούχων'
+];
+noRoot.forEach( w => { noRootKeywords.push(kb_trans(w)); });
+
+
+// list of linked-words
+// -----------------------------------------------------------------------------
+linkedWords = [
+    'Χωρίς-Γλουτένη',
+    'Χωρίς-Ζάχαρη',
+    'Χωρίς-Αλάτι',
+    'Χωρίς-Λακτόζη',
+    'Χωρίς-Συντηρητικά',
+    'Χωρίς-Αλκοόλ',
+    'Χωρίς-Kαφεϊνη',
+    'Χωρίς-Γλυκάνισο',
+    'Χωρίς-Ανθρακικό',
+    'Υψηλής-Παστερίωσης',
+    'Ολικής-Άλεσης',
+    'Ολικής-Aλέσεως',
+    'Χαρτί-Υγείας',
+    'ρολό-υγείας',
+    'χαρτί-τουαλέτας',
+    'Χαρτί-Κουζίνας',
+    'Μπάρες-Δημητριακών',
+    'Μπαρμπα-Στάθης',
+    'COCA-COLA',
+    'Aς-Μαγειρέψουμε',
+    'ΚΡΙΣ-ΚΡΙΣ',
+    'ΚΡΙ-ΚΡΙ',
+    'ΕΛ-ΓΚΡΕΚΟ',
+    'FREE-STEP',
+    'EL-SABOR',
+    'LE-PETIT-MARSEILLAIS',
+    'DOUWE-EGBERTS',
+    'ΕΝ-ΕΛΛΑΔΙ',
+    'SPIN-SPAN',
+    'CRETA-FARMS',
+    'CRETA-FARM',
+    'NES-CAFE',
+    'Ολες-τις-Χρήσεις',
+    'Το-Μάννα',
+    'Χωρίς-προσθήκη-ζάχαρης'
+];
+
+
+// list of words to exclude from keywords
+// -----------------------------------------------------------------------------
+// NOTE: APPLIED in PER-WORD base -> after spliting description to words
+removeList = []
+removeOriginals = 'κατά παρά υπό από μετά προς μας με σε για του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ')
+removeOriginals.forEach( w => { removeList.push( kb_trans(w)); });
+
+
+// depricated
+/// // read frequency (local) data
+/// let rawdata = fs.readFileSync('../results/freq.json');
+/// let fr_ = JSON.parse(rawdata);      // frequencies
+/// console.log('frequencies loaded');
+
+
+
+
+// #3
+// SUPPORTING FUNCTIONS
+////////////////////////////////////////////////////////////////////////////////
+
+
+/** freq
+ * 
+ * @param id (int): product's eys_code
+ * @return frequency of product
+ */
+function freq_of(id) {
+    fr_.forEach( rec => {
+        if (rec.id == id) return rec.fq
+    });
+
+    return 0;
+}
+
+
+
+
+// proccess data
+// The MAIN proccess 
+// -----------------------------------------------------------------------------
+function extract_linked_keywords(obj) {   
+    // console.log(JSON.stringify(obj, null, 2));
+
+    obj.forEach( rec => {
+
+        if (rec.img != 0) {
+
+            var description = preproccess_text(rec.txt);
+            var pid = rec.eys;
+            var fq = freq_of(pid)+1;
+
+            prods_.push({
+                id: pid,
+                w: description
+            })
+
+            var keys = [];
+            var words = description.split(' ');
+
+            // filter words; keep only significant
+            words.forEach( w => {
+                if (removeList.indexOf(kb_trans(w)) == -1)      // if not excluded
+                    if (is_significant(w))                  // and significant
+                        keys.push(w);               // add it to keys
+            });
+            // console.log(pid, description, keys);
+
+            keys.forEach( w => {
+                var wl = synonym_keys(w);
+
+                // if key CAN be a root word (not a no-Root-keyword) update root-node
+                if (noRootKeywords.indexOf(kb_trans(w)) == -1) {
+
+                    root_key(wl, fq);       // update root keyword stats
+
+                    // if key is the only in the list of product's keywords
+                    // connect it with a dummy key (to preserve the reference to the product)
+                    if (keys.length == 1)  connect_keys(wl, ['*'], pid, fq);
+
+                    // connect w with all the other product's keywords
+                    keys.forEach( w2 => {
+                        if (w2 != w) {
+                            var w2syns = synonym_keys(w2);
+                            connect_keys( wl, w2syns, pid, fq);
+                        }
+                    });
+                }
+
+            });
+        }
+    });    // main proccessing finished;
+    
+    // post proccess 
+    // -------------------------------------------------------------------------
+
+    // remove cached keys from final array
+    kwlinks_.forEach( ro => { 
+        delete ro.kb;
+        ro.c.forEach( ch => { delete ch.kb; });
+    });
+
+    // sort root and child nodes by frequency descanding
+    kwlinks_.forEach( it => {
+        it.c = it.c.sort((a, b) => b.f - a.f );
+    });
+    kwlinks_ = kwlinks_.sort((a, b) => b.f - a.f );
+
+}
+
+
+// functions for linking words in keywords dictionary
+////////////////////////////////////////////////////////////////////////////////
+
+// set root-keyword: wl (if not exist)
+// update frequency: f
+// * wl is a list of synonym-words
+// ** comparison is based on the *keyboard* format
+//  ---
+function root_key ( wl, f ) {
+    var wkb = kb_trans(wl[0])     // cache kb format
+
+    // check if exists in root keys already
+    // NOTE: you only need to check the 1st word of synonyms-list
+    for (i=0; i< kwlinks_.length ; i++) {
+        if (kwlinks_[i].kb == wkb) {
+            kwlinks_[i].f += f;
+            return true;
+        }
+    }
+    // if not exists, append keyword
+    kwlinks_.push({
+        w  : wl,
+        kb : wkb,
+        f  : f,
+        c  : []
+    });
+    return true;
+}
+
+
+// connect keys: a , b (each one is a list of synonmyms)
+// of product with id: i
+// with frequency: f
+// ---
+function connect_keys( a, b, id, f ) {
+    var kbA = kb_trans(a[0]);
+    var kbB = kb_trans(b[0]);
+    var bExists = false;
+
+    if (kbA == kbB)     return false;       // exclude just-in-case
+    
+    for (i=0; i< kwlinks_.length ; i++) {
+        if (kwlinks_[i].kb == kbA) {        // found: a;
+            // update connection to: b
+            bExists = false;
+            for (j=0 ; j < kwlinks_[i].c.length ; j++) {
+                if (kwlinks_[i].c[j].kb == kbB) {
+                    bExists = true;
+                    // update the connection's data
+                    kwlinks_[i].c[j].f += f;
+                    kwlinks_[i].c[j].p.push(id)
+                    break;
+                }
+            }
+            // if connection not exist, init a new one
+            if (bExists == false) {
+                // create connection with: b
+                kwlinks_[i].c.push({
+                    w  : b,
+                    kb : kbB,
+                    f  : f,
+                    p  : [ i ]
+                });
+            }
+            return true;
+        }
+    }
+}
+
+
+// other supplementary functions
+////////////////////////////////////////////////////////////////////////////////
+
+
+// kb_trans translates string to keyboard-latin keys;
+// ---
+function kb_trans(str) {
+    str = str.replace('\'','');
+    var out = '';
+    for (var i=0 ; i< str.length; i++)  out += map.get(str[i]);
+    return out;
+}
+
+
+// clean text trims some characters (+.') and internal multiple-spaces
+// ---
+function clean_text(txt) {
+    return txt.replace('+',' ').replace('.',' ')
+                .replace('   ',' ')
+                .replace('  ',' ');
+}
+
+
+// check if term is significant
+// (if not, the term will be excluded from keywords dicionary)
+// ---
+function is_significant(str) {
+    if (str == '')     return false;
+    if (significantExceptios.indexOf(str) !== -1)   return true;
+    return !(/\d/.test(str));
+}
+
+
+// check if word: w
+// ...has synonyms; return list of synonyms
+// ---
+function synonym_keys(w) {
+    w_kb = kb_trans(w);
+    for (i=0 ; i < synonym_kbs.length ; i++) {
+        if (synonym_kbs.indexOf(w_kb) !== -1)
+            return synonyms[i];
+    }
+    return [ w ];
+}
+
+
+// edit common mistakes
+// with suggested replaces
+function do_replaces(str) {
+    replaces.forEach( it => { str = str.replace(it.src, it.trg); });
+    return str;
+}
+
+
+// preproccess description
+// ---
+function preproccess_text(str) {
+    str = do_replaces(str);
+    str = clean_text(str);
+    str = mark_linked_words(str);
+    return str;
+}
+
+
+// mark linked words (connect them with a dash)
+// return new text after "all-links" are marked
+// ---
+function mark_linked_words(txt) {
+    linkedWords.forEach( lw => { txt = mark_link( lw, txt ); });
+    return txt
+}
+
+
+// mark a link (lws) to a text (source)
+// conecting them with a dash/minus character
+// ---
+function mark_link(lws, source) {
+    var src_kb = kb_trans(source.replace(' ', '-'));
+    var lws_kb = kb_trans(lws.replace(' ', '-'));
+    var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase())
+    if (_left !== -1 ) {
+        return source.slice(0, _left) + lws + source.slice(_left + lws.length);
+    }
+    else return source;
+}
+
+
+
+// #4
+// OUTPUT FUNCTIONS
+////////////////////////////////////////////////////////////////////////////////
+
+
+// Save Local
+// -----------------------------------------------------------------------------
+function save_local(jsonArray, filePath) {
+    let fileStr = JSON.stringify(jsonArray);    // convert json to string
+    
+    // write string to file
+    fs.writeFileSync(filePath, fileStr, 'utf8', (err) => {
+        if (err) {
+            console.log("An error occured while writing keywords.json");
+            return console.log(err);
+        }
+        // console.log("JSON file has been saved.");
+    });
+
+    return true;
+}
+
+
+
+// Save to Google Cloud Storage
+// -----------------------------------------------------------------------------
+async function upload_file( bucketName, srcFilePath, trgFilePath ) {
+    // Creates a client
+    const storage = new Storage();
+
+    try {
+        await storage.bucket(bucketName).upload(srcFilePath, {
+            destination: trgFilePath, 
+            gzip: true,     // serve compressed
+            metadata: {     // cache for 8 hours
+                cacheControl: 'public, max-age=60'      // production set: 28800
+            }
+        });
+        console.log(`${srcFilePath} uploaded to ${bucketName}`);
+    }
+
+    catch(err) {
+        console.error('ERROR:', err);
+    }
+
+}
+
+
+
+
+// #5
+// MAIN function (exposed function to run the whole proccess)
+////////////////////////////////////////////////////////////////////////////////
+
+
+/** main()
+ * 
+ * nodejs's exported function
+ * used as entry-point function (in case of Google Cloud function)
+ */
+
+// keep only 1 of next 2 lines
+// exports.main = () => {       // entry point functioon when google cloud function
+var main = () => {              // concise function when direct script
+
+    fetch(frequency_endpoint, request_settings)
+    .then(res => res.json())
+    .then((json) => {
+        fr_ = json;
+        console.log('frequencies loaded from CDN');
+
+        fetch(products_endpoint, request_settings)
+        .then(res => res.json())
+        .then((json) => {
+
+            console.log('products data loaded from emarket api/endpoint');
+
+            // proccess data
+            extract_linked_keywords(json.data);
+
+            save_local(kwlinks_, temp_kw_file);
+            save_local(kwlinks_, temp_prod_file);
+
+            // upload temp files to cloud
+            upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json');
+            upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json');
+
+            // check
+            // echo 100 most frequent
+            let i = 0;
+            kwlinks_.forEach(rec => {
+                if (i<100) {
+                    console.log(i, JSON.stringify(rec.w))
+                }
+                i++;
+            });
+
+            // echo memory stats
+            const used = process.memoryUsage();
+            for (let key in used) {
+                console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
+            }
+
+        });
+
+    });
+
+}