linkeysearch:/javascript new async functions for parallel run of slow proccesses

3 files changed, 107 insertions, 64 deletions

diff --git a/javascript/freq.js b/javascript/freq.js
index 86bba22..bf52008 100644
--- a/javascript/freq.js
+++ b/javascript/freq.js
@@ -19,13 +19,9 @@
 // REQUIREMENTS
 ////////////////////////////////////////////////////////////////////////////////
 
-
 var mysql = require('mysql');
-
 const fs = require('fs');
-
 const os = require('os');
-
 const {Storage} = require('@google-cloud/storage');     // import Google Cloud client library
 
 require('dotenv').config({ path: '../auth/.env' })
@@ -166,7 +162,7 @@ var main = () => {
         console.log("Connected!");
 
         // SQL to get needed data
-        var sql = "SELECT count(pl.eys_code) as FREQuency,\
+        var sql = "SELECT count(pl.eys_code)*5+SUM(dop.quantity) as FREQuency,\
                 pl.product_id as product_id,\
                 pb.brand_name,\
                 pl.barcode, pl.skl_code, pl.eys_code,\
diff --git a/javascript/keygen.js b/javascript/keygen.js
index 59407a9..dbd5717 100644
--- a/javascript/keygen.js
+++ b/javascript/keygen.js
@@ -19,7 +19,6 @@
 const fs = require('fs');
 const os = require('os');
 const fetch = require('node-fetch');
-const { ECDH } = require('crypto');
 const {Storage} = require('@google-cloud/storage');     // Google Cloud Storage
 
 
@@ -52,11 +51,22 @@ const temp_prod_file = '../results/products.json';
 // Global Variables
 // -----------------------------------------------------------------------------
 
+// Global arrays
 // keyword links (word-links dictionary; array of objects)
 // -----------------------------------------------------------------------------
-var kwlinks_ = [];      ////////////////////////////// MAIN OUTPUT OF THE SCRIPT
-var prods_ = [];
-var fr_ =[];
+var kwlinks_ = [];  // keyword-links structure (structure to construct)
+var prods_ = [];    // products (array to construct)
+var _fr =[];        // product frequencies array (as read from endpiont) 
+var _pr =[];        // product array (as read from endpiont) 
+
+
+// Console Colors
+// -----------------------------------------------------------------------------
+const Reset = "\x1b[0m";
+const FgRed = "\x1b[31m";
+const FgGreen = "\x1b[32m";
+
+
 
 
 // #2.2
@@ -178,7 +188,10 @@ noRoot = [
     'Ολικής-Άλεσης',
     'Ολικής-Aλέσεως',
     'Ολικής',
+    'Σοκολάτας',
+    'Υγείας',
     'Γαϊδούρας',
+    'Δημητριακών',
     'Γαϊδάρου',
     'Ρούχων',
     'Πιάτων',
@@ -209,6 +222,9 @@ noRoot = [
     'Καλαθάκι',
     'Σκύλου',
     'Γάτας',
+    'ΠΓΕ',
+    'ΤΟ',
+    'Μελί',
     'Σώματος',
     'Αδυνατίσματος',
     'Προστασίας',
@@ -271,13 +287,6 @@ removeOriginals = 'κατά παρά υπό από μετά προς μας με
 removeOriginals.forEach( w => { removeList.push( kb_trans(w)); });
 
 
-// depricated
-/// // read frequency (local) data
-/// let rawdata = fs.readFileSync('../results/freq.json');
-/// let fr_ = JSON.parse(rawdata);      // frequencies
-/// console.log('frequencies loaded');
-
-
 
 
 // #3
@@ -291,7 +300,7 @@ removeOriginals.forEach( w => { removeList.push( kb_trans(w)); });
  * @return frequency of product
  */
 function freq_of(id) {
-    fr_.forEach( rec => {
+    _fr.forEach( rec => {
         if (rec.id == id) return rec.fq
     });
 
@@ -553,8 +562,8 @@ function do_replaces(str) {
 // preproccess description
 // ---
 function preproccess_text(str) {
-    str = do_replaces(str);
     str = clean_text(str);
+    str = do_replaces(str);
     str = mark_linked_words(str);
     return str;
 }
@@ -589,21 +598,24 @@ function mark_link(lws, source) {
 ////////////////////////////////////////////////////////////////////////////////
 
 
+
 // Save Local
 // -----------------------------------------------------------------------------
-function save_local(jsonArray, filePath) {
+async function save_local(jsonArray, filePath) {
     let fileStr = JSON.stringify(jsonArray);    // convert json to string
-    
-    // write string to file
-    fs.writeFileSync(filePath, fileStr, 'utf8', (err) => {
-        if (err) {
-            console.log("An error occured while writing keywords.json");
-            return console.log(err);
-        }
-        // console.log("JSON file has been saved.");
-    });
 
-    return true;
+    // return new Promise(function(resolve, reject) {
+
+        // write string to file
+        fs.writeFileSync(filePath, fileStr, 'utf8', (err) => {
+            if (err) {
+                return err;
+            } else {
+                return true;
+            }
+        });
+
+    // });
 }
 
 
@@ -649,43 +661,79 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) {
 // exports.main = () => {       // entry point functioon when google cloud function
 var main = () => {              // concise function when direct script
 
-    fetch(frequency_endpoint, request_settings)
-    .then(res => res.json())
-    .then((json) => {
-        fr_ = json;
-        console.log('frequencies loaded from CDN');
+    // PROMISE ...
+    // fetch all jsons from endpints ...........................................
+    Promise.all([
 
         fetch(products_endpoint, request_settings)
-        .then(res => res.json())
-        .then((json) => {
+            .then(res => res.json())
+            .then((json) => {
+                _pr = json;
+                console.log('products data loaded from emarket api/endpoint');
+            }),
+
+        fetch(frequency_endpoint, request_settings)
+            .then(res => res.json())
+            .then((json) => {
+                _fr = json;
+                console.log('frequencies loaded from CDN')
+            })
+
+    ])
+    .then (() => {  // then...
+        // proccess data and extract ketwords
+
+        console.log(`... All data from endpoints: ${FgGreen}LOADED!${Reset}`);
+
+        // proccess data
+        extract_linked_keywords(_pr.data);
+
+        console.log('keywords extracted;\ntry to save results localy...');
 
-            console.log('products data loaded from emarket api/endpoint');
+        // PROMISE ...
+        // save local files ....................................................
+        Promise.all([
+            save_local(kwlinks_, temp_kw_file)
+            .then(() => { console.log('keywords saced localy');}),
 
-            // proccess data
-            extract_linked_keywords(json.data);
+            save_local(prods_, temp_prod_file)
+            .then(() => { console.log('products saved localy');})
+        ])
+        .then(() => {   // then ...
 
-            save_local(kwlinks_, temp_kw_file);
-            save_local(kwlinks_, temp_prod_file);
+            console.log(`... All local saves: ${FgGreen}DONE!${Reset}`);
 
-            // upload temp files to cloud
-            upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json');
-            upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json');
+            // PROMISE ...
+            // upload keytword and product jsons into Cloud Storage ............
+            Promise.all([
+                upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'),
+                upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json')
+            ])
+            .then(() => {   // then ...
+
+                console.log(`... Uploads to cloud storage: ${FgGreen}DONE!${Reset}`);
+
+                // echo usage stats; then END
+
+                // check
+                // echo 100 most frequent
+                let i = 0;
+                kwlinks_.forEach(rec => {
+                    if (i<100) {
+                        console.log(i, JSON.stringify(rec.w))
+                    }
+                    i++;
+                });
 
-            // check
-            // echo 100 most frequent
-            let i = 0;
-            kwlinks_.forEach(rec => {
-                if (i<100) {
-                    console.log(i, JSON.stringify(rec.w))
+                // echo memory stats
+                const used = process.memoryUsage();
+                for (let key in used) {
+                    console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
                 }
-                i++;
-            });
 
-            // echo memory stats
-            const used = process.memoryUsage();
-            for (let key in used) {
-                console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
-            }
+                console.log(`${FgGreen}THE END!${Reset}`);
+
+            });
 
         });
 
diff --git a/javascript/keygen_GCF.js b/javascript/keygen_GCF.js
index e6568ff..1e22f18 100644
--- a/javascript/keygen_GCF.js
+++ b/javascript/keygen_GCF.js
@@ -524,23 +524,23 @@ function mark_link(lws, source) {
 
 // Save Local
 // -----------------------------------------------------------------------------
-function save_local(jsonArray, filePath) {
+async function save_local(jsonArray, fileName) {
     let fileStr = JSON.stringify(jsonArray);    // convert json to string
     
     // write string to file
-    fs.writeFileSync(filePath, fileStr, 'utf8', (err) => {
+    fs.writeFileSync(fileName, fileStr, 'utf8', (err) => {
         if (err) {
             console.log("An error occured while writing keywords.json");
             return console.log(err);
         }
-        // console.log("JSON file has been saved.");
+        console.log("JSON file", fileName, "has been saved.");
+
     });
 
     return true;
 }
 
 
-
 // Save to Google Cloud Storage
 // -----------------------------------------------------------------------------
 async function upload_file( bucketName, srcFilePath, trgFilePath ) {
@@ -579,8 +579,8 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) {
  */
 
 // keep only 1 of next 2 lines
-// exports.main = () => {       // entry point functioon when google cloud function
-var main = () => {              // concise function when direct script
+exports.main = () => {       // entry point functioon when google cloud function
+
 
     fetch(frequency_endpoint, request_settings)
     .then(res => res.json())
@@ -604,7 +604,6 @@ var main = () => {              // concise function when direct script
             upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json');
             upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json');
 
-
         });
 
     });