linkeysearch: 2 step implementation; prepare scripts to run as Google Cloud Funcctions

2 files changed, 306 insertions, 185 deletions

diff --git a/javascript/freq.js b/javascript/freq.js
index 5f740b7..0ac7779 100644
--- a/javascript/freq.js
+++ b/javascript/freq.js
@@ -1,6 +1,21 @@
-// requirements
+/** freq.js
+ * 
+ * this script extracts products' order-frequency 
+ * -----------------------------------------------------------------------------
+ * 
+ * Contents:
+ * #1 Requirements
+ * #2 Personalized constants and parametres
+ * #3 Supporting functions
+ * #4 Output functions
+ * #5 Entry-point function main()
+ */
+
+// #1
+// REQUIREMENTS
 ////////////////////////////////////////////////////////////////////////////////
 
+
 var mysql = require('mysql');
 
 const fs = require('fs');
@@ -9,16 +24,26 @@ const {Storage} = require('@google-cloud/storage');     // import Google Cloud c
 
 
 
-// (setup/personalize) constants
+
+// #2
+// SETUP PERSONALIZED CONSTANTS AND PARAMETRES
 ////////////////////////////////////////////////////////////////////////////////
 
+
 // google-storage parametres
 // --- -- -- - - -
 const projectId = 'pythia-251711';
 const keyFilename = '../auth/pythia-251711-047e3d5e6608.json';
 
+// temp file of frequencies (json)
+const temp_file = '../results/freq.json';
+
+
 // db connection parametres
+// (keep the one that suits your environment; comment out the other)
 // -----------------------------------------------------------------------------
+
+// while on local machine using cloud_sql_proxy
 var con = mysql.createConnection({  
     host: "127.0.0.1",
     user: "pythia_db_user_dev",
@@ -26,109 +51,54 @@ var con = mysql.createConnection({
     database: "dev_pythia_db"
 });
 
+// // while on Google Cloud Functions
+// var con = mysql.createConnection({  
+//     socketPath: "/cloudsql/pythia-251711:europe-west4:pythia-db-eu",
+//     user: "pythia_services",
+//     password: process.env.DB_PASSWORD,
+//     database: "pythia_db"
+// });
 
 // MAIN OUTPUT OF THE SCRIPT
 // arrays that need to be constructed and filled with data
 // -----------------------------------------------------------------------------
 
 /** array of product frequencies
- * array of obj:{ id, fq }
+ * array of obj: { id: , fq: }
  * @var id (int): product's eys code
  * @var fq (int): (order) frerquency 
  */
 var freq_ = [];
 
-/** array of products
- * array of obj: { id, w }
- * 
- * @var id (int): product's eys_code
- * @var w (string): descripion
- */
-var prods_ = [];    // products array
 
 
-// connect;
-// get records to proccess;
-// call main proccess function;
-// save dictionary;
-// end script;
-// -----------------------------------------------------------------------------
-con.connect(function(err) {
-    // connect;
-    if (err) throw err;
-    console.log("Connected!");
-
-    // SQL to get needed data 
-    var sql = "SELECT count(pl.eys_code) as FREQuency,\
-            pl.product_id as product_id,\
-            pb.brand_name,\
-            pl.barcode, pl.skl_code, pl.eys_code,\
-            IF( pd.description IS NOT NULL , pd.description , pl.product_description ) AS product_description,\
-            pl.bpcs_code,\
-            pd.image_path\
-        FROM product_list as pl\
-        LEFT JOIN delivery_orders_products AS dop ON dop.product_id = pl.eys_code\
-        LEFT JOIN delivery_orders AS do ON dop.order_id = do.id\
-        LEFT JOIN product_list AS replacement ON dop.replacement_for = replacement.eys_code\
-        LEFT JOIN product_details AS pd ON pl.eys_code = pd.eys_code\
-        LEFT JOIN product_brands pb ON pl.brand_id = pb.id\
-        WHERE pl.active = 1 AND pl.sap_code IS NOT NULL AND pl.product_category_sap_4 NOT LIKE '72%'\
-        GROUP BY pl.product_id\
-        ORDER BY FREQuency DESC\
-        LIMIT 60000";
-
-    // query sql
-    con.query(sql, function (err, result) {
-        if (err) throw err;
-        console.log('Records from database received!')
-
-        do_proccess(result);    // proccess
-        console.log('Keywords proccesed!')
-
-        save_local(freq_, '../results/freq.json');        // save frequencies
-        save_local(prods_, '../results/prods.json');      // save product (descriptionb per id)
-        console.log('Results saved! Exiting.')
-
-        // echo memory stats
-        const used = process.memoryUsage();
-        for (let key in used) {
-            console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
-        }
-        process.exit(1);        // exit
-    });
-});
+// #3
+// SUPPORTING FUNCTIONS
+////////////////////////////////////////////////////////////////////////////////
 
 
-// main proccess
-// -----------------------------------------------------------------------------
-function do_proccess(obj) {
-    // console.log(JSON.stringify(obj, null, 2));
 
-    var freq = [];
+/** extract frequencies
+ * 
+ * @param obj (array): array of products
+ */
+function extract_freq(obj) {
 
     obj.forEach( rec => {
         var description = rec.product_description;
         var fq  = rec.FREQuency;
-        var pid = rec.eys_code;
-
-
-        
-
-        prods_.push({
-            id: pid,
-            w: description
-        });
+        var pid = rec.eys_code;      
 
         freq_.push({
             fq: fq,
             id: pid
         });
-    });    // main proccessing finished;
-    
+    });
 }
 
 
-// save proccesses
+// #4
+// OUTPUT FUNCTIONS
 ////////////////////////////////////////////////////////////////////////////////
 
 // Save Local
@@ -144,16 +114,16 @@ function save_local(jsonArray, filePath) {
         }
         console.log("JSON file has been saved.");
     });
-}
 
+    return true;
+}
 
-// Save to Cloud Storage
-// -----------------------------------------------------------------------------
 
 
+// Save to Google Cloud Storage
+// -----------------------------------------------------------------------------
 async function upload_file( bucketName, srcFilePath, trgFilePath ) {
     // Creates a client
-
     const storage = new Storage({projectId, keyFilename});
 
     try {
@@ -169,4 +139,83 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) {
     catch(err) {
         console.error('ERROR:', err);
     }
-}
\ No newline at end of file
+}
+
+
+
+// #5
+// MAIN function (exposed function to run the whole proccess)
+////////////////////////////////////////////////////////////////////////////////
+
+
+/** main()
+ * 
+ * nodejs's exported function
+ * used as entry-point function (in case of Google Cloud function)
+ */
+
+// exports.main = () => {   // google cloud function entry point
+var main = () => {
+
+    // connect;
+    // get records to proccess;
+    // call main proccess function; 
+    // save dictionary;
+    // end script;
+    // -----------------------------------------------------------------------------
+    con.connect(function(err) {
+        // connect;
+        if (err) throw err;
+        console.log("Connected!");
+
+        // SQL to get needed data 
+        var sql = "SELECT count(pl.eys_code) as FREQuency,\
+                pl.product_id as product_id,\
+                pb.brand_name,\
+                pl.barcode, pl.skl_code, pl.eys_code,\
+                IF( pd.description IS NOT NULL , pd.description , pl.product_description ) AS product_description,\
+                pl.bpcs_code,\
+                pd.image_path\
+            FROM product_list as pl\
+            LEFT JOIN delivery_orders_products AS dop ON dop.product_id = pl.eys_code\
+            LEFT JOIN delivery_orders AS do ON dop.order_id = do.id\
+            LEFT JOIN product_list AS replacement ON dop.replacement_for = replacement.eys_code\
+            LEFT JOIN product_details AS pd ON pl.eys_code = pd.eys_code\
+            LEFT JOIN product_brands pb ON pl.brand_id = pb.id\
+            WHERE pl.active = 1 AND pl.sap_code IS NOT NULL AND pl.product_category_sap_4 NOT LIKE '72%'\
+            GROUP BY pl.product_id\
+            ORDER BY FREQuency DESC\
+            LIMIT 60000";
+
+        // query sql
+        con.query(sql, function (err, result) {
+            if (err) throw err;
+            console.log('Records from database received!')
+
+            extract_freq(result);    // proccess
+            console.log('Keywords proccesed!')
+
+            save_local(freq_, temp_file);        // save frequencies
+
+            upload_file('pythia-files', temp_file, 'uploads/json/freq.json')
+            .then( () => {
+                console.log('Results saved! Exiting...');
+                process.exit(1);
+            });
+
+            /// // echo memory stats
+            /// const used = process.memoryUsage();
+            /// for (let key in used) {
+            ///     console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
+            /// }
+
+            // process.exit(1);        // exit
+        });
+    });
+}
+
+// NOTE:
+// if running the script directry you need to call the main function
+// if running through google cloud-functions you do NOT need to call main()
+// (you need define main() as the entry-point function instead)
+main();
\ No newline at end of file
diff --git a/javascript/keygen.js b/javascript/keygen.js
index 16e2490..39174b1 100644
--- a/javascript/keygen.js
+++ b/javascript/keygen.js
@@ -1,4 +1,19 @@
-// requirements
+/** keygen
+ * 
+ * this script constucts a linked-wordkeys structure
+ * -----------------------------------------------------------------------------
+ * 
+ * Contents:
+ * #1 Requirements
+ * #2.1 Personalized constants and parametres
+ * #2.2 Preloaded Data
+ * #3 Supporting functions
+ * #4 Output functions
+ * #5 Entry-point function main()
+ */
+
+// #1
+// REQUIREMENTS
 ////////////////////////////////////////////////////////////////////////////////
 
 const fs = require('fs');
@@ -7,17 +22,45 @@ const fetch = require('node-fetch');
 const { ECDH } = require('crypto');
 const {Storage} = require('@google-cloud/storage');     // Google Cloud Storage
 
-// (setup/personalize) constants
+
+
+// #2.1
+// SETUP PERSONALIZED CONSTANTS AND PARAMETRES
 ////////////////////////////////////////////////////////////////////////////////
 
+
+// get products FROM api/endpoint parametres
+const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json';
+const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch";
+const request_settings = { method: "Get" };
+
+
 // google-storage parametres
 // --- -- -- - - -
 const projectId = 'pythia-251711';
 const keyFilename = '../auth/pythia-251711-047e3d5e6608.json';
 
 
+// temp files
+// need to be created prior to save-to-google-storage proccess
+// because upload_file() uploats an existing local file to G-Storage
+// --- -- -- - - -
+const temp_kw_file = '../results/keywords.json';
+const temp_prod_file = '../results/products.json';
+
+
+// Global Variables
+// -----------------------------------------------------------------------------
+
+// keyword links (word-links dictionary; array of objects)
+// -----------------------------------------------------------------------------
+var kwlinks_ = [];      ////////////////////////////// MAIN OUTPUT OF THE SCRIPT
+var prods_ = [];
+var fr_ =[];
+
 
-// preloaded data
+// #2.2
+// PRELOADED DATA
 ////////////////////////////////////////////////////////////////////////////////
 
 // any-character to keyboard-latin mapping
@@ -228,10 +271,19 @@ removeOriginals = 'κατά παρά υπό από μετά προς μας με
 removeOriginals.forEach( w => { removeList.push( kb_trans(w)); });
 
 
-// read frequency data
-let rawdata = fs.readFileSync('../results/freq.json');
-let fr_ = JSON.parse(rawdata);      // frequencies
-console.log('frequencies loaded');
+// depricated
+/// // read frequency (local) data
+/// let rawdata = fs.readFileSync('../results/freq.json');
+/// let fr_ = JSON.parse(rawdata);      // frequencies
+/// console.log('frequencies loaded');
+
+
+
+
+// #3
+// SUPPORTING FUNCTIONS
+////////////////////////////////////////////////////////////////////////////////
+
 
 /** freq
  * 
@@ -247,57 +299,15 @@ function freq_of(id) {
 }
 
 
-// get products from api/endpoint (in Json format)
-// --- -- -- - - -
-let url = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch";
-
-let settings = { method: "Get" };
-
-fetch(url, settings)
-    .then(res => res.json())
-    .then((json) => {
-
-        let temp_file = '../results/keywords.json';
-        let temp_prod_file = '../results/products.json';
-
-        console.log('products data loaded from emarket api/endpoint');
-
-        // do something with JSON
-        var linkeys = do_proccess(json.data);
-
-        save_local(kwlinks_, temp_kw_file);
-        save_local(kwlinks_, temp_prod_file);
-
-        // upload_file('pythia-files', os.tmpdir()+'/keywords.json', 'uploads/orders/keywords.json');
-        upload_file('pythia-files', temp_kw_file, 'uploads/orders/emarket-keywords.json');
-        upload_file('pythia-files', temp_prod_file, 'uploads/orders/emarket-keywords.json');
-
-        // check
-        // echo 100 most frequent
-        let i = 0;
-        kwlinks_.forEach(rec => {
-            if (i<100) {
-                console.log(i, JSON.stringify(rec.w))
-            }
-            i++;
-        })
-});
-
-
-
-// keyword links (word-links dictionary; array of objects)
-// -----------------------------------------------------------------------------
-var kwlinks_ = [];      ////////////////////////////// MAIN OUTPUT OF THE SCRIPT
-var prods_ = [];
 
+/* ---
 // connect;
 // get records to proccess;
 // call main proccess function;
 // save dictionary;
 // end script;
 // -----------------------------------------------------------------------------
-
-exports.main = () => {
+var main = () => {
 
     con.connect(function(err) {
         // connect;
@@ -340,13 +350,13 @@ exports.main = () => {
         });
     });
 }
+--- */
 
 
-var unmatched = 0;
-
-// main proccess
+// proccess data
+// The MAIN proccess 
 // -----------------------------------------------------------------------------
-function do_proccess(obj) {   
+function extract_linked_keywords(obj) {   
     // console.log(JSON.stringify(obj, null, 2));
 
     obj.forEach( rec => {
@@ -362,8 +372,6 @@ function do_proccess(obj) {
                 w: description
             })
 
-            // if (fq == 0) console.log(pid, description);
-
             var keys = [];
             var words = description.split(' ');
 
@@ -418,52 +426,6 @@ function do_proccess(obj) {
 }
 
 
-
-// save proccesses
-////////////////////////////////////////////////////////////////////////////////
-
-// Save Local
-// -----------------------------------------------------------------------------
-function save_local(jsonArray, filePath) {
-    let fileStr = JSON.stringify(jsonArray);    // convert json to string
-    
-    // write string to file
-    fs.writeFileSync(filePath, fileStr, 'utf8', (err) => {
-        if (err) {
-            console.log("An error occured while writing keywords.json");
-            return console.log(err);
-        }
-        console.log("JSON file has been saved.");
-    });
-
-    return true;
-}
-
-
-
-// Save to Cloud Storage
-// -----------------------------------------------------------------------------
-
-
-async function upload_file( bucketName, srcFilePath, trgFilePath ) {
-    // Creates a client
-    const storage = new Storage({projectId, keyFilename});
-
-    try {
-        await storage.bucket(bucketName).upload(srcFilePath, {
-            destination: trgFilePath, 
-            gzip: true,     // serve compressed
-            metadata: {     // cache for 8 hours
-                cacheControl: 'public, max-age=60'      // production set: 28800
-            }
-        });
-        console.log(`${srcFilePath} uploaded to ${bucketName}`);
-    }
-    catch(err) {
-        console.error('ERROR:', err);
-    }
-}
-
 // functions for linking words in keywords dictionary
 ////////////////////////////////////////////////////////////////////////////////
 
@@ -620,3 +582,113 @@ function mark_link(lws, source) {
     else return source;
 }
 
+
+
+// #4
+// OUTPUT FUNCTIONS
+////////////////////////////////////////////////////////////////////////////////
+
+
+// Save Local
+// -----------------------------------------------------------------------------
+function save_local(jsonArray, filePath) {
+    let fileStr = JSON.stringify(jsonArray);    // convert json to string
+    
+    // write string to file
+    fs.writeFileSync(filePath, fileStr, 'utf8', (err) => {
+        if (err) {
+            console.log("An error occured while writing keywords.json");
+            return console.log(err);
+        }
+        // console.log("JSON file has been saved.");
+    });
+
+    return true;
+}
+
+
+
+// Save to Google Cloud Storage
+// -----------------------------------------------------------------------------
+async function upload_file( bucketName, srcFilePath, trgFilePath ) {
+    // Creates a client
+    const storage = new Storage({projectId, keyFilename});
+
+    try {
+        await storage.bucket(bucketName).upload(srcFilePath, {
+            destination: trgFilePath, 
+            gzip: true,     // serve compressed
+            metadata: {     // cache for 8 hours
+                cacheControl: 'public, max-age=60'      // production set: 28800
+            }
+        });
+        console.log(`${srcFilePath} uploaded to ${bucketName}`);
+    }
+
+    catch(err) {
+        console.error('ERROR:', err);
+    }
+
+}
+
+
+
+
+// #5
+// MAIN function (exposed function to run the whole proccess)
+////////////////////////////////////////////////////////////////////////////////
+
+
+/** main()
+ * 
+ * nodejs's exported function
+ * used as entry-point function (in case of Google Cloud function)
+ */
+
+// keep only 1 of next 2 lines
+// exports.main = () => {       // entry point functioon when google cloud function
+var main = () => {              // concise function when direct script
+
+    fetch(frequency_endpoint, request_settings)
+    .then(res => res.json())
+    .then((json) => {
+        fr_ = json;
+        console.log('frequencies loaded from CDN');
+
+        fetch(products_endpoint, request_settings)
+        .then(res => res.json())
+        .then((json) => {
+
+            console.log('products data loaded from emarket api/endpoint');
+
+            // proccess data
+            extract_linked_keywords(json.data);
+
+            save_local(kwlinks_, temp_kw_file);
+            save_local(kwlinks_, temp_prod_file);
+
+            // upload temp files to cloud
+            upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json');
+            upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json');
+
+            /// // check
+            /// // echo 100 most frequent
+            /// let i = 0;
+            /// kwlinks_.forEach(rec => {
+            ///     if (i<100) {
+            ///         console.log(i, JSON.stringify(rec.w))
+            ///     }
+            ///     i++;
+            /// })
+        });
+
+    });
+
+}
+
+
+// NOTE:
+// if running the script directry you need to call the main function
+// if running through google cloud-functions you do NOT need to call main()
+// (you need define main() as the entry-point function instead)
+main();
\ No newline at end of file