freq_GCF: freq implementation as Google Cloud Function; added security options

5 files changed, 55 insertions, 65 deletions

diff --git a/.gitignore b/.gitignore
index 9ad17eb..b41c06d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,4 +54,7 @@ Thumbs.db
 app/public/temp/
 
 # authentication files (json) for Google Cloud Services 
-auth/*.json
\ No newline at end of file
+auth/*.json
+
+# environmental variable files
+.env
\ No newline at end of file
diff --git a/javascript/freq.js b/javascript/freq.js
index 19c42f5..0bbdb5b 100644
--- a/javascript/freq.js
+++ b/javascript/freq.js
@@ -9,6 +9,10 @@
  * #3 Supporting functions
  * #4 Output functions
  * #5 Entry-point function main()
+ * 
+ * Requirements
+ * npm install dotenv --save
+ * 
  */
 
 // #1
@@ -24,7 +28,7 @@ const os = require('os');
 
 const {Storage} = require('@google-cloud/storage');     // import Google Cloud client library
 
-
+require('dotenv').config({ path: '../auth/.env' })
 
 
 // #2
@@ -38,19 +42,9 @@ const projectId = 'pythia-251711';
 const keyFilename = '../auth/pythia-251711-047e3d5e6608.json';
 
 
-// cloud_environment (boolean)
-// set value to:
-// true  : if running on Google Cloud Functions
-// false : if running localy
-// --- -- -- - - -
-const cloud_environment = false;        // ** check
-
-
 // temp file of frequencies (json)
-const saves_dir = '../results';         // ** check
-const temp_file = 'freq.json';          // ** check
+const temp_file = '../results/freq.json';
 
-const local_dir = (cloud_environment) ? os.tmpdir() : saves_dir;
 
 
 // db connection parametres
@@ -58,20 +52,13 @@ const local_dir = (cloud_environment) ? os.tmpdir() : saves_dir;
 // -----------------------------------------------------------------------------
 
 // while on local machine using cloud_sql_proxy
-var con = mysql.createConnection({      // ** check
-    host: "127.0.0.1",
-    user: "pythia_db_user_dev",
-    password: "VnEP0eysjiXDHcfM",
-    database: "dev_pythia_db"
+var con = mysql.createConnection({
+    host: process.env.DB_HOST;
+    user: process.env.DB_USER;
+    password: process.env.DB_PASS;
+    database: process.env.DB_NAME;
 });
 
-// // while on Google Cloud Functions
-// var con = mysql.createConnection({   // ** check
-//     socketPath: "/cloudsql/pythia-251711:europe-west4:pythia-db-eu",
-//     user: "pythia_services",
-//     password: process.env.DB_PASSWORD,
-//     database: "pythia_db"
-// });
 
 // MAIN OUTPUT OF THE SCRIPT
 // arrays that need to be constructed and filled with data
@@ -163,11 +150,8 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) {
 
 /** main()
  * 
- * nodejs's exported function
- * used as entry-point function (in case of Google Cloud function)
+ * the entry-point function
  */
-
-// exports.main = () => {   // google cloud function entry point
 var main = () => {
 
     // connect;
@@ -176,12 +160,12 @@ var main = () => {
     // save dictionary;
     // end script;
     // -----------------------------------------------------------------------------
-    con.connect(function(err) {         // comment this out for 'Cannot enqueue Handshake after already enqueuing' error
-        // connect;                     // comment this out for 'Cannot enqueue Handshake after already enqueuing' error
-        if (err) throw err;             // comment this out for 'Cannot enqueue Handshake after already enqueuing' error
-        console.log("Connected!");      // comment this out for 'Cannot enqueue Handshake after already enqueuing' error
+    con.connect(function(err) {
+        // connect;
+        if (err) throw err;
+        console.log("Connected!");
 
-        // SQL to get needed data 
+        // SQL to get needed data
         var sql = "SELECT count(pl.eys_code) as FREQuency,\
                 pl.product_id as product_id,\
                 pb.brand_name,\
@@ -216,15 +200,15 @@ var main = () => {
                 process.exit(1);
             });
 
-            /// // echo memory stats
-            /// const used = process.memoryUsage();
-            /// for (let key in used) {
-            ///     console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
-            /// }
+            // echo memory stats
+            const used = process.memoryUsage();
+            for (let key in used) {
+                console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
+            }
 
-            // process.exit(1);        // exit
+            // process.exit();        // exit
         });
-    });             // comment this out for 'Cannot enqueue Handshake after already enqueuing' error
+    });
 }
 
 // NOTE:
@@ -232,4 +216,4 @@ var main = () => {
 // if running through google cloud-functions you do NOT need to call main()
 // (you need define main() as the entry-point function instead)
 // --- -- -- - - -
-main();     // ** check
\ No newline at end of file
+main();
\ No newline at end of file
diff --git a/javascript/freq_GCF.js b/javascript/freq_GCF.js
index 79a79bc..79810e6 100644
--- a/javascript/freq_GCF.js
+++ b/javascript/freq_GCF.js
@@ -1,5 +1,6 @@
 /** freq.js
  * 
+ * Google Cloud Functions impementation of `linkeysearch:javascript/freq`
  * this script extracts products' order-frequency 
  * -----------------------------------------------------------------------------
  * 
@@ -17,11 +18,8 @@
 
 
 var mysql = require('mysql');
-
 const fs = require('fs');
-
 const os = require('os');
-
 const {Storage} = require('@google-cloud/storage');     // import Google Cloud client library
 
 
@@ -42,7 +40,6 @@ const projectId = 'pythia-251711';
 const temp_file = os.tmpdir() + '/freq.json';
 
 
-
 // db connection parametres
 // (keep the one that suits your environment; comment out the other)
 // -----------------------------------------------------------------------------
@@ -100,7 +97,7 @@ function extract_freq(obj) {
 
 // Save Local
 // -----------------------------------------------------------------------------
-function save_local(jsonArray, fileName) {
+async function save_local(jsonArray, fileName) {
     let fileStr = JSON.stringify(jsonArray);    // convert json to string
     
     // write string to file
@@ -109,7 +106,8 @@ function save_local(jsonArray, fileName) {
             console.log("An error occured while writing keywords.json");
             return console.log(err);
         }
-        console.log("JSON file has been saved.");
+        console.log("JSON file", fileName, "has been saved.");
+
     });
 
     return true;
@@ -120,7 +118,7 @@ function save_local(jsonArray, fileName) {
 // -----------------------------------------------------------------------------
 async function upload_file( bucketName, srcFilePath, trgFilePath ) {
     // Creates a client
-    const storage = new Storage({projectId, keyFilename});
+    const storage = new Storage();
 
     try {
         await storage.bucket(bucketName).upload(srcFilePath, {
@@ -138,7 +136,6 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) {
 }
 
 
-
 // #5
 // MAIN function (exposed function to run the whole proccess)
 ////////////////////////////////////////////////////////////////////////////////
@@ -152,6 +149,8 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) {
 
 exports.main = () => {   // google cloud function entry point
 
+    console.log('temporary file:', temp_file);
+
     // connect;
     // get records to proccess;
     // call main proccess function; 
@@ -176,7 +175,9 @@ exports.main = () => {   // google cloud function entry point
         WHERE pl.active = 1 AND pl.sap_code IS NOT NULL AND pl.product_category_sap_4 NOT LIKE '72%'\
         GROUP BY pl.product_id\
         ORDER BY FREQuency DESC\
-        LIMIT 60000";
+        LIMIT 50000";
+    
+    console.log('Frequencies SQL:', sql)
 
     // query sql
     con.query(sql, function (err, result) {
@@ -186,21 +187,13 @@ exports.main = () => {   // google cloud function entry point
         extract_freq(result);    // proccess
         console.log('Keywords proccesed!')
 
-        save_local(freq_, temp_file);        // save frequencies
-
-        upload_file('pythia-files', temp_file, 'uploads/json/freq.json')
+        save_local(freq_, temp_file)        // save frequencies
         .then( () => {
-            console.log('Results saved! Exiting...');
-            process.exit(1);
+            upload_file('pythia-files', temp_file, 'uploads/json/freq.json')
+            .then( () => {
+                console.log('Results saved! Exiting...');
+            });
         });
-
-        /// // echo memory stats
-        /// const used = process.memoryUsage();
-        /// for (let key in used) {
-        ///     console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
-        /// }
-
-        // process.exit(1);        // exit
+            
     });
 }
-
diff --git a/package-lock.json b/package-lock.json
index 8fc45f9..9252abe 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -5,6 +5,7 @@
   "packages": {
     "": {
       "dependencies": {
+        "dotenv": "^16.0.3",
         "mysql": "^2.18.1",
         "npm": "^9.6.2"
       }
@@ -22,6 +23,14 @@
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
       "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
     },
+    "node_modules/dotenv": {
+      "version": "16.0.3",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz",
+      "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
diff --git a/package.json b/package.json
index 0ceff80..6dd37c3 100644
--- a/package.json
+++ b/package.json
@@ -1,5 +1,6 @@
 {
   "dependencies": {
+    "dotenv": "^16.0.3",
     "mysql": "^2.18.1",
     "npm": "^9.6.2"
   }