linkeysearch: 2 step implementation dtails; shall run as google funcction

1 files changed, 103 insertions, 97 deletions

diff --git a/javascript/keygen.js b/javascript/keygen.js
index 57b90ee..16e2490 100644
--- a/javascript/keygen.js
+++ b/javascript/keygen.js
@@ -2,11 +2,19 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 const fs = require('fs');
-
 const os = require('os');
-
 const fetch = require('node-fetch');
 const { ECDH } = require('crypto');
+const {Storage} = require('@google-cloud/storage');     // Google Cloud Storage
+
+// (setup/personalize) constants
+////////////////////////////////////////////////////////////////////////////////
+
+// google-storage parametres
+// --- -- -- - - -
+const projectId = 'pythia-251711';
+const keyFilename = '../auth/pythia-251711-047e3d5e6608.json';
+
 
 
 // preloaded data
@@ -22,6 +30,7 @@ for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
 
 // synonyms
 // -----------------------------------------------------------------------------
+
 var synonyms = [];      // groups of synonyms
 var synonym_kbs = [];   // cache kb-formats for performance
 var synonyms_Originals = [
@@ -86,8 +95,10 @@ replaceSource = [
     'ΕΝΕΛΛΑΔ ;Εν-Ελλάδι ',
     'ΓΑΛΟΠΟΥΛ ;Γαλοπούλα ',
     '7 DAYS ;7DAYS ',
-    'ΜΠΑΡΜΠΑ ΣΤΑΘΗ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ '
-]
+    'ΜΠΑΡΜΠΑ ΣΤΑΘΗ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ ',
+    'Zero% ;Zero ',
+    
+];
 replaceSource.forEach( it => {
     st = it.split(';');
     replaces.push({ src: st[0], trg: st[1] });
@@ -147,9 +158,24 @@ noRoot = [
     'Ελάτου',
     'Άγριων',
     'Βοτάνων',
+    'Κατσαρίδες',
     'Λακωνίας',
-    'ΠΑΡΑΓΓΕΛΙΩΝ'
-]
+    'ΠΑΡΑΓΓΕΛΙΩΝ',
+    'Λήμνου',
+    'Αργινίνης',
+    'Καλαθάκι',
+    'Σκύλου',
+    'Γάτας',
+    'Σώματος',
+    'Αδυνατίσματος',
+    'Προστασίας',
+    'Ακράτειας',
+    'Προσώπου',
+    'προσθήκη',
+    'Βρεφικής',
+    'Καθαρισμού',
+    'Ρούχων'
+];
 noRoot.forEach( w => { noRootKeywords.push(kb_trans(w)); });
 
 
@@ -191,42 +217,21 @@ linkedWords = [
     'Ολες-τις-Χρήσεις',
     'Το-Μάννα',
     'Χωρίς-προσθήκη-ζάχαρης'
-]
+];
 
 
 // list of words to exclude from keywords
 // -----------------------------------------------------------------------------
 // NOTE: APPLIED in PER-WORD base -> after spliting description to words
 removeList = []
-removeOriginals = 'μας με σε για του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ')
+removeOriginals = 'κατά παρά υπό από μετά προς μας με σε για του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ')
 removeOriginals.forEach( w => { removeList.push( kb_trans(w)); });
 
 
-
-
-
-/*
-// ready to get main data to preccess
-////////////////////////////////////////////////////////////////////////////////
-
-// db connection parametres
-// -----------------------------------------------------------------------------
-var con = mysql.createConnection({  
-  // host: "/cloudsql/pythia-251711:europe-west4:pythia-db-eu",
-  socketPath: "/cloudsql/pythia-251711:europe-west4:pythia-db-eu",
-  user: "pythia_services",
-  password: process.env.DB_PASSWORD,
-  database: "pythia_db"
-});
-
-*/
-
-
 // read frequency data
-
 let rawdata = fs.readFileSync('../results/freq.json');
-let fr_ = JSON.parse(rawdata);
-
+let fr_ = JSON.parse(rawdata);      // frequencies
+console.log('frequencies loaded');
 
 /** freq
  * 
@@ -242,7 +247,8 @@ function freq_of(id) {
 }
 
 
-
+// get products from api/endpoint (in Json format)
+// --- -- -- - - -
 let url = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch";
 
 let settings = { method: "Get" };
@@ -251,12 +257,20 @@ fetch(url, settings)
     .then(res => res.json())
     .then((json) => {
 
-        console.log('data loaded');
+        let temp_file = '../results/keywords.json';
+        let temp_prod_file = '../results/products.json';
+
+        console.log('products data loaded from emarket api/endpoint');
 
         // do something with JSON
         var linkeys = do_proccess(json.data);
 
-        save_local(kwlinks_);
+        save_local(kwlinks_, temp_kw_file);
+        save_local(kwlinks_, temp_prod_file);
+
+        // upload_file('pythia-files', os.tmpdir()+'/keywords.json', 'uploads/orders/keywords.json');
+        upload_file('pythia-files', temp_kw_file, 'uploads/orders/emarket-keywords.json');
+        upload_file('pythia-files', temp_prod_file, 'uploads/orders/emarket-keywords.json');
 
         // check
         // echo 100 most frequent
@@ -274,7 +288,7 @@ fetch(url, settings)
 // keyword links (word-links dictionary; array of objects)
 // -----------------------------------------------------------------------------
 var kwlinks_ = [];      ////////////////////////////// MAIN OUTPUT OF THE SCRIPT
-
+var prods_ = [];
 
 // connect;
 // get records to proccess;
@@ -308,14 +322,15 @@ exports.main = () => {
         // query sql
         con.query(sql, function (err, result) {
             if (err) throw err;
-            console.log('Records from database received!')
+            console.log('Records from database received!');
     
             do_proccess(result);    // proccess
-            console.log('Keywords proccesed!')
+            console.log('Keywords proccesed!');
     
-            save_keywords();        // save
-            upload_file('pythia-files', os.tmpdir()+'/keywords.json', 'uploads/orders/keywords.json');
-            console.log('Results saved! Exiting.')
+            save_local(kwlinks_, '../results/keywords.json');        // save
+            // production (GFunctions): upload_file('pythia-files', os.tmpdir()+'/keywords.json', 'uploads/orders/keywords.json');
+            upload_file('pythia-files', '../results/keywords.json', 'uploads/orders/emarket-keywords.json');
+            console.log('Results saved! Exiting.');
     
             // echo memory stats
             const used = process.memoryUsage();
@@ -327,42 +342,7 @@ exports.main = () => {
 }
 
 
-
-// Google Cloud Functions
-////////////////////////////////////////////////////////////////////////////////
-function upload_file( bucketName, filePath, destFileName ) {
-    // [START storage_upload_file]
-    
-    // Sample code
-    // const bucketName = 'your-unique-bucket-name';    // The ID of your GCS bucket
-    // const filePath = 'path/to/your/file';            // The path to your file to upload
-    // const destFileName = 'your-new-file-name';       // The new ID for your GCS file
-  
-    // Imports the Google Cloud client library
-    const {Storage} = require('@google-cloud/storage');
-  
-    // Creates a client
-    const storage = new Storage();
-  
-    async function uploadFile() {
-        await storage.bucket(bucketName).upload(filePath, {
-            destination: destFileName, 
-            gzip: true,     // serve compressed
-            metadata: {     // cache for 8 hours
-                cacheControl: 'public, max-age=28800',
-            }
-        });
-
-        console.log(`${filePath} uploaded to ${bucketName}`);
-    }
-  
-    uploadFile().catch(console.error);
-    // [END storage_upload_file]
-}
-
-
-
-
+var unmatched = 0;
 
 // main proccess
 // -----------------------------------------------------------------------------
@@ -374,8 +354,15 @@ function do_proccess(obj) {
         if (rec.img != 0) {
 
             var description = preproccess_text(rec.txt);
-            var pid = rec.id;
-            var fq = freq_of(pid);
+            var pid = rec.eys;
+            var fq = freq_of(pid)+1;
+
+            prods_.push({
+                id: pid,
+                w: description
+            })
+
+            // if (fq == 0) console.log(pid, description);
 
             var keys = [];
             var words = description.split(' ');
@@ -431,38 +418,51 @@ function do_proccess(obj) {
 }
 
 
-// save proccess
-// -----------------------------------------------------------------------------
 
-function save_local(json) {  
-    // stringify JSON Object
-    var jsonContent = JSON.stringify(json);
+// save proccesses
+////////////////////////////////////////////////////////////////////////////////
+
+// Save Local
+// -----------------------------------------------------------------------------
+function save_local(jsonArray, filePath) {
+    let fileStr = JSON.stringify(jsonArray);    // convert json to string
     
-    fs.writeFile("../results/keywords-node.json", jsonContent, 'utf8', function (err) {
+    // write string to file
+    fs.writeFileSync(filePath, fileStr, 'utf8', (err) => {
         if (err) {
-            console.log("An error occured while writing JSON Object to File.");
+            console.log("An error occured while writing keywords.json");
             return console.log(err);
         }
-    
         console.log("JSON file has been saved.");
     });
+
+    return true;
 }
 
 
 
-function save_keywords() {
-    let jsonStr = JSON.stringify(kwlinks_);
-    // console.log(jsonStr);
+// Save to Cloud Storage
+// -----------------------------------------------------------------------------
 
-    fs.writeFileSync(os.tmpdir() + "/keywords.json", jsonStr, 'utf8', (err) => {
-        if (err) {
-            console.log("An error occured while writing keywords.json");
-            return console.log(err);
-        }
-        console.log("linkeys file saved.");
-    });
-}
 
+async function upload_file( bucketName, srcFilePath, trgFilePath ) {
+    // Creates a client
+    const storage = new Storage({projectId, keyFilename});
+
+    try {
+        await storage.bucket(bucketName).upload(srcFilePath, {
+            destination: trgFilePath, 
+            gzip: true,     // serve compressed
+            metadata: {     // cache for 8 hours
+                cacheControl: 'public, max-age=60'      // production set: 28800
+            }
+        });
+        console.log(`${srcFilePath} uploaded to ${bucketName}`);
+    }
+    catch(err) {
+        console.error('ERROR:', err);
+    }
+}
 
 // functions for linking words in keywords dictionary
 ////////////////////////////////////////////////////////////////////////////////
@@ -547,6 +547,7 @@ function kb_trans(str) {
     return out;
 }
 
+
 // clean text trims some characters (+.') and internal multiple-spaces
 // ---
 function clean_text(txt) {
@@ -555,6 +556,7 @@ function clean_text(txt) {
                 .replace('  ',' ');
 }
 
+
 // check if term is significant
 // (if not, the term will be excluded from keywords dicionary)
 // ---
@@ -564,6 +566,7 @@ function is_significant(str) {
     return !(/\d/.test(str));
 }
 
+
 // check if word: w
 // ...has synonyms; return list of synonyms
 // ---
@@ -576,6 +579,7 @@ function synonym_keys(w) {
     return [ w ];
 }
 
+
 // edit common mistakes
 // with suggested replaces
 function do_replaces(str) {
@@ -583,6 +587,7 @@ function do_replaces(str) {
     return str;
 }
 
+
 // preproccess description
 // ---
 function preproccess_text(str) {
@@ -601,6 +606,7 @@ function mark_linked_words(txt) {
     return txt
 }
 
+
 // mark a link (lws) to a text (source)
 // conecting them with a dash/minus character
 // ---