diff options
| -rw-r--r-- | javascript/freq.js | 6 | ||||
| -rw-r--r-- | javascript/keygen.js | 152 | ||||
| -rw-r--r-- | javascript/keygen_GCF.js | 13 | ||||
| -rw-r--r-- | package-lock.json | 6 | ||||
| -rw-r--r-- | package.json | 1 |
5 files changed, 114 insertions, 64 deletions
diff --git a/javascript/freq.js b/javascript/freq.js index 86bba22..bf52008 100644 --- a/javascript/freq.js +++ b/javascript/freq.js @@ -19,13 +19,9 @@ // REQUIREMENTS //////////////////////////////////////////////////////////////////////////////// - var mysql = require('mysql'); - const fs = require('fs'); - const os = require('os'); - const {Storage} = require('@google-cloud/storage'); // import Google Cloud client library require('dotenv').config({ path: '../auth/.env' }) @@ -166,7 +162,7 @@ var main = () => { console.log("Connected!"); // SQL to get needed data - var sql = "SELECT count(pl.eys_code) as FREQuency,\ + var sql = "SELECT count(pl.eys_code)*5+SUM(dop.quantity) as FREQuency,\ pl.product_id as product_id,\ pb.brand_name,\ pl.barcode, pl.skl_code, pl.eys_code,\ diff --git a/javascript/keygen.js b/javascript/keygen.js index 59407a9..dbd5717 100644 --- a/javascript/keygen.js +++ b/javascript/keygen.js @@ -19,7 +19,6 @@ const fs = require('fs'); const os = require('os'); const fetch = require('node-fetch'); -const { ECDH } = require('crypto'); const {Storage} = require('@google-cloud/storage'); // Google Cloud Storage @@ -52,11 +51,22 @@ const temp_prod_file = '../results/products.json'; // Global Variables // ----------------------------------------------------------------------------- +// Global arrays // keyword links (word-links dictionary; array of objects) // ----------------------------------------------------------------------------- -var kwlinks_ = []; ////////////////////////////// MAIN OUTPUT OF THE SCRIPT -var prods_ = []; -var fr_ =[]; +var kwlinks_ = []; // keyword-links structure (structure to construct) +var prods_ = []; // products (array to construct) +var _fr =[]; // product frequencies array (as read from endpiont) +var _pr =[]; // product array (as read from endpiont) + + +// Console Colors +// ----------------------------------------------------------------------------- +const Reset = "\x1b[0m"; +const FgRed = "\x1b[31m"; +const FgGreen = "\x1b[32m"; + + // #2.2 @@ -178,7 +188,10 @@ noRoot = [ 'Ολικής-Άλεσης', 'Ολικής-Aλέσεως', 'Ολικής', + 'Σοκολάτας', + 'Υγείας', 'Γαϊδούρας', + 'Δημητριακών', 'Γαϊδάρου', 'Ρούχων', 'Πιάτων', @@ -209,6 +222,9 @@ noRoot = [ 'Καλαθάκι', 'Σκύλου', 'Γάτας', + 'ΠΓΕ', + 'ΤΟ', + 'Μελί', 'Σώματος', 'Αδυνατίσματος', 'Προστασίας', @@ -271,13 +287,6 @@ removeOriginals = 'κατά παρά υπό από μετά προς μας με removeOriginals.forEach( w => { removeList.push( kb_trans(w)); }); -// depricated -/// // read frequency (local) data -/// let rawdata = fs.readFileSync('../results/freq.json'); -/// let fr_ = JSON.parse(rawdata); // frequencies -/// console.log('frequencies loaded'); - - // #3 @@ -291,7 +300,7 @@ removeOriginals.forEach( w => { removeList.push( kb_trans(w)); }); * @return frequency of product */ function freq_of(id) { - fr_.forEach( rec => { + _fr.forEach( rec => { if (rec.id == id) return rec.fq }); @@ -553,8 +562,8 @@ function do_replaces(str) { // preproccess description // --- function preproccess_text(str) { - str = do_replaces(str); str = clean_text(str); + str = do_replaces(str); str = mark_linked_words(str); return str; } @@ -589,21 +598,24 @@ function mark_link(lws, source) { //////////////////////////////////////////////////////////////////////////////// + // Save Local // ----------------------------------------------------------------------------- -function save_local(jsonArray, filePath) { +async function save_local(jsonArray, filePath) { let fileStr = JSON.stringify(jsonArray); // convert json to string - - // write string to file - fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { - if (err) { - console.log("An error occured while writing keywords.json"); - return console.log(err); - } - // console.log("JSON file has been saved."); - }); - return true; + // return new Promise(function(resolve, reject) { + + // write string to file + fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { + if (err) { + return err; + } else { + return true; + } + }); + + // }); } @@ -649,43 +661,79 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) { // exports.main = () => { // entry point functioon when google cloud function var main = () => { // concise function when direct script - fetch(frequency_endpoint, request_settings) - .then(res => res.json()) - .then((json) => { - fr_ = json; - console.log('frequencies loaded from CDN'); + // PROMISE ... + // fetch all jsons from endpints ........................................... + Promise.all([ fetch(products_endpoint, request_settings) - .then(res => res.json()) - .then((json) => { + .then(res => res.json()) + .then((json) => { + _pr = json; + console.log('products data loaded from emarket api/endpoint'); + }), + + fetch(frequency_endpoint, request_settings) + .then(res => res.json()) + .then((json) => { + _fr = json; + console.log('frequencies loaded from CDN') + }) + + ]) + .then (() => { // then... + // proccess data and extract ketwords + + console.log(`... All data from endpoints: ${FgGreen}LOADED!${Reset}`); + + // proccess data + extract_linked_keywords(_pr.data); + + console.log('keywords extracted;\ntry to save results localy...'); - console.log('products data loaded from emarket api/endpoint'); + // PROMISE ... + // save local files .................................................... + Promise.all([ + save_local(kwlinks_, temp_kw_file) + .then(() => { console.log('keywords saced localy');}), - // proccess data - extract_linked_keywords(json.data); + save_local(prods_, temp_prod_file) + .then(() => { console.log('products saved localy');}) + ]) + .then(() => { // then ... - save_local(kwlinks_, temp_kw_file); - save_local(kwlinks_, temp_prod_file); + console.log(`... All local saves: ${FgGreen}DONE!${Reset}`); - // upload temp files to cloud - upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'); - upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json'); + // PROMISE ... + // upload keytword and product jsons into Cloud Storage ............ + Promise.all([ + upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'), + upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json') + ]) + .then(() => { // then ... + + console.log(`... Uploads to cloud storage: ${FgGreen}DONE!${Reset}`); + + // echo usage stats; then END + + // check + // echo 100 most frequent + let i = 0; + kwlinks_.forEach(rec => { + if (i<100) { + console.log(i, JSON.stringify(rec.w)) + } + i++; + }); - // check - // echo 100 most frequent - let i = 0; - kwlinks_.forEach(rec => { - if (i<100) { - console.log(i, JSON.stringify(rec.w)) + // echo memory stats + const used = process.memoryUsage(); + for (let key in used) { + console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`); } - i++; - }); - // echo memory stats - const used = process.memoryUsage(); - for (let key in used) { - console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`); - } + console.log(`${FgGreen}THE END!${Reset}`); + + }); }); diff --git a/javascript/keygen_GCF.js b/javascript/keygen_GCF.js index e6568ff..1e22f18 100644 --- a/javascript/keygen_GCF.js +++ b/javascript/keygen_GCF.js @@ -524,23 +524,23 @@ function mark_link(lws, source) { // Save Local // ----------------------------------------------------------------------------- -function save_local(jsonArray, filePath) { +async function save_local(jsonArray, fileName) { let fileStr = JSON.stringify(jsonArray); // convert json to string // write string to file - fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { + fs.writeFileSync(fileName, fileStr, 'utf8', (err) => { if (err) { console.log("An error occured while writing keywords.json"); return console.log(err); } - // console.log("JSON file has been saved."); + console.log("JSON file", fileName, "has been saved."); + }); return true; } - // Save to Google Cloud Storage // ----------------------------------------------------------------------------- async function upload_file( bucketName, srcFilePath, trgFilePath ) { @@ -579,8 +579,8 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) { */ // keep only 1 of next 2 lines -// exports.main = () => { // entry point functioon when google cloud function -var main = () => { // concise function when direct script +exports.main = () => { // entry point functioon when google cloud function + fetch(frequency_endpoint, request_settings) .then(res => res.json()) @@ -604,7 +604,6 @@ var main = () => { // concise function when direct script upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'); upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json'); - }); }); diff --git a/package-lock.json b/package-lock.json index 9252abe..ea49282 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "packages": { "": { "dependencies": { + "bluebird": "^3.7.2", "dotenv": "^16.0.3", "mysql": "^2.18.1", "npm": "^9.6.2" @@ -18,6 +19,11 @@ "node": "*" } }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", diff --git a/package.json b/package.json index 6dd37c3..fb14b33 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "bluebird": "^3.7.2", "dotenv": "^16.0.3", "mysql": "^2.18.1", "npm": "^9.6.2" |