diff options
Diffstat (limited to 'javascript/keygen.js')
| -rw-r--r-- | javascript/keygen.js | 278 |
1 files changed, 175 insertions, 103 deletions
diff --git a/javascript/keygen.js b/javascript/keygen.js index 16e2490..39174b1 100644 --- a/javascript/keygen.js +++ b/javascript/keygen.js @@ -1,4 +1,19 @@ -// requirements +/** keygen + * + * this script constucts a linked-wordkeys structure + * ----------------------------------------------------------------------------- + * + * Contents: + * #1 Requirements + * #2.1 Personalized constants and parametres + * #2.2 Preloaded Data + * #3 Supporting functions + * #4 Output functions + * #5 Entry-point function main() + */ + +// #1 +// REQUIREMENTS //////////////////////////////////////////////////////////////////////////////// const fs = require('fs'); @@ -7,17 +22,45 @@ const fetch = require('node-fetch'); const { ECDH } = require('crypto'); const {Storage} = require('@google-cloud/storage'); // Google Cloud Storage -// (setup/personalize) constants + + +// #2.1 +// SETUP PERSONALIZED CONSTANTS AND PARAMETRES //////////////////////////////////////////////////////////////////////////////// + +// get products FROM api/endpoint parametres +const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json'; +const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch"; +const request_settings = { method: "Get" }; + + // google-storage parametres // --- -- -- - - - const projectId = 'pythia-251711'; const keyFilename = '../auth/pythia-251711-047e3d5e6608.json'; +// temp files +// need to be created prior to save-to-google-storage proccess +// because upload_file() uploats an existing local file to G-Storage +// --- -- -- - - - +const temp_kw_file = '../results/keywords.json'; +const temp_prod_file = '../results/products.json'; + + +// Global Variables +// ----------------------------------------------------------------------------- + +// keyword links (word-links dictionary; array of objects) +// ----------------------------------------------------------------------------- +var kwlinks_ = []; ////////////////////////////// MAIN OUTPUT OF THE SCRIPT +var prods_ = []; +var fr_ =[]; + -// preloaded data +// #2.2 +// PRELOADED DATA //////////////////////////////////////////////////////////////////////////////// // any-character to keyboard-latin mapping @@ -228,10 +271,19 @@ removeOriginals = 'κατά παρά υπό από μετά προς μας με removeOriginals.forEach( w => { removeList.push( kb_trans(w)); }); -// read frequency data -let rawdata = fs.readFileSync('../results/freq.json'); -let fr_ = JSON.parse(rawdata); // frequencies -console.log('frequencies loaded'); +// depricated +/// // read frequency (local) data +/// let rawdata = fs.readFileSync('../results/freq.json'); +/// let fr_ = JSON.parse(rawdata); // frequencies +/// console.log('frequencies loaded'); + + + + +// #3 +// SUPPORTING FUNCTIONS +//////////////////////////////////////////////////////////////////////////////// + /** freq * @@ -247,57 +299,15 @@ function freq_of(id) { } -// get products from api/endpoint (in Json format) -// --- -- -- - - - -let url = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch"; - -let settings = { method: "Get" }; - -fetch(url, settings) - .then(res => res.json()) - .then((json) => { - - let temp_file = '../results/keywords.json'; - let temp_prod_file = '../results/products.json'; - - console.log('products data loaded from emarket api/endpoint'); - - // do something with JSON - var linkeys = do_proccess(json.data); - - save_local(kwlinks_, temp_kw_file); - save_local(kwlinks_, temp_prod_file); - - // upload_file('pythia-files', os.tmpdir()+'/keywords.json', 'uploads/orders/keywords.json'); - upload_file('pythia-files', temp_kw_file, 'uploads/orders/emarket-keywords.json'); - upload_file('pythia-files', temp_prod_file, 'uploads/orders/emarket-keywords.json'); - - // check - // echo 100 most frequent - let i = 0; - kwlinks_.forEach(rec => { - if (i<100) { - console.log(i, JSON.stringify(rec.w)) - } - i++; - }) -}); - - - -// keyword links (word-links dictionary; array of objects) -// ----------------------------------------------------------------------------- -var kwlinks_ = []; ////////////////////////////// MAIN OUTPUT OF THE SCRIPT -var prods_ = []; +/* --- // connect; // get records to proccess; // call main proccess function; // save dictionary; // end script; // ----------------------------------------------------------------------------- - -exports.main = () => { +var main = () => { con.connect(function(err) { // connect; @@ -340,13 +350,13 @@ exports.main = () => { }); }); } +--- */ -var unmatched = 0; - -// main proccess +// proccess data +// The MAIN proccess // ----------------------------------------------------------------------------- -function do_proccess(obj) { +function extract_linked_keywords(obj) { // console.log(JSON.stringify(obj, null, 2)); obj.forEach( rec => { @@ -362,8 +372,6 @@ function do_proccess(obj) { w: description }) - // if (fq == 0) console.log(pid, description); - var keys = []; var words = description.split(' '); @@ -418,52 +426,6 @@ function do_proccess(obj) { } - -// save proccesses -//////////////////////////////////////////////////////////////////////////////// - -// Save Local -// ----------------------------------------------------------------------------- -function save_local(jsonArray, filePath) { - let fileStr = JSON.stringify(jsonArray); // convert json to string - - // write string to file - fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { - if (err) { - console.log("An error occured while writing keywords.json"); - return console.log(err); - } - console.log("JSON file has been saved."); - }); - - return true; -} - - - -// Save to Cloud Storage -// ----------------------------------------------------------------------------- - - -async function upload_file( bucketName, srcFilePath, trgFilePath ) { - // Creates a client - const storage = new Storage({projectId, keyFilename}); - - try { - await storage.bucket(bucketName).upload(srcFilePath, { - destination: trgFilePath, - gzip: true, // serve compressed - metadata: { // cache for 8 hours - cacheControl: 'public, max-age=60' // production set: 28800 - } - }); - console.log(`${srcFilePath} uploaded to ${bucketName}`); - } - catch(err) { - console.error('ERROR:', err); - } -} - // functions for linking words in keywords dictionary //////////////////////////////////////////////////////////////////////////////// @@ -620,3 +582,113 @@ function mark_link(lws, source) { else return source; } + + +// #4 +// OUTPUT FUNCTIONS +//////////////////////////////////////////////////////////////////////////////// + + +// Save Local +// ----------------------------------------------------------------------------- +function save_local(jsonArray, filePath) { + let fileStr = JSON.stringify(jsonArray); // convert json to string + + // write string to file + fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { + if (err) { + console.log("An error occured while writing keywords.json"); + return console.log(err); + } + // console.log("JSON file has been saved."); + }); + + return true; +} + + + +// Save to Google Cloud Storage +// ----------------------------------------------------------------------------- +async function upload_file( bucketName, srcFilePath, trgFilePath ) { + // Creates a client + const storage = new Storage({projectId, keyFilename}); + + try { + await storage.bucket(bucketName).upload(srcFilePath, { + destination: trgFilePath, + gzip: true, // serve compressed + metadata: { // cache for 8 hours + cacheControl: 'public, max-age=60' // production set: 28800 + } + }); + console.log(`${srcFilePath} uploaded to ${bucketName}`); + } + + catch(err) { + console.error('ERROR:', err); + } + +} + + + + +// #5 +// MAIN function (exposed function to run the whole proccess) +//////////////////////////////////////////////////////////////////////////////// + + +/** main() + * + * nodejs's exported function + * used as entry-point function (in case of Google Cloud function) + */ + +// keep only 1 of next 2 lines +// exports.main = () => { // entry point functioon when google cloud function +var main = () => { // concise function when direct script + + fetch(frequency_endpoint, request_settings) + .then(res => res.json()) + .then((json) => { + fr_ = json; + console.log('frequencies loaded from CDN'); + + fetch(products_endpoint, request_settings) + .then(res => res.json()) + .then((json) => { + + console.log('products data loaded from emarket api/endpoint'); + + // proccess data + extract_linked_keywords(json.data); + + save_local(kwlinks_, temp_kw_file); + save_local(kwlinks_, temp_prod_file); + + // upload temp files to cloud + upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'); + upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json'); + + /// // check + /// // echo 100 most frequent + /// let i = 0; + /// kwlinks_.forEach(rec => { + /// if (i<100) { + /// console.log(i, JSON.stringify(rec.w)) + /// } + /// i++; + /// }) + }); + + }); + +} + + +// NOTE: +// if running the script directry you need to call the main function +// if running through google cloud-functions you do NOT need to call main() +// (you need define main() as the entry-point function instead) +main();
\ No newline at end of file |