From e712800cc853e45c6365b3e3f0c5c7db6f9eb4b3 Mon Sep 17 00:00:00 2001 From: George Halkiadakis Date: Thu, 23 Mar 2023 22:35:50 +0200 Subject: freq_GCF: freq implementation as Google Cloud Function; added security options --- .gitignore | 5 +++- javascript/freq.js | 66 +++++++++++++++++++------------------------------- javascript/freq_GCF.js | 39 ++++++++++++----------------- package-lock.json | 9 +++++++ package.json | 1 + 5 files changed, 55 insertions(+), 65 deletions(-) diff --git a/.gitignore b/.gitignore index 9ad17eb..b41c06d 100644 --- a/.gitignore +++ b/.gitignore @@ -54,4 +54,7 @@ Thumbs.db app/public/temp/ # authentication files (json) for Google Cloud Services -auth/*.json \ No newline at end of file +auth/*.json + +# environmental variable files +.env \ No newline at end of file diff --git a/javascript/freq.js b/javascript/freq.js index 19c42f5..0bbdb5b 100644 --- a/javascript/freq.js +++ b/javascript/freq.js @@ -9,6 +9,10 @@ * #3 Supporting functions * #4 Output functions * #5 Entry-point function main() + * + * Requirements + * npm install dotenv --save + * */ // #1 @@ -24,7 +28,7 @@ const os = require('os'); const {Storage} = require('@google-cloud/storage'); // import Google Cloud client library - +require('dotenv').config({ path: '../auth/.env' }) // #2 @@ -38,19 +42,9 @@ const projectId = 'pythia-251711'; const keyFilename = '../auth/pythia-251711-047e3d5e6608.json'; -// cloud_environment (boolean) -// set value to: -// true : if running on Google Cloud Functions -// false : if running localy -// --- -- -- - - - -const cloud_environment = false; // ** check - - // temp file of frequencies (json) -const saves_dir = '../results'; // ** check -const temp_file = 'freq.json'; // ** check +const temp_file = '../results/freq.json'; -const local_dir = (cloud_environment) ? os.tmpdir() : saves_dir; // db connection parametres @@ -58,20 +52,13 @@ const local_dir = (cloud_environment) ? os.tmpdir() : saves_dir; // ----------------------------------------------------------------------------- // while on local machine using cloud_sql_proxy -var con = mysql.createConnection({ // ** check - host: "127.0.0.1", - user: "pythia_db_user_dev", - password: "VnEP0eysjiXDHcfM", - database: "dev_pythia_db" +var con = mysql.createConnection({ + host: process.env.DB_HOST; + user: process.env.DB_USER; + password: process.env.DB_PASS; + database: process.env.DB_NAME; }); -// // while on Google Cloud Functions -// var con = mysql.createConnection({ // ** check -// socketPath: "/cloudsql/pythia-251711:europe-west4:pythia-db-eu", -// user: "pythia_services", -// password: process.env.DB_PASSWORD, -// database: "pythia_db" -// }); // MAIN OUTPUT OF THE SCRIPT // arrays that need to be constructed and filled with data @@ -163,11 +150,8 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) { /** main() * - * nodejs's exported function - * used as entry-point function (in case of Google Cloud function) + * the entry-point function */ - -// exports.main = () => { // google cloud function entry point var main = () => { // connect; @@ -176,12 +160,12 @@ var main = () => { // save dictionary; // end script; // ----------------------------------------------------------------------------- - con.connect(function(err) { // comment this out for 'Cannot enqueue Handshake after already enqueuing' error - // connect; // comment this out for 'Cannot enqueue Handshake after already enqueuing' error - if (err) throw err; // comment this out for 'Cannot enqueue Handshake after already enqueuing' error - console.log("Connected!"); // comment this out for 'Cannot enqueue Handshake after already enqueuing' error + con.connect(function(err) { + // connect; + if (err) throw err; + console.log("Connected!"); - // SQL to get needed data + // SQL to get needed data var sql = "SELECT count(pl.eys_code) as FREQuency,\ pl.product_id as product_id,\ pb.brand_name,\ @@ -216,15 +200,15 @@ var main = () => { process.exit(1); }); - /// // echo memory stats - /// const used = process.memoryUsage(); - /// for (let key in used) { - /// console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`); - /// } + // echo memory stats + const used = process.memoryUsage(); + for (let key in used) { + console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`); + } - // process.exit(1); // exit + // process.exit(); // exit }); - }); // comment this out for 'Cannot enqueue Handshake after already enqueuing' error + }); } // NOTE: @@ -232,4 +216,4 @@ var main = () => { // if running through google cloud-functions you do NOT need to call main() // (you need define main() as the entry-point function instead) // --- -- -- - - - -main(); // ** check \ No newline at end of file +main(); \ No newline at end of file diff --git a/javascript/freq_GCF.js b/javascript/freq_GCF.js index 79a79bc..79810e6 100644 --- a/javascript/freq_GCF.js +++ b/javascript/freq_GCF.js @@ -1,5 +1,6 @@ /** freq.js * + * Google Cloud Functions impementation of `linkeysearch:javascript/freq` * this script extracts products' order-frequency * ----------------------------------------------------------------------------- * @@ -17,11 +18,8 @@ var mysql = require('mysql'); - const fs = require('fs'); - const os = require('os'); - const {Storage} = require('@google-cloud/storage'); // import Google Cloud client library @@ -42,7 +40,6 @@ const projectId = 'pythia-251711'; const temp_file = os.tmpdir() + '/freq.json'; - // db connection parametres // (keep the one that suits your environment; comment out the other) // ----------------------------------------------------------------------------- @@ -100,7 +97,7 @@ function extract_freq(obj) { // Save Local // ----------------------------------------------------------------------------- -function save_local(jsonArray, fileName) { +async function save_local(jsonArray, fileName) { let fileStr = JSON.stringify(jsonArray); // convert json to string // write string to file @@ -109,7 +106,8 @@ function save_local(jsonArray, fileName) { console.log("An error occured while writing keywords.json"); return console.log(err); } - console.log("JSON file has been saved."); + console.log("JSON file", fileName, "has been saved."); + }); return true; @@ -120,7 +118,7 @@ function save_local(jsonArray, fileName) { // ----------------------------------------------------------------------------- async function upload_file( bucketName, srcFilePath, trgFilePath ) { // Creates a client - const storage = new Storage({projectId, keyFilename}); + const storage = new Storage(); try { await storage.bucket(bucketName).upload(srcFilePath, { @@ -138,7 +136,6 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) { } - // #5 // MAIN function (exposed function to run the whole proccess) //////////////////////////////////////////////////////////////////////////////// @@ -152,6 +149,8 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) { exports.main = () => { // google cloud function entry point + console.log('temporary file:', temp_file); + // connect; // get records to proccess; // call main proccess function; @@ -176,7 +175,9 @@ exports.main = () => { // google cloud function entry point WHERE pl.active = 1 AND pl.sap_code IS NOT NULL AND pl.product_category_sap_4 NOT LIKE '72%'\ GROUP BY pl.product_id\ ORDER BY FREQuency DESC\ - LIMIT 60000"; + LIMIT 50000"; + + console.log('Frequencies SQL:', sql) // query sql con.query(sql, function (err, result) { @@ -186,21 +187,13 @@ exports.main = () => { // google cloud function entry point extract_freq(result); // proccess console.log('Keywords proccesed!') - save_local(freq_, temp_file); // save frequencies - - upload_file('pythia-files', temp_file, 'uploads/json/freq.json') + save_local(freq_, temp_file) // save frequencies .then( () => { - console.log('Results saved! Exiting...'); - process.exit(1); + upload_file('pythia-files', temp_file, 'uploads/json/freq.json') + .then( () => { + console.log('Results saved! Exiting...'); + }); }); - - /// // echo memory stats - /// const used = process.memoryUsage(); - /// for (let key in used) { - /// console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`); - /// } - - // process.exit(1); // exit + }); } - diff --git a/package-lock.json b/package-lock.json index 8fc45f9..9252abe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,6 +5,7 @@ "packages": { "": { "dependencies": { + "dotenv": "^16.0.3", "mysql": "^2.18.1", "npm": "^9.6.2" } @@ -22,6 +23,14 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, + "node_modules/dotenv": { + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz", + "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==", + "engines": { + "node": ">=12" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", diff --git a/package.json b/package.json index 0ceff80..6dd37c3 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "dotenv": "^16.0.3", "mysql": "^2.18.1", "npm": "^9.6.2" } -- cgit v1.2.3