diff options
Diffstat (limited to 'javascript/keygen.js')
| -rw-r--r-- | javascript/keygen.js | 200 |
1 files changed, 103 insertions, 97 deletions
diff --git a/javascript/keygen.js b/javascript/keygen.js index 57b90ee..16e2490 100644 --- a/javascript/keygen.js +++ b/javascript/keygen.js @@ -2,11 +2,19 @@ //////////////////////////////////////////////////////////////////////////////// const fs = require('fs'); - const os = require('os'); - const fetch = require('node-fetch'); const { ECDH } = require('crypto'); +const {Storage} = require('@google-cloud/storage'); // Google Cloud Storage + +// (setup/personalize) constants +//////////////////////////////////////////////////////////////////////////////// + +// google-storage parametres +// --- -- -- - - - +const projectId = 'pythia-251711'; +const keyFilename = '../auth/pythia-251711-047e3d5e6608.json'; + // preloaded data @@ -22,6 +30,7 @@ for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]); // synonyms // ----------------------------------------------------------------------------- + var synonyms = []; // groups of synonyms var synonym_kbs = []; // cache kb-formats for performance var synonyms_Originals = [ @@ -86,8 +95,10 @@ replaceSource = [ 'ΕΝΕΛΛΑΔ ;Εν-Ελλάδι ', 'ΓΑΛΟΠΟΥΛ ;Γαλοπούλα ', '7 DAYS ;7DAYS ', - 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ ' -] + 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ ', + 'Zero% ;Zero ', + +]; replaceSource.forEach( it => { st = it.split(';'); replaces.push({ src: st[0], trg: st[1] }); @@ -147,9 +158,24 @@ noRoot = [ 'Ελάτου', 'Άγριων', 'Βοτάνων', + 'Κατσαρίδες', 'Λακωνίας', - 'ΠΑΡΑΓΓΕΛΙΩΝ' -] + 'ΠΑΡΑΓΓΕΛΙΩΝ', + 'Λήμνου', + 'Αργινίνης', + 'Καλαθάκι', + 'Σκύλου', + 'Γάτας', + 'Σώματος', + 'Αδυνατίσματος', + 'Προστασίας', + 'Ακράτειας', + 'Προσώπου', + 'προσθήκη', + 'Βρεφικής', + 'Καθαρισμού', + 'Ρούχων' +]; noRoot.forEach( w => { noRootKeywords.push(kb_trans(w)); }); @@ -191,42 +217,21 @@ linkedWords = [ 'Ολες-τις-Χρήσεις', 'Το-Μάννα', 'Χωρίς-προσθήκη-ζάχαρης' -] +]; // list of words to exclude from keywords // ----------------------------------------------------------------------------- // NOTE: APPLIED in PER-WORD base -> after spliting description to words removeList = [] -removeOriginals = 'μας με σε για του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ') +removeOriginals = 'κατά παρά υπό από μετά προς μας με σε για του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ') removeOriginals.forEach( w => { removeList.push( kb_trans(w)); }); - - - -/* -// ready to get main data to preccess -//////////////////////////////////////////////////////////////////////////////// - -// db connection parametres -// ----------------------------------------------------------------------------- -var con = mysql.createConnection({ - // host: "/cloudsql/pythia-251711:europe-west4:pythia-db-eu", - socketPath: "/cloudsql/pythia-251711:europe-west4:pythia-db-eu", - user: "pythia_services", - password: process.env.DB_PASSWORD, - database: "pythia_db" -}); - -*/ - - // read frequency data - let rawdata = fs.readFileSync('../results/freq.json'); -let fr_ = JSON.parse(rawdata); - +let fr_ = JSON.parse(rawdata); // frequencies +console.log('frequencies loaded'); /** freq * @@ -242,7 +247,8 @@ function freq_of(id) { } - +// get products from api/endpoint (in Json format) +// --- -- -- - - - let url = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch"; let settings = { method: "Get" }; @@ -251,12 +257,20 @@ fetch(url, settings) .then(res => res.json()) .then((json) => { - console.log('data loaded'); + let temp_file = '../results/keywords.json'; + let temp_prod_file = '../results/products.json'; + + console.log('products data loaded from emarket api/endpoint'); // do something with JSON var linkeys = do_proccess(json.data); - save_local(kwlinks_); + save_local(kwlinks_, temp_kw_file); + save_local(kwlinks_, temp_prod_file); + + // upload_file('pythia-files', os.tmpdir()+'/keywords.json', 'uploads/orders/keywords.json'); + upload_file('pythia-files', temp_kw_file, 'uploads/orders/emarket-keywords.json'); + upload_file('pythia-files', temp_prod_file, 'uploads/orders/emarket-keywords.json'); // check // echo 100 most frequent @@ -274,7 +288,7 @@ fetch(url, settings) // keyword links (word-links dictionary; array of objects) // ----------------------------------------------------------------------------- var kwlinks_ = []; ////////////////////////////// MAIN OUTPUT OF THE SCRIPT - +var prods_ = []; // connect; // get records to proccess; @@ -308,14 +322,15 @@ exports.main = () => { // query sql con.query(sql, function (err, result) { if (err) throw err; - console.log('Records from database received!') + console.log('Records from database received!'); do_proccess(result); // proccess - console.log('Keywords proccesed!') + console.log('Keywords proccesed!'); - save_keywords(); // save - upload_file('pythia-files', os.tmpdir()+'/keywords.json', 'uploads/orders/keywords.json'); - console.log('Results saved! Exiting.') + save_local(kwlinks_, '../results/keywords.json'); // save + // production (GFunctions): upload_file('pythia-files', os.tmpdir()+'/keywords.json', 'uploads/orders/keywords.json'); + upload_file('pythia-files', '../results/keywords.json', 'uploads/orders/emarket-keywords.json'); + console.log('Results saved! Exiting.'); // echo memory stats const used = process.memoryUsage(); @@ -327,42 +342,7 @@ exports.main = () => { } - -// Google Cloud Functions -//////////////////////////////////////////////////////////////////////////////// -function upload_file( bucketName, filePath, destFileName ) { - // [START storage_upload_file] - - // Sample code - // const bucketName = 'your-unique-bucket-name'; // The ID of your GCS bucket - // const filePath = 'path/to/your/file'; // The path to your file to upload - // const destFileName = 'your-new-file-name'; // The new ID for your GCS file - - // Imports the Google Cloud client library - const {Storage} = require('@google-cloud/storage'); - - // Creates a client - const storage = new Storage(); - - async function uploadFile() { - await storage.bucket(bucketName).upload(filePath, { - destination: destFileName, - gzip: true, // serve compressed - metadata: { // cache for 8 hours - cacheControl: 'public, max-age=28800', - } - }); - - console.log(`${filePath} uploaded to ${bucketName}`); - } - - uploadFile().catch(console.error); - // [END storage_upload_file] -} - - - - +var unmatched = 0; // main proccess // ----------------------------------------------------------------------------- @@ -374,8 +354,15 @@ function do_proccess(obj) { if (rec.img != 0) { var description = preproccess_text(rec.txt); - var pid = rec.id; - var fq = freq_of(pid); + var pid = rec.eys; + var fq = freq_of(pid)+1; + + prods_.push({ + id: pid, + w: description + }) + + // if (fq == 0) console.log(pid, description); var keys = []; var words = description.split(' '); @@ -431,38 +418,51 @@ function do_proccess(obj) { } -// save proccess -// ----------------------------------------------------------------------------- -function save_local(json) { - // stringify JSON Object - var jsonContent = JSON.stringify(json); +// save proccesses +//////////////////////////////////////////////////////////////////////////////// + +// Save Local +// ----------------------------------------------------------------------------- +function save_local(jsonArray, filePath) { + let fileStr = JSON.stringify(jsonArray); // convert json to string - fs.writeFile("../results/keywords-node.json", jsonContent, 'utf8', function (err) { + // write string to file + fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { if (err) { - console.log("An error occured while writing JSON Object to File."); + console.log("An error occured while writing keywords.json"); return console.log(err); } - console.log("JSON file has been saved."); }); + + return true; } -function save_keywords() { - let jsonStr = JSON.stringify(kwlinks_); - // console.log(jsonStr); +// Save to Cloud Storage +// ----------------------------------------------------------------------------- - fs.writeFileSync(os.tmpdir() + "/keywords.json", jsonStr, 'utf8', (err) => { - if (err) { - console.log("An error occured while writing keywords.json"); - return console.log(err); - } - console.log("linkeys file saved."); - }); -} +async function upload_file( bucketName, srcFilePath, trgFilePath ) { + // Creates a client + const storage = new Storage({projectId, keyFilename}); + + try { + await storage.bucket(bucketName).upload(srcFilePath, { + destination: trgFilePath, + gzip: true, // serve compressed + metadata: { // cache for 8 hours + cacheControl: 'public, max-age=60' // production set: 28800 + } + }); + console.log(`${srcFilePath} uploaded to ${bucketName}`); + } + catch(err) { + console.error('ERROR:', err); + } +} // functions for linking words in keywords dictionary //////////////////////////////////////////////////////////////////////////////// @@ -547,6 +547,7 @@ function kb_trans(str) { return out; } + // clean text trims some characters (+.') and internal multiple-spaces // --- function clean_text(txt) { @@ -555,6 +556,7 @@ function clean_text(txt) { .replace(' ',' '); } + // check if term is significant // (if not, the term will be excluded from keywords dicionary) // --- @@ -564,6 +566,7 @@ function is_significant(str) { return !(/\d/.test(str)); } + // check if word: w // ...has synonyms; return list of synonyms // --- @@ -576,6 +579,7 @@ function synonym_keys(w) { return [ w ]; } + // edit common mistakes // with suggested replaces function do_replaces(str) { @@ -583,6 +587,7 @@ function do_replaces(str) { return str; } + // preproccess description // --- function preproccess_text(str) { @@ -601,6 +606,7 @@ function mark_linked_words(txt) { return txt } + // mark a link (lws) to a text (source) // conecting them with a dash/minus character // --- |