diff options
Diffstat (limited to 'javascript/keygen_GCF.js')
| -rw-r--r-- | javascript/keygen_GCF.js | 629 |
1 files changed, 629 insertions, 0 deletions
diff --git a/javascript/keygen_GCF.js b/javascript/keygen_GCF.js new file mode 100644 index 0000000..8b13dd9 --- /dev/null +++ b/javascript/keygen_GCF.js @@ -0,0 +1,629 @@ +/** keygen + * + * this script constucts a linked-wordkeys structure + * ----------------------------------------------------------------------------- + * + * Contents: + * #1 Requirements + * #2.1 Personalized constants and parametres + * #2.2 Preloaded Data + * #3 Supporting functions + * #4 Output functions + * #5 Entry-point function main() + */ + +// #1 +// REQUIREMENTS +//////////////////////////////////////////////////////////////////////////////// + +const fs = require('fs'); +const os = require('os'); +const fetch = require('node-fetch'); +const { ECDH } = require('crypto'); +const {Storage} = require('@google-cloud/storage'); // Google Cloud Storage + + + +// #2.1 +// SETUP PERSONALIZED CONSTANTS AND PARAMETRES +//////////////////////////////////////////////////////////////////////////////// + + +// get products FROM api/endpoint parametres +const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json'; +const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch"; +const request_settings = { method: "Get" }; + + + +// Global Variables +// ----------------------------------------------------------------------------- + +// keyword links (word-links dictionary; array of objects) +// ----------------------------------------------------------------------------- +var kwlinks_ = []; ////////////////////////////// MAIN OUTPUT OF THE SCRIPT +var prods_ = []; +var fr_ =[]; + + +// #2.2 +// PRELOADED DATA +//////////////////////////////////////////////////////////////////////////////// + +// any-character to keyboard-latin mapping +// ----------------------------------------------------------------------------- +var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); +var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); +const map = new Map(); +for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]); + + +// synonyms +// ----------------------------------------------------------------------------- + +var synonyms = []; // groups of synonyms +var synonym_kbs = []; // cache kb-formats for performance +var synonyms_Originals = [ + 'μπίρα μπύρα μπίρες μπύρες', + 'αυγά αβγά αυγό', + 'σίκαλης σικάλεως', + 'ξηρά ξερά', + 'ρολό ρολλό', + 'coca-cola cocacola coke', + 'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας', + 'χαρτί-κουζίνας ρολό-κουζίνας', + 'οινος κρασι', + 'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ', + 'DR-OETKER OETKER', + 'DR.BECKMANN BECKMANN', + 'NES-CAFE NESCAFE', + 'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής', + 'τσίπουρο ρακή', + 'Βρώμη Βρώμης', + 'Φράουλα Φράουλες Φράουλας', + 'Μαλλιά Μαλλιών', + 'Κέικ Cake', + 'CRETA-FARMS CRETA-FARM', + 'MARSEILLAIS LE-PETIT-MARSEILLAIS PETIT-MARSEILLAIS', + 'Γαϊδούρας Γαϊδάρου', + 'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ', + 'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ', + 'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ', + 'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ', + 'Ντομάτα Ντομάτας', + 'Ελαφρύ Ελαφρά Light', + 'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά', + 'τριμμένη τριμμένο', + 'Τόνος Τόνου', + 'Κριθαρένια κρίθινα', + 'Χωρίς-Kαφεϊνη Decaffeine', + 'Το-Μάννα Μάννα', + 'Κράνμπερι Κράνμπερις', + 'Κρήτης Κρητικό', + 'Πέννες Πένες', + 'Μακαρόνια Σπαγγέτι Σπαγγετίνι Σπαγγετόνι', + 'Καρτέλλα Καρτέλα Καρτέλλες' +] +synonyms_Originals.forEach( grp => { + synonyms.push( grp.split(' ') ); + synonym_kbs.push( kb_trans(grp).split(' ') ); +}) + +// significant terms +// ----------------------------------------------------------------------------- +significantExceptios = '7UP 3ΑΛΦΑ 17 3Π 7DAYS K2R'.split(' ') + + +// replaces (correcting descriptions) +// ----------------------------------------------------------------------------- +replaces = []; +replaceSource = [ + '3 ΑΛΦΑ ;3ΑΛΦΑ ', + 'HEAD & SHOULDERS ;HEAD&SHOULDERS ', + 'W.K Kellogg ; ', + 'ΦΙΛΕΤ ;Φιλέτο ', + 'ΕΝΕΛΛΑΔ ;Εν-Ελλάδι ', + 'ΓΑΛΟΠΟΥΛ ;Γαλοπούλα ', + '7 DAYS ;7DAYS ', + 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ ', + 'Zero% ;Zero ', + +]; +replaceSource.forEach( it => { + st = it.split(';'); + replaces.push({ src: st[0], trg: st[1] }); +}); + + +// words that shall not be searched first +// ----------------------------------------------------------------------------- +var noRootKeywords = []; +noRoot = [ + 'χωρίς', + 'εισαγωγής', + 'δώρο', + 'γεύση', + 'γεύσεις', + 'φέτες', + 'Χωρίς-Γλουτένη', + 'Γλουτένη', + 'Λακτόζη', + 'Παστερίωσης', + 'Ανθρακικό', + 'Συντηρητικά', + 'Χωρίς-Ζάχαρη', + 'Άλεσης', + 'Χωρίς-Αλάτι', + 'Χωρίς-Λακτόζη', + 'Χωρίς-Συντηρητικά', + 'Χωρίς-Αλκοόλ', + 'Χωρίς-Kαφεϊνη', + 'Χωρίς-Γλυκάνισο', + 'Χωρίς-Ανθρακικό', + 'Υψηλής-Παστερίωσης', + 'Ολες-τις-Χρήσεις', + 'Ολικής-Άλεσης', + 'Ολικής-Aλέσεως', + 'Ολικής', + 'Γαϊδούρας', + 'Γαϊδάρου', + 'Ρούχων', + 'Πιάτων', + 'πλύσεις', + 'Πλυντηρίου', + 'Φύλλων', + 'Γάλακτος', + 'Χρήσης', + 'Τύπου', + 'Ολλανδίας', + 'Απορριμμάτων', + 'Medium', + 'Μαλλιά', + 'Μαλλιών', + 'Γενικής', + 'Plus', + 'Classic', + 'Έκπληξη', + 'Μάνης', + 'Ελάτου', + 'Άγριων', + 'Βοτάνων', + 'Κατσαρίδες', + 'Λακωνίας', + 'ΠΑΡΑΓΓΕΛΙΩΝ', + 'Λήμνου', + 'Αργινίνης', + 'Καλαθάκι', + 'Σκύλου', + 'Γάτας', + 'Σώματος', + 'Αδυνατίσματος', + 'Προστασίας', + 'Ακράτειας', + 'Προσώπου', + 'προσθήκη', + 'Βρεφικής', + 'Καθαρισμού', + 'Ρούχων' +]; +noRoot.forEach( w => { noRootKeywords.push(kb_trans(w)); }); + + +// list of linked-words +// ----------------------------------------------------------------------------- +linkedWords = [ + 'Χωρίς-Γλουτένη', + 'Χωρίς-Ζάχαρη', + 'Χωρίς-Αλάτι', + 'Χωρίς-Λακτόζη', + 'Χωρίς-Συντηρητικά', + 'Χωρίς-Αλκοόλ', + 'Χωρίς-Kαφεϊνη', + 'Χωρίς-Γλυκάνισο', + 'Χωρίς-Ανθρακικό', + 'Υψηλής-Παστερίωσης', + 'Ολικής-Άλεσης', + 'Ολικής-Aλέσεως', + 'Χαρτί-Υγείας', + 'ρολό-υγείας', + 'χαρτί-τουαλέτας', + 'Χαρτί-Κουζίνας', + 'Μπάρες-Δημητριακών', + 'Μπαρμπα-Στάθης', + 'COCA-COLA', + 'Aς-Μαγειρέψουμε', + 'ΚΡΙΣ-ΚΡΙΣ', + 'ΚΡΙ-ΚΡΙ', + 'ΕΛ-ΓΚΡΕΚΟ', + 'FREE-STEP', + 'EL-SABOR', + 'LE-PETIT-MARSEILLAIS', + 'DOUWE-EGBERTS', + 'ΕΝ-ΕΛΛΑΔΙ', + 'SPIN-SPAN', + 'CRETA-FARMS', + 'CRETA-FARM', + 'NES-CAFE', + 'Ολες-τις-Χρήσεις', + 'Το-Μάννα', + 'Χωρίς-προσθήκη-ζάχαρης' +]; + + +// list of words to exclude from keywords +// ----------------------------------------------------------------------------- +// NOTE: APPLIED in PER-WORD base -> after spliting description to words +removeList = [] +removeOriginals = 'κατά παρά υπό από μετά προς μας με σε για του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ') +removeOriginals.forEach( w => { removeList.push( kb_trans(w)); }); + + +// depricated +/// // read frequency (local) data +/// let rawdata = fs.readFileSync('../results/freq.json'); +/// let fr_ = JSON.parse(rawdata); // frequencies +/// console.log('frequencies loaded'); + + + + +// #3 +// SUPPORTING FUNCTIONS +//////////////////////////////////////////////////////////////////////////////// + + +/** freq + * + * @param id (int): product's eys_code + * @return frequency of product + */ +function freq_of(id) { + fr_.forEach( rec => { + if (rec.id == id) return rec.fq + }); + + return 0; +} + + + + +// proccess data +// The MAIN proccess +// ----------------------------------------------------------------------------- +function extract_linked_keywords(obj) { + // console.log(JSON.stringify(obj, null, 2)); + + obj.forEach( rec => { + + if (rec.img != 0) { + + var description = preproccess_text(rec.txt); + var pid = rec.eys; + var fq = freq_of(pid)+1; + + prods_.push({ + id: pid, + w: description + }) + + var keys = []; + var words = description.split(' '); + + // filter words; keep only significant + words.forEach( w => { + if (removeList.indexOf(kb_trans(w)) == -1) // if not excluded + if (is_significant(w)) // and significant + keys.push(w); // add it to keys + }); + // console.log(pid, description, keys); + + keys.forEach( w => { + var wl = synonym_keys(w); + + // if key CAN be a root word (not a no-Root-keyword) update root-node + if (noRootKeywords.indexOf(kb_trans(w)) == -1) { + + root_key(wl, fq); // update root keyword stats + + // if key is the only in the list of product's keywords + // connect it with a dummy key (to preserve the reference to the product) + if (keys.length == 1) connect_keys(wl, ['*'], pid, fq); + + // connect w with all the other product's keywords + keys.forEach( w2 => { + if (w2 != w) { + var w2syns = synonym_keys(w2); + connect_keys( wl, w2syns, pid, fq); + } + }); + } + + }); + } + }); // main proccessing finished; + + // post proccess + // ------------------------------------------------------------------------- + + // remove cached keys from final array + kwlinks_.forEach( ro => { + delete ro.kb; + ro.c.forEach( ch => { delete ch.kb; }); + }); + + // sort root and child nodes by frequency descanding + kwlinks_.forEach( it => { + it.c = it.c.sort((a, b) => b.f - a.f ); + }); + kwlinks_ = kwlinks_.sort((a, b) => b.f - a.f ); + +} + + +// functions for linking words in keywords dictionary +//////////////////////////////////////////////////////////////////////////////// + +// set root-keyword: wl (if not exist) +// update frequency: f +// * wl is a list of synonym-words +// ** comparison is based on the *keyboard* format +// --- +function root_key ( wl, f ) { + var wkb = kb_trans(wl[0]) // cache kb format + + // check if exists in root keys already + // NOTE: you only need to check the 1st word of synonyms-list + for (i=0; i< kwlinks_.length ; i++) { + if (kwlinks_[i].kb == wkb) { + kwlinks_[i].f += f; + return true; + } + } + // if not exists, append keyword + kwlinks_.push({ + w : wl, + kb : wkb, + f : f, + c : [] + }); + return true; +} + + +// connect keys: a , b (each one is a list of synonmyms) +// of product with id: i +// with frequency: f +// --- +function connect_keys( a, b, id, f ) { + var kbA = kb_trans(a[0]); + var kbB = kb_trans(b[0]); + var bExists = false; + + if (kbA == kbB) return false; // exclude just-in-case + + for (i=0; i< kwlinks_.length ; i++) { + if (kwlinks_[i].kb == kbA) { // found: a; + // update connection to: b + bExists = false; + for (j=0 ; j < kwlinks_[i].c.length ; j++) { + if (kwlinks_[i].c[j].kb == kbB) { + bExists = true; + // update the connection's data + kwlinks_[i].c[j].f += f; + kwlinks_[i].c[j].p.push(id) + break; + } + } + // if connection not exist, init a new one + if (bExists == false) { + // create connection with: b + kwlinks_[i].c.push({ + w : b, + kb : kbB, + f : f, + p : [ i ] + }); + } + return true; + } + } +} + + +// other supplementary functions +//////////////////////////////////////////////////////////////////////////////// + + +// kb_trans translates string to keyboard-latin keys; +// --- +function kb_trans(str) { + str = str.replace('\'',''); + var out = ''; + for (var i=0 ; i< str.length; i++) out += map.get(str[i]); + return out; +} + + +// clean text trims some characters (+.') and internal multiple-spaces +// --- +function clean_text(txt) { + return txt.replace('+',' ').replace('.',' ') + .replace(' ',' ') + .replace(' ',' '); +} + + +// check if term is significant +// (if not, the term will be excluded from keywords dicionary) +// --- +function is_significant(str) { + if (str == '') return false; + if (significantExceptios.indexOf(str) !== -1) return true; + return !(/\d/.test(str)); +} + + +// check if word: w +// ...has synonyms; return list of synonyms +// --- +function synonym_keys(w) { + w_kb = kb_trans(w); + for (i=0 ; i < synonym_kbs.length ; i++) { + if (synonym_kbs.indexOf(w_kb) !== -1) + return synonyms[i]; + } + return [ w ]; +} + + +// edit common mistakes +// with suggested replaces +function do_replaces(str) { + replaces.forEach( it => { str = str.replace(it.src, it.trg); }); + return str; +} + + +// preproccess description +// --- +function preproccess_text(str) { + str = do_replaces(str); + str = clean_text(str); + str = mark_linked_words(str); + return str; +} + + +// mark linked words (connect them with a dash) +// return new text after "all-links" are marked +// --- +function mark_linked_words(txt) { + linkedWords.forEach( lw => { txt = mark_link( lw, txt ); }); + return txt +} + + +// mark a link (lws) to a text (source) +// conecting them with a dash/minus character +// --- +function mark_link(lws, source) { + var src_kb = kb_trans(source.replace(' ', '-')); + var lws_kb = kb_trans(lws.replace(' ', '-')); + var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase()) + if (_left !== -1 ) { + return source.slice(0, _left) + lws + source.slice(_left + lws.length); + } + else return source; +} + + + +// #4 +// OUTPUT FUNCTIONS +//////////////////////////////////////////////////////////////////////////////// + + +// Save Local +// ----------------------------------------------------------------------------- +function save_local(jsonArray, filePath) { + let fileStr = JSON.stringify(jsonArray); // convert json to string + + // write string to file + fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { + if (err) { + console.log("An error occured while writing keywords.json"); + return console.log(err); + } + // console.log("JSON file has been saved."); + }); + + return true; +} + + + +// Save to Google Cloud Storage +// ----------------------------------------------------------------------------- +async function upload_file( bucketName, srcFilePath, trgFilePath ) { + // Creates a client + const storage = new Storage(); + + try { + await storage.bucket(bucketName).upload(srcFilePath, { + destination: trgFilePath, + gzip: true, // serve compressed + metadata: { // cache for 8 hours + cacheControl: 'public, max-age=60' // production set: 28800 + } + }); + console.log(`${srcFilePath} uploaded to ${bucketName}`); + } + + catch(err) { + console.error('ERROR:', err); + } + +} + + + + +// #5 +// MAIN function (exposed function to run the whole proccess) +//////////////////////////////////////////////////////////////////////////////// + + +/** main() + * + * nodejs's exported function + * used as entry-point function (in case of Google Cloud function) + */ + +// keep only 1 of next 2 lines +// exports.main = () => { // entry point functioon when google cloud function +var main = () => { // concise function when direct script + + fetch(frequency_endpoint, request_settings) + .then(res => res.json()) + .then((json) => { + fr_ = json; + console.log('frequencies loaded from CDN'); + + fetch(products_endpoint, request_settings) + .then(res => res.json()) + .then((json) => { + + console.log('products data loaded from emarket api/endpoint'); + + // proccess data + extract_linked_keywords(json.data); + + save_local(kwlinks_, temp_kw_file); + save_local(kwlinks_, temp_prod_file); + + // upload temp files to cloud + upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'); + upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json'); + + // check + // echo 100 most frequent + let i = 0; + kwlinks_.forEach(rec => { + if (i<100) { + console.log(i, JSON.stringify(rec.w)) + } + i++; + }); + + // echo memory stats + const used = process.memoryUsage(); + for (let key in used) { + console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`); + } + + }); + + }); + +} |