/** keygen * * this script constucts a linked-wordkeys structure * ----------------------------------------------------------------------------- * * Contents: * --- * #1 Requirements * -* fs * -* os * -* node-fetch * -* @google-cloud/storage * * #2.1 Personalized constants and parametres * -* frequency endpoin * -* products endpoint * * #2.2 Purify string functions * -* keyboardize() * -* sanitize() * -* clean() * * #2.3 Preloaded Data (TODO: load via endpoint) * -* synonyms * -* significant exceptions * -* replaces (linked words) * -* noRootKeywords * -* removeList * * #3 Supporting functions * -* freq_of( product's eys-code) * -* extract_linked_keywords -> NOTE: MAIN PROCEDURE to do the job * -* root_key( word_list, frequency ) * -* connect_keys( wordlist_#1, wordlist_#2 , product_id, frequency ) * -* is_significant * -* synonym_keys( word ) * -* mark_explicit_links() * -* preproccess_text() * * #4 Output functions * -* save_local() * -* upload_file * * #5 Entry-point function main() * -* main() NOTE: entry-point function */ /** #1 * --- * REQUIREMENTS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// const fs = require('fs'); const os = require('os'); const fetch = require('node-fetch'); const {Storage} = require('@google-cloud/storage'); // Google Cloud Storage /** #2.1 * --- * SETUP PERSONALIZED CONSTANTS AND PARAMETRES * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// // get products FROM api/endpoint parametres // --- -- -- - - - const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json'; const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch"; const request_settings = { method: "Get" }; // google-storage parametres // --- -- -- - - - const projectId = 'pythia-251711'; const keyFilename = '../auth/pythia-251711-047e3d5e6608.json'; // temp files // need to be created prior to save-to-google-storage proccess // because upload_file() uploads an existing local file to G-Storage // --- -- -- - - - const temp_kw_file = '../results/keywords.json'; const temp_prod_file = '../results/products.json'; // Global Variables // ----------------------------------------------------------------------------- // Global arrays // keyword links (word-links dictionary; array of objects) // ----------------------------------------------------------------------------- var kwlinks_ = []; // keyword-links structure (structure to construct) var prods_ = []; // products (array to construct) var _fr =[]; // product frequencies array (as read from endpiont) var _pr =[]; // product array (as read from endpiont) // Console Colors // ----------------------------------------------------------------------------- const Reset = "\x1b[0m"; const FgRed = "\x1b[31m"; const FgGreen = "\x1b[32m"; /** #2.2 * --- * PURIFY STRING FUNCTIONS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// /** keyboardize * ----------------------------------------------------------------------------- * translates string to keyboard-latin keys * (the ones that used whan typing each letter of the word) * * @param str (string): original string (utf8 of latin or greek subgroups) * @return (string): latin/ascii equivalent string */ // cache (keeo in global) any-character to keyboard-latin mapping // --- -- -- - - - var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); const map = new Map(); for (var i=0; i { ap = pair.split(' '); accented_vowels.push({ a: ap[0], // accented p: ap[1] // pure = non accended }); }) // the sanitize_GR function // --- -- -- - - - function sanitize_GR(str) { str = str.toLowerCase(); // replace accended vowels with pure ones accented_vowels.forEach( v => { str = str.replaceAll(v.a, v.p); }); // replace sigma on the end of words str = str + ' '; str = str.replaceAll('σ-', 'ς-'); str = str.replaceAll('σ ', 'ς '); return str; } /** clean text * --- -- -- - - - * removes non keyword characters [+ . , !] and internal multiple-spaces * @param txt (string): product description */ function clean_text(txt) { return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space .replace('!','').replace('\"', '') // remove character .replace(' ',' ').replace(' ',' '); // remove multiple spaces } /** #2.3 * --- * PRELOADED DATA * in the future these data may be loaded from a database / api-endpoint * ----------------------------------------------------------------------------- * ///////////////////////////////////////////////////////////////////////////// /** synonyms /////////////////////////////////////////////////////////////////// * ----------------------------------------------------------------------------- */ var synonyms = []; // groups of synonyms var synonym_kbs = []; // cache kb-formats for performance var synonyms_Originals = [ 'μπίρα μπύρα μπίρες μπύρες', 'αυγά αβγά αυγό', 'σίκαλης σικάλεως', 'ξηρά ξερά', 'ρολό ρολλό', 'coca-cola cocacola coke', 'max-factor, maxfactor', 'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας', 'χαρτί-κουζίνας ρολό-κουζίνας', 'οινος κρασι', 'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ', 'DR-OETKER OETKER', 'DR.BECKMANN BECKMANN', 'NES-CAFE NESCAFE', 'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής', 'τσίπουρο ρακή', 'Βρώμη Βρώμης', 'Φράουλα Φράουλες Φράουλας', 'Μαλλιά Μαλλιών', 'Κέικ Cake', 'CRETA-FARMS CRETA-FARM', 'MARSEILLAIS LE-PETIT-MARSEILLAIS PETIT-MARSEILLAIS', 'Γαϊδούρας Γαϊδάρου', 'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ', 'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ', 'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ', 'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ', 'Ντομάτα Ντομάτας Τομάτα', 'Ελαφρύ Ελαφρά Light', 'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά Ελληνικός', 'τριμμένη τριμμένο τριμμένα', 'Τόνος Τόνου', 'Κριθαρένια κρίθινα', 'Χωρίς-Kαφεϊνη Decaffeine', 'Το-Μάννα Μάννα', 'Κράνμπερι Κράνμπερις', 'Κρήτης Κρητικό Κρητική', 'Πέννες Πένες', 'σοκολατένιος σοκολατένια σοκολατένιο', 'Μακαρόνια Ζυμαρικά Σπαγγέτι Σπαγγετίνι Σπαγγετόνι', 'Λαζάνια Λιγκουίνι Ματσάτα Linguine', 'Καρτέλλα Καρτέλα Καρτέλλες', 'κλασική κλασικός κλασικό', 'tea τσάι', 'ωτοκαθαριστές μπατονέτες' ] synonyms_Originals.forEach( grp => { // remove accends so any replaces can be done flawlessly grp = sanitize_GR(grp).trim(); // push it to array of synonyms (in lowerCase); synonyms.push( grp.toLowerCase().split(' ') ); // push it the keyboardized array of synonyms synonym_kbs.push( keyboardize(grp).toLowerCase().split(' ') ); }) // console.log(synonyms); process.exit(); // console.log(synonym_kbs); process.exit(); /** significant terms ////////////////////////////////////////////////////////// * ----------------------------------------------------------------------------- */ significantExceptios = sanitize_GR('7-UP 7UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ') /** replaces (correcting descriptions) * = construct unequivocally liked words /////////////////////////////////////// * ----------------------------------------------------------------------------- * * NOTE: TODO: * in future implementations multi-word keywords * may use the non-breaking space as conecting character (\u00A0) instead of dush (-) * (or maybe both of them) * --- -- -- - - - */ replaces = []; replaceSource = [ '3Α;3-ΑΛΦΑ', '3 ΑΛΦΑ;3-ΑΛΦΑ', 'HEAD & SHOULDERS;HEAD&SHOULDERS', 'HEAD N SHOULDERS;HEAD&SHOULDERS', 'W.K Kellogg; W-K-Kellogg', 'W.K Kellogg;', '7 DAYS;7-DAYS', '7 UP;7UP', '7-UP;7UP', 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', 'COCA COLA;COCA-COLA', 'J P. CHENET; J.P.CHENET', 'J.P. CHENET; J.P.CHENET', 'COCACOLA;COCA-COLA', 'NES CAFE;NESCAFE', 'NES-CAFE;NESCAFE', 'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS', 'PETIT MARSEILLAIS;PETIT-MARSEILLAIS', 'Το Μάννα;Το-Μάννα', 'Χωρίς Γλουτένη;Χωρίς-Γλουτένη', 'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη', 'Χωρίς Αλάτι;Χωρίς-Αλάτι', 'Χωρίς Λακτόζη;Χωρίς-Λακτόζη', 'Χωρίς Συντηρητικά;Χωρίς-Συντηρητικά', 'Χωρίς Αλκοόλ;Χωρίς-Αλκοόλ', 'Χωρίς Καφεϊνη;Χωρίς-καφεϊνη', 'Χωρίς Καφεΐνη;Χωρίς-καφεϊνη', 'Χωρίς Γλυκάνισο;Χωρίς-Γλυκάνισο', 'Χωρίς Ανθρακικό;Χωρίς-Ανθρακικό', 'Υψηλής Παστερίωσης;Υψηλής-Παστερίωσης', 'Ολικής Άλεσης;Ολικής-Άλεσης', 'Ολικής Aλέσεως;Ολικής-Aλέσεως', 'Χαρτί Υγείας;Χαρτί-Υγείας', 'ρολό υγείας;ρολό-υγείας', 'χαρτί τουαλέτας;χαρτί-τουαλέτας', 'Χαρτί Κουζίνας;Χαρτί-Κουζίνας', 'ρολό κουζίνας;ρολό-κουζίνας', 'Μπάρες Δημητριακών;Μπάρες-Δημητριακών', 'Ας Μαγειρέψουμε;Ας-Μαγειρέψουμε', 'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', 'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', 'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ', 'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ', 'ΕΛ ΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', 'ΕΛΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', 'FREE STEP;FREE-STEP', 'EL SABOR;EL-SABOR', 'ELSABOR;EL-SABOR', 'DOUWE EGBERTS;DOUWE-EGBERTS', 'DOUWEEGBERTS;DOUWE-EGBERTS', 'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', 'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', 'SPIN SPAN;SPIN-SPAN', 'SPINSPAN;SPIN-SPAN', 'CRETA-FARMS;CRETA-FARM', 'CRETA-FARM;CRETA-FARM', 'CRETAFARM;CRETA-FARM', 'Ολες-τις-Χρήσεις;Ολες-τις-Χρήσεις', 'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη', 'φρουι ζελε, φρουί-ζελε', 'DR BECKMANN, DR-BECKMANN', 'Χωρίς Kαφεϊνη;Χωρίς-καφεϊνη', 'Χωρίς Kαφεΐνη;Χωρίς-καφεϊνη', ]; replaceSource.forEach( it => { st = sanitize_GR( it.toLowerCase() ).split(';'); replaces.push({ src: ' '+ st[0] +' ', // encolse between spaces trg: ' '+ st[1] +' ' // to separate from before/after words }); }); // console.log(replaces); process.exit(); // words that shall not be searched first ////////////////////////////////////// // ----------------------------------------------------------------------------- var noRootKeywords = []; noRoot = [ 'χωρίς', 'εισαγωγής', 'δώρο', 'γεύση', 'γεύσεις', 'φέτες', 'Χωρίς-Γλουτένη', 'Γλουτένη', 'Λακτόζη', 'Παστερίωσης', 'Ανθρακικό', 'Συντηρητικά', 'Χωρίς-Ζάχαρη', 'Άλεσης', 'Χωρίς-Αλάτι', 'Χωρίς-Λακτόζη', 'Χωρίς-Συντηρητικά', 'Χωρίς-Αλκοόλ', 'Χωρίς-Kαφεϊνη', 'Χωρίς-Γλυκάνισο', 'Χωρίς-Ανθρακικό', 'Υψηλής-Παστερίωσης', 'Ολες-τις-Χρήσεις', 'Ολικής-Άλεσης', 'Ολικής-Aλέσεως', 'Ολικής', 'Σοκολάτας', 'Υγείας', 'Γαϊδούρας', 'Δημητριακών', 'Γαϊδάρου', 'Ρούχων', 'Πιάτων', 'πλύσεις', 'Πλυντηρίου', 'Φύλλων', 'Γάλακτος', 'Χρήσης', 'Τύπου', 'Ολλανδίας', 'Απορριμμάτων', 'Medium', 'Μαλλιά', 'Μαλλιών', 'Γενικής', 'Plus', 'Classic', 'Έκπληξη', 'Μάνης', 'Ελάτου', 'Άγριων', 'Βοτάνων', 'Κατσαρίδες', 'Λακωνίας', 'ΠΑΡΑΓΓΕΛΙΩΝ', 'Λήμνου', 'Αργινίνης', 'Καλαθάκι', 'Σκύλου', 'Γάτας', 'ΠΓΕ', 'ΤΟ', 'Σώματος', 'Αδυνατίσματος', 'Προστασίας', 'Ακράτειας', 'Προσώπου', 'προσθήκη', 'Βρεφικής', 'Καθαρισμού', 'Ρούχων', 'birthday', 'δημιουργικοτητας', 'καφεϊνη', 'kαφεϊνη', 'καφετιερες', 'Βραστήρες', 'χωρις kαφεϊνη', 'ξεχωρισμα' ]; noRoot.forEach( w => { noRootKeywords.push(keyboardize(w)); }); // list of linked-words //////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- linkedWords = [ sanitize_GR('Χωρίς-Γλουτένη'), sanitize_GR('Χωρίς-Ζάχαρη'), sanitize_GR('Χωρίς-Αλάτι'), sanitize_GR('Χωρίς-Λακτόζη'), sanitize_GR('Χωρίς-Συντηρητικά'), sanitize_GR('Χωρίς-Αλκοόλ'), sanitize_GR('Χωρίς-Kαφεϊνη'), sanitize_GR('Χωρίς-Γλυκάνισο'), sanitize_GR('Χωρίς-Ανθρακικό'), sanitize_GR('Υψηλής-Παστερίωσης'), sanitize_GR('Ολικής-Άλεσης'), sanitize_GR('Ολικής-Aλέσεως'), sanitize_GR('Χαρτί-Υγείας'), sanitize_GR('ρολό-υγείας'), sanitize_GR('χαρτί-τουαλέτας'), sanitize_GR('Χαρτί-Κουζίνας'), sanitize_GR('Μπάρες-Δημητριακών'), sanitize_GR('Μπαρμπα-Στάθης'), sanitize_GR('COCA-COLA'), sanitize_GR('Aς-Μαγειρέψουμε'), sanitize_GR('ΚΡΙΣ-ΚΡΙΣ'), sanitize_GR('ΚΡΙ-ΚΡΙ'), sanitize_GR('ΕΛ-ΓΚΡΕΚΟ'), sanitize_GR('FREE-STEP'), sanitize_GR('EL-SABOR'), sanitize_GR('LE-PETIT-MARSEILLAIS'), sanitize_GR('DOUWE-EGBERTS'), sanitize_GR('ΕΝ-ΕΛΛΑΔΙ'), sanitize_GR('SPIN-SPAN'), sanitize_GR('CRETA-FARMS'), sanitize_GR('CRETA-FARM'), sanitize_GR('NES-CAFE'), sanitize_GR('Ολες-τις-Χρήσεις'), sanitize_GR('Το-Μάννα'), sanitize_GR('Χωρίς-προσθήκη-ζάχαρης'), sanitize_GR('φρουί-ζελέ') ]; // list of words to exclude from keywords ////////////////////////////////////// // ----------------------------------------------------------------------------- // NOTE: APPLIED in PER-WORD base -> after spliting description to words removeList = [] removeOriginals = sanitize_GR([ 'κατά παρά υπό από μετά προς', 'μας με σε για του της τον τη την των από στο το στον', '& r s ft l τ e g h k m n o p s x c ω', 'cc cv gr lt' ].join(' ')).split(' ') removeOriginals.forEach( w => { removeList.push( keyboardize(w)); }); /** #3 * --- * SUPPORTING FUNCTIONS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// /** freq * * @param id (int): product's eys_code * @return frequency of product */ function freq_of(id) { _fr.forEach( rec => { if (rec.id == id) return rec.fq }); return 0; } /** proccess data ///////////////////////////////////////////////////////////// * --- -- -- - - - * * central proccessing procedure * ----------------------------------------------------------------------------- * * Operation: * --- * for each title { * + clean text, get frequency * + preproccess and split into individual words * + for each significant word { * + update root keywords * + update linked words * } * } * post-process results * * ///////////////////////////////////////////////////////////////////////////// */ function extract_linked_keywords(obj) { // console.log(JSON.stringify(obj, null, 2)); obj.forEach( rec => { if (rec.img != 0) { //var description = preproccess_text(rec.txt); var description = clean_text(rec.txt); var pid = rec.eys; var fq = freq_of(pid)+1; prods_.push({ id: pid, w: description.trim() }) var keys = []; var words = preproccess_text(description).split(' '); // filter words; keep only significant words.forEach( w => { if (removeList.indexOf(keyboardize(w)) == -1) // if not excluded if (is_significant(w)) // and significant keys.push(w); // add it to keys }); // console.log(pid, description, keys); keys.forEach( w => { var wl = synonym_keys(w); // if key CAN be a root word (not a no-Root-keyword) update root-node if (noRootKeywords.indexOf(keyboardize(w)) == -1) { root_key(wl, fq); // update root keyword stats // if key is the only in the list of product's keywords // connect it with a dummy key (to preserve the reference to the product) if (keys.length == 1) connect_keys(wl, ['*'], pid, fq); // connect w with all the other product's keywords keys.forEach( w2 => { if (w2 != w) { var w2syns = synonym_keys(w2); connect_keys( wl, w2syns, pid, fq); } }); } }); } }); // main proccessing finished; // post proccess // ------------------------------------------------------------------------- // remove cached keys from final array kwlinks_.forEach( ro => { delete ro.kb; ro.c.forEach( ch => { delete ch.kb; }); }); // sort root and child nodes by frequency descanding kwlinks_.forEach( it => { it.c = it.c.sort((a, b) => b.f - a.f ); }); kwlinks_ = kwlinks_.sort((a, b) => b.f - a.f ); } // functions for linking words in keywords dictionary //////////////////////////////////////////////////////////////////////////////// /** root_key( wl, f ) * --- -- -- - - - * set root keyword: wl (if not exist) * update frequency: f * * @param wl (list) :a list of synonym-words * @param f (int): frequency of product * * ** comparison is based on the *keyboard* format */ function root_key ( wl, f ) { var wkb = keyboardize(wl[0]) // cache kb format // check if exists in root keys already // NOTE: you only need to check the 1st word of synonyms-list for (i=0; i< kwlinks_.length ; i++) { if (kwlinks_[i].kb == wkb) { kwlinks_[i].f += f; return true; } } // if not exists, append keyword kwlinks_.push({ w : wl, kb : wkb, f : f, c : [] }); return true; } /** connect_keys * --- -- -- - - - * @param (list) : connect keyword a (synonyms list) * @param (list) : with keyword b (synonyms list) * @param (int) : id (eys code) of product where keywords a and b are linked * @param (int) : frequency of product (now many times has been purchased) * @returns true */ function connect_keys( a, b, id, f ) { var kbA = keyboardize(a[0]); var kbB = keyboardize(b[0]); var bExists = false; if (kbA == kbB) return false; // exclude just-in-case for (i=0; i< kwlinks_.length ; i++) { if (kwlinks_[i].kb == kbA) { // found: a; // update connection to: b bExists = false; for (j=0 ; j < kwlinks_[i].c.length ; j++) { if (kwlinks_[i].c[j].kb == kbB) { bExists = true; // update the connection's data kwlinks_[i].c[j].f += f; kwlinks_[i].c[j].p.push(id) break; } } // if connection not exist, init a new one if (bExists == false) { // create connection with: b kwlinks_[i].c.push({ w : b, kb : kbB, f : f, p : [ id ] }); } return true; } } } // other supplementary functions //////////////////////////////////////////////////////////////////////////////// /** is significant * --- -- -- - - - * check if term is significant (and needs to be indexed) * (if not, the term will be excluded from keywords dicionary) * * @param str * @return (boolean) */ function is_significant(str) { if (str == '') return false; if (significantExceptios.indexOf(str) !== -1) return true; return !(/\d/.test(str)); } /** synonym_keys * --- -- -- - - - * check if word: w * has synonyms; return list of synonyms * * @param w (string) * @return list of synonyms * * NOTE: in no synonyms exist, retutn the word as a list */ function synonym_keys(w) { w_kb = keyboardize(w); for (i=0 ; i < synonym_kbs.length ; i++) { if (synonym_kbs[i].indexOf(w_kb) !== -1) return synonyms[i]; } return [ w ]; } /** mark_explicit_links * * mark linked words shall be handled as one-(key)word * (also edit common mistakes with suggested replaces) * * @param str * @return */ function mark_explicit_links(str) { str = ' '+ str +' '; replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); }); return str.replaceAll(' ', ' ').trim(); } /** preproccess description * == do... * + clean * + sanitize * + mark lined words */ function preproccess_text(description) { str = clean_text(description); // remove non-keyword characters str = mark_explicit_links(sanitize_GR( str.toLowerCase() )); // mark explicit links // replaces.forEach( lw => { // reg = new RegExp( lw.src, "gi"); // prepare regex for case insensitive (i) replace all (g) // str = str.toLowerCase().replace(reg, lw.trg); // }) // depricated: str = mark_linked_words(str); return str; } /* --- DEPRICATED: // mark linked words (connect them with a dash) // return new text after "all-links" are marked // --- function mark_linked_words(txt) { linkedWords.forEach( lw => { txt = mark_link( lw, txt ); }); return txt } */ /* --- DEPRICATED: // mark a link (lws) to a text (source) // conecting them with a dash/minus character // --- function mark_link(lws, source) { var src_kb = keyboardize(source.replace(' ', '-')); var lws_kb = keyboardize(lws.replace(' ', '-')); var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase()) if (_left !== -1 ) { return source.slice(0, _left) + lws + source.slice(_left + lws.length); } else return source; } */ /** #4 * --- * OUTPUT FUNCTIONS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// // Save Local // ----------------------------------------------------------------------------- async function save_local(jsonArray, filePath) { let fileStr = JSON.stringify(jsonArray); // convert json to string // write string to file fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { if (err) { return err; } else { return true; } }); } // Save to Google Cloud Storage // ----------------------------------------------------------------------------- async function upload_file( bucketName, srcFilePath, trgFilePath ) { // Creates a client const storage = new Storage({projectId, keyFilename}); try { await storage.bucket(bucketName).upload(srcFilePath, { destination: trgFilePath, gzip: true, // serve compressed metadata: { // cache for 8 hours cacheControl: 'public, max-age=60' // production set: 28800 } }); console.log(`${srcFilePath} uploaded to ${bucketName}`); } catch(err) { console.error('ERROR:', err); } } /** #5 * --- * MAIN function (exposed function to run the whole proccess) * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// /** main() * * nodejs's exported function * used as entry-point function (in case of Google Cloud function) * * Operation * + 1: load products and frequencies * + 2: forward to extract_linked_keywords() * + 3: Save results (localy, then to cloud) */ // keep only 1 of next 2 lines // exports.main = () => { // entry point functioon when google cloud function var main = () => { // concise function when direct script // PROMISE ... // fetch all jsons from endpints ........................................... Promise.all([ fetch(products_endpoint, request_settings) .then(res => res.json()) .then((json) => { _pr = json; console.log('products data loaded from emarket api/endpoint'); }), fetch(frequency_endpoint, request_settings) .then(res => res.json()) .then((json) => { _fr = json; console.log('frequencies loaded from CDN') }) ]) .then (() => { // then... // proccess data and extract ketwords console.log(`... All data from endpoints: ${FgGreen}LOADED!${Reset}`); // proccess data extract_linked_keywords(_pr.data); console.log('keywords extracted;\ntry to save results localy...'); // PROMISE ... // save local files .................................................... Promise.all([ save_local(kwlinks_, temp_kw_file) .then(() => { console.log('keywords saced localy');}), save_local(prods_, temp_prod_file) .then(() => { console.log('products saved localy');}) ]) .then(() => { // then ... console.log(`... All local saves: ${FgGreen}DONE!${Reset}`); // PROMISE ... // upload keytword and product jsons into Cloud Storage ............ Promise.all([ upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'), upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json') ]) .then(() => { // then ... console.log(`... Uploads to cloud storage: ${FgGreen}DONE!${Reset}`); // echo usage stats; then END // check // echo 100 most frequent let i = 0; kwlinks_.forEach(rec => { if (i<100) { console.log(i, JSON.stringify(rec.w)) } i++; }); // echo memory stats const used = process.memoryUsage(); for (let key in used) { console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`); } console.log(`${FgGreen}THE END!${Reset}`); }); }); }); } // NOTE: // if running the script directry you need to call the main function // if running through google cloud-functions you do NOT need to call main() // (you need define main() as the entry-point function instead) main();