/** keygen * * this script constucts a linked-wordkeys structure * ----------------------------------------------------------------------------- * * Contents: * #1 Requirements * #2.1 Personalized constants and parametres * #2.2 Purify string functions * #2.3 Preloaded Data * #3 Supporting functions * #4 Output functions * #5 Entry-point function main() */ /** #1 * --- * REQUIREMENTS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// const fs = require('fs'); const os = require('os'); const fetch = require('node-fetch'); const {Storage} = require('@google-cloud/storage'); // Google Cloud Storage /** #2.1 * --- * SETUP PERSONALIZED CONSTANTS AND PARAMETRES * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// // get products FROM api/endpoint parametres // --- -- -- - - - const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json'; const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch"; const request_settings = { method: "Get" }; // google-storage parametres // --- -- -- - - - const projectId = 'pythia-251711'; const keyFilename = '../auth/pythia-251711-047e3d5e6608.json'; // temp files // need to be created prior to save-to-google-storage proccess // because upload_file() uploats an existing local file to G-Storage // --- -- -- - - - const temp_kw_file = '../results/keywords.json'; const temp_prod_file = '../results/products.json'; // Global Variables // ----------------------------------------------------------------------------- // Global arrays // keyword links (word-links dictionary; array of objects) // ----------------------------------------------------------------------------- var kwlinks_ = []; // keyword-links structure (structure to construct) var prods_ = []; // products (array to construct) var _fr =[]; // product frequencies array (as read from endpiont) var _pr =[]; // product array (as read from endpiont) // Console Colors // ----------------------------------------------------------------------------- const Reset = "\x1b[0m"; const FgRed = "\x1b[31m"; const FgGreen = "\x1b[32m"; /** #2.2 * --- * PURIFY STRING FUNCTIONS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// /** keyboardize * ----------------------------------------------------------------------------- * translates string to keyboard-latin keys * (the ones that used whan typing each letter of the word) * * @param str (string): original string (utf8 of latin or greek subgroups) * @return (string): latin/ascii equivalent string */ // cache (keeo in global) any-character to keyboard-latin mapping // --- -- -- - - - var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); const map = new Map(); for (var i=0; i { ap = pair.split(' '); accented_vowels.push({ a: ap[0], // accented p: ap[1] // pure = non accended }); }) // the sanitize_GR function // --- -- -- - - - function sanitize_GR(str) { str = str.toLowerCase(); // replace accended vowels with pure ones accented_vowels.forEach( v => { str = str.replaceAll(v.a, v.p); }); // replace sigma on the end of words str = str + ' '; str = str.replaceAll('σ-', 'ς-'); str = str.replaceAll('σ ', 'ς '); return str; } /** #2.3 * --- * PRELOADED DATA * in the future these data may be loaded from a database / api-endpoint * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// // synonyms //////////////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- var synonyms = []; // groups of synonyms var synonym_kbs = []; // cache kb-formats for performance var synonyms_Originals = [ 'μπίρα μπύρα μπίρες μπύρες', 'αυγά αβγά αυγό', 'σίκαλης σικάλεως', 'ξηρά ξερά', 'ρολό ρολλό', 'coca-cola cocacola coke', 'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας', 'χαρτί-κουζίνας ρολό-κουζίνας', 'οινος κρασι', 'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ', 'DR-OETKER OETKER', 'DR.BECKMANN BECKMANN', 'NES-CAFE NESCAFE', 'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής', 'τσίπουρο ρακή', 'Βρώμη Βρώμης', 'Φράουλα Φράουλες Φράουλας', 'Μαλλιά Μαλλιών', 'Κέικ Cake', 'CRETA-FARMS CRETA-FARM', 'MARSEILLAIS LE-PETIT-MARSEILLAIS PETIT-MARSEILLAIS', 'Γαϊδούρας Γαϊδάρου', 'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ', 'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ', 'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ', 'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ', 'Ντομάτα Ντομάτας Τομάτα', 'Ελαφρύ Ελαφρά Light', 'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά', 'τριμμένη τριμμένο τριμμένα', 'Τόνος Τόνου', 'Κριθαρένια κρίθινα', 'Χωρίς-Kαφεϊνη Decaffeine', 'Το-Μάννα Μάννα', 'Κράνμπερι Κράνμπερις', 'Κρήτης Κρητικό Κρητική', 'Πέννες Πένες', 'σοκολατένιος σοκολατένια σοκολατένιο', 'Μακαρόνια Ζυμαρικά Σπαγγέτι Σπαγγετίνι Σπαγγετόνι', 'Καρτέλλα Καρτέλα Καρτέλλες' ] synonyms_Originals.forEach( grp => { // remove accends so any replaces can be done flawlessly grp = sanitize_GR(grp).trim(); // push it to array of synonyms (in lowerCase); synonyms.push( grp.toLowerCase().split(' ') ); // push it the keyboardized array of synonyms synonym_kbs.push( keyboardize(grp).toLowerCase().split(' ') ); }) // console.log(synonyms); process.exit(); // console.log(synonym_kbs); process.exit(); // significant terms /////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- significantExceptios = sanitize_GR('7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ') // replaces (correcting descriptions) // = construct unequivocally liked words /////////////////////////////////////// // ----------------------------------------------------------------------------- // NOTE: TODO: // in future implementations multi-word keywords // may use the non-breaking space as conecting character (\u00A0) instead of dush (-) // (or maybe both of them) // --- -- -- - - - replaces = []; replaceSource = [ '3 ΑΛΦΑ;3-ΑΛΦΑ', 'HEAD & SHOULDERS;HEAD&SHOULDERS', 'HEAD N SHOULDERS;HEAD&SHOULDERS', 'W.K Kellogg; W-K-Kellogg', 'W.K Kellogg;', '7 DAYS;7-DAYS', '7 UP;7-UP', 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', 'COCA COLA;COCA-COLA', 'COCACOLA;COCA-COLA', // 'KELLOGS;KELLOGG\'S', 'NES CAFE;NESCAFE', 'NES-CAFE;NESCAFE', 'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS', 'PETIT MARSEILLAIS;PETIT-MARSEILLAIS', 'Το Μάννα;Το-Μάννα', 'Χωρίς Γλουτένη;Χωρίς-Γλουτένη', 'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη', 'Χωρίς Αλάτι;Χωρίς-Αλάτι', 'Χωρίς Λακτόζη;Χωρίς-Λακτόζη', 'Χωρίς Συντηρητικά;Χωρίς-Συντηρητικά', 'Χωρίς Αλκοόλ;Χωρίς-Αλκοόλ', 'Χωρίς Kαφεϊνη;Χωρίς-Kαφεϊνη', 'Χωρίς Γλυκάνισο;Χωρίς-Γλυκάνισο', 'Χωρίς Ανθρακικό;Χωρίς-Ανθρακικό', 'Υψηλής Παστερίωσης;Υψηλής-Παστερίωσης', 'Ολικής Άλεσης;Ολικής-Άλεσης', 'Ολικής Aλέσεως;Ολικής-Aλέσεως', 'Χαρτί Υγείας;Χαρτί-Υγείας', 'ρολό υγείας;ρολό-υγείας', 'χαρτί τουαλέτας;χαρτί-τουαλέτας', 'Χαρτί Κουζίνας;Χαρτί-Κουζίνας', 'ρολό κουζίνας;ρολό-κουζίνας', 'Μπάρες Δημητριακών;Μπάρες-Δημητριακών', 'Ας Μαγειρέψουμε;Ας-Μαγειρέψουμε', 'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', 'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', 'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ', 'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ', 'ΕΛ ΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', 'ΕΛΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', 'FREE STEP;FREE-STEP', 'EL SABOR;EL-SABOR', 'ELSABOR;EL-SABOR', 'DOUWE EGBERTS;DOUWE-EGBERTS', 'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', 'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', 'SPIN SPAN;SPIN-SPAN', 'SPINSPAN;SPIN-SPAN', 'CRETA-FARMS;CRETA-FARM', 'CRETA-FARM;CRETA-FARM', 'CRETAFARM;CRETA-FARM', 'Ολες-τις-Χρήσεις;Ολες-τις-Χρήσεις', 'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη' ]; replaceSource.forEach( it => { st = sanitize_GR( it.toLowerCase() ).split(';'); replaces.push({ src: ' '+ st[0] +' ', // encolse between spaces trg: ' '+ st[1] +' ' // to separate from before/after words }); }); // console.log(replaces); process.exit(); // words that shall not be searched first ////////////////////////////////////// // ----------------------------------------------------------------------------- var noRootKeywords = []; noRoot = [ 'χωρίς', 'εισαγωγής', 'δώρο', 'γεύση', 'γεύσεις', 'φέτες', 'Χωρίς-Γλουτένη', 'Γλουτένη', 'Λακτόζη', 'Παστερίωσης', 'Ανθρακικό', 'Συντηρητικά', 'Χωρίς-Ζάχαρη', 'Άλεσης', 'Χωρίς-Αλάτι', 'Χωρίς-Λακτόζη', 'Χωρίς-Συντηρητικά', 'Χωρίς-Αλκοόλ', 'Χωρίς-Kαφεϊνη', 'Χωρίς-Γλυκάνισο', 'Χωρίς-Ανθρακικό', 'Υψηλής-Παστερίωσης', 'Ολες-τις-Χρήσεις', 'Ολικής-Άλεσης', 'Ολικής-Aλέσεως', 'Ολικής', 'Σοκολάτας', 'Υγείας', 'Γαϊδούρας', 'Δημητριακών', 'Γαϊδάρου', 'Ρούχων', 'Πιάτων', 'πλύσεις', 'Πλυντηρίου', 'Φύλλων', 'Γάλακτος', 'Χρήσης', 'Τύπου', 'Ολλανδίας', 'Απορριμμάτων', 'Medium', 'Μαλλιά', 'Μαλλιών', 'Γενικής', 'Plus', 'Classic', 'Έκπληξη', 'Μάνης', 'Ελάτου', 'Άγριων', 'Βοτάνων', 'Κατσαρίδες', 'Λακωνίας', 'ΠΑΡΑΓΓΕΛΙΩΝ', 'Λήμνου', 'Αργινίνης', 'Καλαθάκι', 'Σκύλου', 'Γάτας', 'ΠΓΕ', 'ΤΟ', 'Μελί', 'Σώματος', 'Αδυνατίσματος', 'Προστασίας', 'Ακράτειας', 'Προσώπου', 'προσθήκη', 'Βρεφικής', 'Καθαρισμού', 'Ρούχων' ]; noRoot.forEach( w => { noRootKeywords.push(keyboardize(w)); }); // list of linked-words //////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- linkedWords = [ sanitize_GR('Χωρίς-Γλουτένη'), sanitize_GR('Χωρίς-Ζάχαρη'), sanitize_GR('Χωρίς-Αλάτι'), sanitize_GR('Χωρίς-Λακτόζη'), sanitize_GR('Χωρίς-Συντηρητικά'), sanitize_GR('Χωρίς-Αλκοόλ'), sanitize_GR('Χωρίς-Kαφεϊνη'), sanitize_GR('Χωρίς-Γλυκάνισο'), sanitize_GR('Χωρίς-Ανθρακικό'), sanitize_GR('Υψηλής-Παστερίωσης'), sanitize_GR('Ολικής-Άλεσης'), sanitize_GR('Ολικής-Aλέσεως'), sanitize_GR('Χαρτί-Υγείας'), sanitize_GR('ρολό-υγείας'), sanitize_GR('χαρτί-τουαλέτας'), sanitize_GR('Χαρτί-Κουζίνας'), sanitize_GR('Μπάρες-Δημητριακών'), sanitize_GR('Μπαρμπα-Στάθης'), sanitize_GR('COCA-COLA'), sanitize_GR('Aς-Μαγειρέψουμε'), sanitize_GR('ΚΡΙΣ-ΚΡΙΣ'), sanitize_GR('ΚΡΙ-ΚΡΙ'), sanitize_GR('ΕΛ-ΓΚΡΕΚΟ'), sanitize_GR('FREE-STEP'), sanitize_GR('EL-SABOR'), sanitize_GR('LE-PETIT-MARSEILLAIS'), sanitize_GR('DOUWE-EGBERTS'), sanitize_GR('ΕΝ-ΕΛΛΑΔΙ'), sanitize_GR('SPIN-SPAN'), sanitize_GR('CRETA-FARMS'), sanitize_GR('CRETA-FARM'), sanitize_GR('NES-CAFE'), sanitize_GR('Ολες-τις-Χρήσεις'), sanitize_GR('Το-Μάννα'), sanitize_GR('Χωρίς-προσθήκη-ζάχαρης') ]; // list of words to exclude from keywords ////////////////////////////////////// // ----------------------------------------------------------------------------- // NOTE: APPLIED in PER-WORD base -> after spliting description to words removeList = [] removeOriginals = sanitize_GR( 'κατά παρά υπό από μετά προς μας με σε για του της των από στο το στον & r s ft l τ e g h k m n o p s x' ) .split(' ') removeOriginals.forEach( w => { removeList.push( keyboardize(w)); }); /** #3 * --- * SUPPORTING FUNCTIONS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// /** freq * * @param id (int): product's eys_code * @return frequency of product */ function freq_of(id) { _fr.forEach( rec => { if (rec.id == id) return rec.fq }); return 0; } /** proccess data * --- -- -- - - - * * central proccessing procedure * ----------------------------------------------------------------------------- */ function extract_linked_keywords(obj) { // console.log(JSON.stringify(obj, null, 2)); obj.forEach( rec => { if (rec.img != 0) { //var description = preproccess_text(rec.txt); var description = clean_text(rec.txt); var pid = rec.eys; var fq = freq_of(pid)+1; prods_.push({ id: pid, w: description.trim() }) var keys = []; var words = preproccess_text(description).split(' '); // filter words; keep only significant words.forEach( w => { if (removeList.indexOf(keyboardize(w)) == -1) // if not excluded if (is_significant(w)) // and significant keys.push(w); // add it to keys }); // console.log(pid, description, keys); keys.forEach( w => { var wl = synonym_keys(w); // if key CAN be a root word (not a no-Root-keyword) update root-node if (noRootKeywords.indexOf(keyboardize(w)) == -1) { root_key(wl, fq); // update root keyword stats // if key is the only in the list of product's keywords // connect it with a dummy key (to preserve the reference to the product) if (keys.length == 1) connect_keys(wl, ['*'], pid, fq); // connect w with all the other product's keywords keys.forEach( w2 => { if (w2 != w) { var w2syns = synonym_keys(w2); connect_keys( wl, w2syns, pid, fq); } }); } }); } }); // main proccessing finished; // post proccess // ------------------------------------------------------------------------- // remove cached keys from final array kwlinks_.forEach( ro => { delete ro.kb; ro.c.forEach( ch => { delete ch.kb; }); }); // sort root and child nodes by frequency descanding kwlinks_.forEach( it => { it.c = it.c.sort((a, b) => b.f - a.f ); }); kwlinks_ = kwlinks_.sort((a, b) => b.f - a.f ); } // functions for linking words in keywords dictionary //////////////////////////////////////////////////////////////////////////////// /** root_key( wl, f ) * --- -- -- - - - * set root keyword: wl (if not exist) * update frequency: f * * @param wl (list) :a list of synonym-words * @param f (int): frequency of product * * ** comparison is based on the *keyboard* format */ function root_key ( wl, f ) { var wkb = keyboardize(wl[0]) // cache kb format // check if exists in root keys already // NOTE: you only need to check the 1st word of synonyms-list for (i=0; i< kwlinks_.length ; i++) { if (kwlinks_[i].kb == wkb) { kwlinks_[i].f += f; return true; } } // if not exists, append keyword kwlinks_.push({ w : wl, kb : wkb, f : f, c : [] }); return true; } /** connect_keys * --- -- -- - - - * @param (list) : connect keyword a (synonyms list) * @param (list) : with keyword b (synonyms list) * @param (int) : id (eys code) of product where keywords a and b are linked * @param (int) : frequency of product (now many times has been purchased) * @returns true */ function connect_keys( a, b, id, f ) { var kbA = keyboardize(a[0]); var kbB = keyboardize(b[0]); var bExists = false; if (kbA == kbB) return false; // exclude just-in-case for (i=0; i< kwlinks_.length ; i++) { if (kwlinks_[i].kb == kbA) { // found: a; // update connection to: b bExists = false; for (j=0 ; j < kwlinks_[i].c.length ; j++) { if (kwlinks_[i].c[j].kb == kbB) { bExists = true; // update the connection's data kwlinks_[i].c[j].f += f; kwlinks_[i].c[j].p.push(id) break; } } // if connection not exist, init a new one if (bExists == false) { // create connection with: b kwlinks_[i].c.push({ w : b, kb : kbB, f : f, p : [ id ] }); } return true; } } } // other supplementary functions //////////////////////////////////////////////////////////////////////////////// /** clean text * --- -- -- - - - * removes non keyword characters [+ . , !] and internal multiple-spaces * @param txt (string): product description */ function clean_text(txt) { return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space .replace('!','').replace('\"', '') // remove character .replace(' ',' ').replace(' ',' '); // remove multiple spaces } /** is significant * --- -- -- - - - * check if term is significant (and needs to be indexed) * (if not, the term will be excluded from keywords dicionary) * * @param str * @return (boolean) */ function is_significant(str) { if (str == '') return false; if (significantExceptios.indexOf(str) !== -1) return true; return !(/\d/.test(str)); } /** synonym_keys * --- -- -- - - - * check if word: w * has synonyms; return list of synonyms * * @param w (string) * @return list of synonyms * * NOTE: in no synonyms exist, retutn the word as a list */ function synonym_keys(w) { w_kb = keyboardize(w); for (i=0 ; i < synonym_kbs.length ; i++) { if (synonym_kbs[i].indexOf(w_kb) !== -1) return synonyms[i]; } return [ w ]; } /** mark_explicit_links * * mark linked words * shall be handled as one-(key)word * * @param str * @return */ // edit common mistakes // with suggested replaces function mark_explicit_links(str) { str = ' '+ str +' '; replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); }); return str.replaceAll(' ', ' ').trim(); } // preproccess description // --- function preproccess_text(description) { str = clean_text(description); // remove non-keyword characters str = mark_explicit_links(sanitize_GR( str.toLowerCase() )); // mark explicit links // replaces.forEach( lw => { // reg = new RegExp( lw.src, "gi"); // prepare regex for case insensitive (i) replace all (g) // str = str.toLowerCase().replace(reg, lw.trg); // }) // depricated: str = mark_linked_words(str); return str; } /* --- DEPRICATED: // mark linked words (connect them with a dash) // return new text after "all-links" are marked // --- function mark_linked_words(txt) { linkedWords.forEach( lw => { txt = mark_link( lw, txt ); }); return txt } */ /* --- DEPRICATED: // mark a link (lws) to a text (source) // conecting them with a dash/minus character // --- function mark_link(lws, source) { var src_kb = keyboardize(source.replace(' ', '-')); var lws_kb = keyboardize(lws.replace(' ', '-')); var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase()) if (_left !== -1 ) { return source.slice(0, _left) + lws + source.slice(_left + lws.length); } else return source; } */ /** #4 * --- * OUTPUT FUNCTIONS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// // Save Local // ----------------------------------------------------------------------------- async function save_local(jsonArray, filePath) { let fileStr = JSON.stringify(jsonArray); // convert json to string // write string to file fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { if (err) { return err; } else { return true; } }); } // Save to Google Cloud Storage // ----------------------------------------------------------------------------- async function upload_file( bucketName, srcFilePath, trgFilePath ) { // Creates a client const storage = new Storage({projectId, keyFilename}); try { await storage.bucket(bucketName).upload(srcFilePath, { destination: trgFilePath, gzip: true, // serve compressed metadata: { // cache for 8 hours cacheControl: 'public, max-age=60' // production set: 28800 } }); console.log(`${srcFilePath} uploaded to ${bucketName}`); } catch(err) { console.error('ERROR:', err); } } /** #5 * --- * MAIN function (exposed function to run the whole proccess) * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// /** main() * * nodejs's exported function * used as entry-point function (in case of Google Cloud function) */ // keep only 1 of next 2 lines // exports.main = () => { // entry point functioon when google cloud function var main = () => { // concise function when direct script // PROMISE ... // fetch all jsons from endpints ........................................... Promise.all([ fetch(products_endpoint, request_settings) .then(res => res.json()) .then((json) => { _pr = json; console.log('products data loaded from emarket api/endpoint'); }), fetch(frequency_endpoint, request_settings) .then(res => res.json()) .then((json) => { _fr = json; console.log('frequencies loaded from CDN') }) ]) .then (() => { // then... // proccess data and extract ketwords console.log(`... All data from endpoints: ${FgGreen}LOADED!${Reset}`); // proccess data extract_linked_keywords(_pr.data); console.log('keywords extracted;\ntry to save results localy...'); // PROMISE ... // save local files .................................................... Promise.all([ save_local(kwlinks_, temp_kw_file) .then(() => { console.log('keywords saced localy');}), save_local(prods_, temp_prod_file) .then(() => { console.log('products saved localy');}) ]) .then(() => { // then ... console.log(`... All local saves: ${FgGreen}DONE!${Reset}`); // PROMISE ... // upload keytword and product jsons into Cloud Storage ............ Promise.all([ upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'), upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json') ]) .then(() => { // then ... console.log(`... Uploads to cloud storage: ${FgGreen}DONE!${Reset}`); // echo usage stats; then END // check // echo 100 most frequent let i = 0; kwlinks_.forEach(rec => { if (i<100) { console.log(i, JSON.stringify(rec.w)) } i++; }); // echo memory stats const used = process.memoryUsage(); for (let key in used) { console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`); } console.log(`${FgGreen}THE END!${Reset}`); }); }); }); } // NOTE: // if running the script directry you need to call the main function // if running through google cloud-functions you do NOT need to call main() // (you need define main() as the entry-point function instead) main();