/** keygen * * this script constucts a linked-wordkeys structure * ----------------------------------------------------------------------------- * * Contents: * #1 Requirements * #2.1 Personalized constants and parametres * #2.2 Purify string functions * #2.3 Preloaded Data * #3 Supporting functions * #4 Output functions * #5 Entry-point function main() */ /** #1 * --- * REQUIREMENTS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// const fs = require('fs'); const os = require('os'); const fetch = require('node-fetch'); const {Storage} = require('@google-cloud/storage'); // Google Cloud Storage /** #2.1 * --- * SETUP PERSONALIZED CONSTANTS AND PARAMETRES * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// // get products FROM api/endpoint parametres // --- -- -- - - - const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json'; const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch"; const request_settings = { method: "Get" }; // google-storage parametres // --- -- -- - - - const projectId = 'pythia-251711'; const keyFilename = '../auth/pythia-251711-047e3d5e6608.json'; // temp files // need to be created prior to save-to-google-storage proccess // because upload_file() uploats an existing local file to G-Storage // --- -- -- - - - const temp_kw_file = '../results/keywords.json'; const temp_prod_file = '../results/products.json'; // Global Variables // ----------------------------------------------------------------------------- // Global arrays // keyword links (word-links dictionary; array of objects) // ----------------------------------------------------------------------------- var kwlinks_ = []; // keyword-links structure (structure to construct) var prods_ = []; // products (array to construct) var _fr =[]; // product frequencies array (as read from endpiont) var _pr =[]; // product array (as read from endpiont) // Console Colors // ----------------------------------------------------------------------------- const Reset = "\x1b[0m"; const FgRed = "\x1b[31m"; const FgGreen = "\x1b[32m"; /** #2.2 * --- * PURIFY STRING FUNCTIONS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// /** keyboardize * ----------------------------------------------------------------------------- * translates string to keyboard-latin keys * (the ones that used whan typing each letter of the word) * * @param str (string): original string (utf8 of latin or greek subgroups) * @return (string): latin/ascii equivalent string */ // cache (keeo in global) any-character to keyboard-latin mapping // --- -- -- - - - var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); const map = new Map(); for (var i=0; i { ap = pair.split(' '); accented_vowels.push({ a: ap[0], // accented p: ap[1] // pure = non accended }); }) // the sanitize_GR function // --- -- -- - - - function sanitize_GR(str) { str = str.toLowerCase(); // replace accended vowels with pure ones accented_vowels.forEach( v => { str = str.replaceAll(v.a, v.p); }); // replace sigma on the end of words str = str + ' '; str = str.replaceAll('σ-', 'ς-'); str = str.replaceAll('σ ', 'ς '); return str.trim(); } /** #2.3 * --- * PRELOADED DATA * in the future these data may be loaded from a database / api-endpoint * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// // synonyms //////////////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- var synonyms = []; // groups of synonyms var synonym_kbs = []; // cache kb-formats for performance var synonyms_Originals = [ 'μπίρα μπύρα μπίρες μπύρες', 'αυγά αβγά αυγό', 'σίκαλης σικάλεως', 'ξηρά ξερά', 'ρολό ρολλό', 'coca-cola cocacola coke', 'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας', 'χαρτί-κουζίνας ρολό-κουζίνας', 'οινος κρασι', 'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ', 'DR-OETKER OETKER', 'DR.BECKMANN BECKMANN', 'NES-CAFE NESCAFE', 'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής', 'τσίπουρο ρακή', 'Βρώμη Βρώμης', 'Φράουλα Φράουλες Φράουλας', 'Μαλλιά Μαλλιών', 'Κέικ Cake', 'CRETA-FARMS CRETA-FARM', 'MARSEILLAIS LE-PETIT-MARSEILLAIS PETIT-MARSEILLAIS', 'Γαϊδούρας Γαϊδάρου', 'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ', 'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ', 'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ', 'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ', 'Ντομάτα Ντομάτας', 'Ελαφρύ Ελαφρά Light', 'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά', 'τριμμένη τριμμένο', 'Τόνος Τόνου', 'Κριθαρένια κρίθινα', 'Χωρίς-Kαφεϊνη Decaffeine', 'Το-Μάννα Μάννα', 'Κράνμπερι Κράνμπερις', 'Κρήτης Κρητικό', 'Πέννες Πένες', 'Μακαρόνια Ζυμαρικά Σπαγγέτι Σπαγγετίνι Σπαγγετόνι', 'Καρτέλλα Καρτέλα Καρτέλλες' ] synonyms_Originals.forEach( grp => { // remove accends so any replaces can be done flawlessly grp = sanitize_GR(grp); // create array of synonyms; // push it to sanitized and keyboardized arrays synonyms.push( grp.split(' ') ); synonym_kbs.push( keyboardize(grp).split(' ') ); }) // significant terms /////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- significantExceptios = sanitize_GR('7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ') // replaces (correcting descriptions) // = construct unequivocally liked words /////////////////////////////////////// // ----------------------------------------------------------------------------- // NOTE: TODO: // in future implementations multi-word keywords // may use the non-breaking space as conecting character (\u00A0) instead of dush (-) // (or maybe both of them) // --- -- -- - - - replaces = []; replaceSource = [ '3 ΑΛΦΑ ;3-ΑΛΦΑ ', 'HEAD & SHOULDERS ;HEAD&SHOULDERS ', 'HEAD N SHOULDERS ;HEAD&SHOULDERS ', 'W.K Kellogg ; W-K-Kellogg ', 'W.K Kellogg ; ', '7 DAYS ;7-DAYS ', '7 UP ;7-UP ', 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ ', 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ ', 'COCA COLA ;COCA-COLA', 'COCACOLA ;COCA-COLA', 'KELLOGS ;KELLOGG\'S', 'NES CAFE ;NESCAFE', 'NES-CAFE ;NESCAFE', 'LE PETIT MARSEILLAIS ;LE-PETIT-MARSEILLAIS', 'PETIT MARSEILLAIS ;PETIT-MARSEILLAIS', 'Το Μάννα ;Το-Μάννα', 'Χωρίς Γλουτένη ;Χωρίς-Γλουτένη', 'Χωρίς Ζάχαρη ;Χωρίς-Ζάχαρη', 'Χωρίς Αλάτι ;Χωρίς-Αλάτι', 'Χωρίς Λακτόζη ;Χωρίς-Λακτόζη', 'Χωρίς Συντηρητικά ;Χωρίς-Συντηρητικά', 'Χωρίς Αλκοόλ ;Χωρίς-Αλκοόλ', 'Χωρίς Kαφεϊνη ;Χωρίς-Kαφεϊνη', 'Χωρίς Γλυκάνισο ;Χωρίς-Γλυκάνισο', 'Χωρίς Ανθρακικό ;Χωρίς-Ανθρακικό', 'Υψηλής Παστερίωσης ;Υψηλής-Παστερίωσης', 'Ολικής Άλεσης ;Ολικής-Άλεσης', 'Ολικής Aλέσεως ;Ολικής-Aλέσεως', 'Χαρτί Υγείας ;Χαρτί-Υγείας', 'ρολό υγείας ;ρολό-υγείας', 'χαρτί τουαλέτας ;χαρτί-τουαλέτας', 'Χαρτί Κουζίνας ;Χαρτί-Κουζίνας', 'ρολό κουζίνας ;ρολό-κουζίνας', 'Μπάρες Δημητριακών ;Μπάρες-Δημητριακών', 'Aς Μαγειρέψουμε ;Aς-Μαγειρέψουμε', 'ΚΡΙΣ ΚΡΙΣ ;ΚΡΙΣ-ΚΡΙΣ', 'ΚΡΙΣΚΡΙΣ ;ΚΡΙΣ-ΚΡΙΣ', 'ΚΡΙ ΚΡΙ ;ΚΡΙ-ΚΡΙ', 'ΚΡΙΚΡΙ ;ΚΡΙ-ΚΡΙ', 'ΕΛ ΓΚΡΕΚΟ ;ΕΛ-ΓΚΡΕΚΟ', 'ΕΛΓΚΡΕΚΟ ;ΕΛ-ΓΚΡΕΚΟ', 'FREE STEP ;FREE-STEP', 'EL SABOR ;EL-SABOR', 'ELSABOR ;EL-SABOR', 'DOUWE EGBERTS ;DOUWE-EGBERTS', 'ΕΝ ΕΛΛΑΔΙ ;ΕΝ-ΕΛΛΑΔΙ', 'ΕΝΕΛΛΑΔΙ ;ΕΝ-ΕΛΛΑΔΙ', 'SPIN SPAN ;SPIN-SPAN', 'SPINSPAN ;SPIN-SPAN', 'CRETA-FARMS ;CRETA-FARM', 'CRETA-FARM ;CRETA-FARM', 'CRETAFARM ;CRETA-FARM', 'Ολες-τις-Χρήσεις ;Ολες-τις-Χρήσεις', 'Χωρίς προσθήκη ζάχαρης ;Χωρίς-ζάχαρη' ]; replaceSource.forEach( it => { st = sanitize_GR( it ).split(';'); replaces.push({ src: st[0], trg: st[1] }); }); // words that shall not be searched first ////////////////////////////////////// // ----------------------------------------------------------------------------- var noRootKeywords = []; noRoot = [ 'χωρίς', 'εισαγωγής', 'δώρο', 'γεύση', 'γεύσεις', 'φέτες', 'Χωρίς-Γλουτένη', 'Γλουτένη', 'Λακτόζη', 'Παστερίωσης', 'Ανθρακικό', 'Συντηρητικά', 'Χωρίς-Ζάχαρη', 'Άλεσης', 'Χωρίς-Αλάτι', 'Χωρίς-Λακτόζη', 'Χωρίς-Συντηρητικά', 'Χωρίς-Αλκοόλ', 'Χωρίς-Kαφεϊνη', 'Χωρίς-Γλυκάνισο', 'Χωρίς-Ανθρακικό', 'Υψηλής-Παστερίωσης', 'Ολες-τις-Χρήσεις', 'Ολικής-Άλεσης', 'Ολικής-Aλέσεως', 'Ολικής', 'Σοκολάτας', 'Υγείας', 'Γαϊδούρας', 'Δημητριακών', 'Γαϊδάρου', 'Ρούχων', 'Πιάτων', 'πλύσεις', 'Πλυντηρίου', 'Φύλλων', 'Γάλακτος', 'Χρήσης', 'Τύπου', 'Ολλανδίας', 'Απορριμμάτων', 'Medium', 'Μαλλιά', 'Μαλλιών', 'Γενικής', 'Plus', 'Classic', 'Έκπληξη', 'Μάνης', 'Ελάτου', 'Άγριων', 'Βοτάνων', 'Κατσαρίδες', 'Λακωνίας', 'ΠΑΡΑΓΓΕΛΙΩΝ', 'Λήμνου', 'Αργινίνης', 'Καλαθάκι', 'Σκύλου', 'Γάτας', 'ΠΓΕ', 'ΤΟ', 'Μελί', 'Σώματος', 'Αδυνατίσματος', 'Προστασίας', 'Ακράτειας', 'Προσώπου', 'προσθήκη', 'Βρεφικής', 'Καθαρισμού', 'Ρούχων' ]; noRoot.forEach( w => { noRootKeywords.push(keyboardize(w)); }); // list of linked-words //////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- linkedWords = [ sanitize_GR('Χωρίς-Γλουτένη'), sanitize_GR('Χωρίς-Ζάχαρη'), sanitize_GR('Χωρίς-Αλάτι'), sanitize_GR('Χωρίς-Λακτόζη'), sanitize_GR('Χωρίς-Συντηρητικά'), sanitize_GR('Χωρίς-Αλκοόλ'), sanitize_GR('Χωρίς-Kαφεϊνη'), sanitize_GR('Χωρίς-Γλυκάνισο'), sanitize_GR('Χωρίς-Ανθρακικό'), sanitize_GR('Υψηλής-Παστερίωσης'), sanitize_GR('Ολικής-Άλεσης'), sanitize_GR('Ολικής-Aλέσεως'), sanitize_GR('Χαρτί-Υγείας'), sanitize_GR('ρολό-υγείας'), sanitize_GR('χαρτί-τουαλέτας'), sanitize_GR('Χαρτί-Κουζίνας'), sanitize_GR('Μπάρες-Δημητριακών'), sanitize_GR('Μπαρμπα-Στάθης'), sanitize_GR('COCA-COLA'), sanitize_GR('Aς-Μαγειρέψουμε'), sanitize_GR('ΚΡΙΣ-ΚΡΙΣ'), sanitize_GR('ΚΡΙ-ΚΡΙ'), sanitize_GR('ΕΛ-ΓΚΡΕΚΟ'), sanitize_GR('FREE-STEP'), sanitize_GR('EL-SABOR'), sanitize_GR('LE-PETIT-MARSEILLAIS'), sanitize_GR('DOUWE-EGBERTS'), sanitize_GR('ΕΝ-ΕΛΛΑΔΙ'), sanitize_GR('SPIN-SPAN'), sanitize_GR('CRETA-FARMS'), sanitize_GR('CRETA-FARM'), sanitize_GR('NES-CAFE'), sanitize_GR('Ολες-τις-Χρήσεις'), sanitize_GR('Το-Μάννα'), sanitize_GR('Χωρίς-προσθήκη-ζάχαρης') ]; // list of words to exclude from keywords ////////////////////////////////////// // ----------------------------------------------------------------------------- // NOTE: APPLIED in PER-WORD base -> after spliting description to words removeList = [] removeOriginals = sanitize_GR( 'κατά παρά υπό από μετά προς μας με σε για του της των από στο το στον & r s ft l τ e g h k m n o p s x' ) .split(' ') removeOriginals.forEach( w => { removeList.push( keyboardize(w)); }); /** #3 * --- * SUPPORTING FUNCTIONS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// /** freq * * @param id (int): product's eys_code * @return frequency of product */ function freq_of(id) { _fr.forEach( rec => { if (rec.id == id) return rec.fq }); return 0; } /** proccess data * --- -- -- - - - * * central proccessing procedure * ----------------------------------------------------------------------------- */ function extract_linked_keywords(obj) { // console.log(JSON.stringify(obj, null, 2)); obj.forEach( rec => { if (rec.img != 0) { var description = preproccess_text(rec.txt); var pid = rec.eys; var fq = freq_of(pid)+1; prods_.push({ id: pid, w: description }) var keys = []; var words = description.split(' '); // filter words; keep only significant words.forEach( w => { if (removeList.indexOf(keyboardize(w)) == -1) // if not excluded if (is_significant(w)) // and significant keys.push(w); // add it to keys }); // console.log(pid, description, keys); keys.forEach( w => { var wl = synonym_keys(w); // if key CAN be a root word (not a no-Root-keyword) update root-node if (noRootKeywords.indexOf(keyboardize(w)) == -1) { root_key(wl, fq); // update root keyword stats // if key is the only in the list of product's keywords // connect it with a dummy key (to preserve the reference to the product) if (keys.length == 1) connect_keys(wl, ['*'], pid, fq); // connect w with all the other product's keywords keys.forEach( w2 => { if (w2 != w) { var w2syns = synonym_keys(w2); connect_keys( wl, w2syns, pid, fq); } }); } }); } }); // main proccessing finished; // post proccess // ------------------------------------------------------------------------- // remove cached keys from final array kwlinks_.forEach( ro => { delete ro.kb; ro.c.forEach( ch => { delete ch.kb; }); }); // sort root and child nodes by frequency descanding kwlinks_.forEach( it => { it.c = it.c.sort((a, b) => b.f - a.f ); }); kwlinks_ = kwlinks_.sort((a, b) => b.f - a.f ); } // functions for linking words in keywords dictionary //////////////////////////////////////////////////////////////////////////////// /** root_key( wl, f ) * --- -- -- - - - * set root keyword: wl (if not exist) * update frequency: f * * @param wl (list) :a list of synonym-words * @param f (int): frequency of product * * ** comparison is based on the *keyboard* format */ function root_key ( wl, f ) { var wkb = keyboardize(wl[0]) // cache kb format // check if exists in root keys already // NOTE: you only need to check the 1st word of synonyms-list for (i=0; i< kwlinks_.length ; i++) { if (kwlinks_[i].kb == wkb) { kwlinks_[i].f += f; return true; } } // if not exists, append keyword kwlinks_.push({ w : wl, kb : wkb, f : f, c : [] }); return true; } /** connect_keys * --- -- -- - - - * @param (list) : connect keyword a (synonyms list) * @param (list) : with keyword b (synonyms list) * @param (int) : id (eys code) of product where keywords a and b are linked * @param (int) : frequency of product (now many times has been purchased) * @returns true */ function connect_keys( a, b, id, f ) { var kbA = keyboardize(a[0]); var kbB = keyboardize(b[0]); var bExists = false; if (kbA == kbB) return false; // exclude just-in-case for (i=0; i< kwlinks_.length ; i++) { if (kwlinks_[i].kb == kbA) { // found: a; // update connection to: b bExists = false; for (j=0 ; j < kwlinks_[i].c.length ; j++) { if (kwlinks_[i].c[j].kb == kbB) { bExists = true; // update the connection's data kwlinks_[i].c[j].f += f; kwlinks_[i].c[j].p.push(id) break; } } // if connection not exist, init a new one if (bExists == false) { // create connection with: b kwlinks_[i].c.push({ w : b, kb : kbB, f : f, p : [ i ] }); } return true; } } } // other supplementary functions //////////////////////////////////////////////////////////////////////////////// /** clean text * --- -- -- - - - * removes non keyword characters [\+, \., \'] and internal multiple-spaces * @param txt (string): product description */ function clean_text(txt) { return sanitize_GR(txt).replace('+',' ').replace('.',' ') .replace(' ',' ') .replace(' ',' '); } /** is significant * --- -- -- - - - * check if term is significant (and needs to be indexed) * (if not, the term will be excluded from keywords dicionary) * * @param str * @return (boolean) */ function is_significant(str) { if (str == '') return false; if (significantExceptios.indexOf(str) !== -1) return true; return !(/\d/.test(str)); } /** synonym_keys * --- -- -- - - - * check if word: w * has synonyms; return list of synonyms * * @param w (string) * @return list of synonyms * * NOTE: in no synonyms exist, retutn the word as a list */ function synonym_keys(w) { w_kb = keyboardize(w); for (i=0 ; i < synonym_kbs.length ; i++) { if (synonym_kbs.indexOf(w_kb) !== -1) return synonyms[i]; } return [ w ]; } /** mark_explicit_links * * mark linked words * shall be handled as one-(key)word * * @param str * @return */ // edit common mistakes // with suggested replaces function mark_explicit_links(str) { replaces.forEach( it => { str = str.replace(it.src, it.trg); }); return str; } // preproccess description // --- function preproccess_text(str) { str = clean_text(str); // remove non-keyword characters str = mark_explicit_links(sanitize_GR(str)); // mark explicit links replaces.forEach( lw => { reg = new RegExp( lw.src, "gi"); // prepare regex for case insensitive (i) replace all (g) str = str.toLowerCase().replace(reg, lw.trg); }) // depricated: str = mark_linked_words(str); return str; } /* --- DEPRICATED: // mark linked words (connect them with a dash) // return new text after "all-links" are marked // --- function mark_linked_words(txt) { linkedWords.forEach( lw => { txt = mark_link( lw, txt ); }); return txt } */ /* --- DEPRICATED: // mark a link (lws) to a text (source) // conecting them with a dash/minus character // --- function mark_link(lws, source) { var src_kb = keyboardize(source.replace(' ', '-')); var lws_kb = keyboardize(lws.replace(' ', '-')); var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase()) if (_left !== -1 ) { return source.slice(0, _left) + lws + source.slice(_left + lws.length); } else return source; } */ /** #4 * --- * OUTPUT FUNCTIONS * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// // Save Local // ----------------------------------------------------------------------------- async function save_local(jsonArray, filePath) { let fileStr = JSON.stringify(jsonArray); // convert json to string // write string to file fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { if (err) { return err; } else { return true; } }); } // Save to Google Cloud Storage // ----------------------------------------------------------------------------- async function upload_file( bucketName, srcFilePath, trgFilePath ) { // Creates a client const storage = new Storage({projectId, keyFilename}); try { await storage.bucket(bucketName).upload(srcFilePath, { destination: trgFilePath, gzip: true, // serve compressed metadata: { // cache for 8 hours cacheControl: 'public, max-age=60' // production set: 28800 } }); console.log(`${srcFilePath} uploaded to ${bucketName}`); } catch(err) { console.error('ERROR:', err); } } /** #5 * --- * MAIN function (exposed function to run the whole proccess) * ----------------------------------------------------------------------------- *////////////////////////////////////////////////////////////////////////////// /** main() * * nodejs's exported function * used as entry-point function (in case of Google Cloud function) */ // keep only 1 of next 2 lines // exports.main = () => { // entry point functioon when google cloud function var main = () => { // concise function when direct script // PROMISE ... // fetch all jsons from endpints ........................................... Promise.all([ fetch(products_endpoint, request_settings) .then(res => res.json()) .then((json) => { _pr = json; console.log('products data loaded from emarket api/endpoint'); }), fetch(frequency_endpoint, request_settings) .then(res => res.json()) .then((json) => { _fr = json; console.log('frequencies loaded from CDN') }) ]) .then (() => { // then... // proccess data and extract ketwords console.log(`... All data from endpoints: ${FgGreen}LOADED!${Reset}`); // proccess data extract_linked_keywords(_pr.data); console.log('keywords extracted;\ntry to save results localy...'); // PROMISE ... // save local files .................................................... Promise.all([ save_local(kwlinks_, temp_kw_file) .then(() => { console.log('keywords saced localy');}), save_local(prods_, temp_prod_file) .then(() => { console.log('products saved localy');}) ]) .then(() => { // then ... console.log(`... All local saves: ${FgGreen}DONE!${Reset}`); // PROMISE ... // upload keytword and product jsons into Cloud Storage ............ Promise.all([ upload_file('pythia-files', temp_kw_file, 'uploads/json/emarket-keywords.json'), upload_file('pythia-files', temp_prod_file, 'uploads/json/emarket-products.json') ]) .then(() => { // then ... console.log(`... Uploads to cloud storage: ${FgGreen}DONE!${Reset}`); // echo usage stats; then END // check // echo 100 most frequent let i = 0; kwlinks_.forEach(rec => { if (i<100) { console.log(i, JSON.stringify(rec.w)) } i++; }); // echo memory stats const used = process.memoryUsage(); for (let key in used) { console.log(`${key} ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`); } console.log(`${FgGreen}THE END!${Reset}`); }); }); }); } // NOTE: // if running the script directry you need to call the main function // if running through google cloud-functions you do NOT need to call main() // (you need define main() as the entry-point function instead) main();