From 883edd61fe0f375ef3e6d86cbac5ea99cdf7d752 Mon Sep 17 00:00:00 2001 From: Geo Halkiadakis Date: Thu, 13 Apr 2023 17:34:02 +0300 Subject: patches for keygan; patches shall be absorbed into the main proccedure (work in progress) --- javascript/freq.js | 6 +- javascript/keygen.js | 187 ++++++----- javascript/test-linkwords.js | 781 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 884 insertions(+), 90 deletions(-) create mode 100644 javascript/test-linkwords.js diff --git a/javascript/freq.js b/javascript/freq.js index 75c862e..f760926 100644 --- a/javascript/freq.js +++ b/javascript/freq.js @@ -22,7 +22,7 @@ const os = require('os'); const {Storage} = require('@google-cloud/storage'); // import Google Cloud client library - +console.log('dependensies initialized'); // #2 // SETUP PERSONALIZED CONSTANTS AND PARAMETRES @@ -142,7 +142,7 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) { * used as entry-point function (in case of Google Cloud function) */ -exports.main = () => { // google cloud function entry point +var main = () => { // google cloud function entry point console.log('temporary file:', temp_file); @@ -192,3 +192,5 @@ exports.main = () => { // google cloud function entry point }); } + +main(); \ No newline at end of file diff --git a/javascript/keygen.js b/javascript/keygen.js index 2349901..5d1b1cf 100644 --- a/javascript/keygen.js +++ b/javascript/keygen.js @@ -114,7 +114,7 @@ function keyboardize(str) { -/** sanitize_EL +/** sanitize_GR * ----------------------------------------------------------------------------- * replaces greek accended vowels with non accended ones * takes care of sigma on the end of words @@ -154,7 +154,7 @@ function sanitize_GR(str) { str = str.replaceAll('σ-', 'ς-'); str = str.replaceAll('σ ', 'ς '); - return str.trim(); + return str; } @@ -199,30 +199,35 @@ var synonyms_Originals = [ 'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ', 'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ', 'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ', - 'Ντομάτα Ντομάτας', + 'Ντομάτα Ντομάτας Τομάτα', 'Ελαφρύ Ελαφρά Light', 'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά', - 'τριμμένη τριμμένο', + 'τριμμένη τριμμένο τριμμένα', 'Τόνος Τόνου', 'Κριθαρένια κρίθινα', 'Χωρίς-Kαφεϊνη Decaffeine', 'Το-Μάννα Μάννα', 'Κράνμπερι Κράνμπερις', - 'Κρήτης Κρητικό', + 'Κρήτης Κρητικό Κρητική', 'Πέννες Πένες', + 'σοκολατένιος σοκολατένια σοκολατένιο', 'Μακαρόνια Ζυμαρικά Σπαγγέτι Σπαγγετίνι Σπαγγετόνι', 'Καρτέλλα Καρτέλα Καρτέλλες' ] synonyms_Originals.forEach( grp => { // remove accends so any replaces can be done flawlessly - grp = sanitize_GR(grp); + grp = sanitize_GR(grp).trim(); + + // push it to array of synonyms (in lowerCase); + synonyms.push( grp.toLowerCase().split(' ') ); - // create array of synonyms; - // push it to sanitized and keyboardized arrays - synonyms.push( grp.split(' ') ); - synonym_kbs.push( keyboardize(grp).split(' ') ); + // push it the keyboardized array of synonyms + synonym_kbs.push( keyboardize(grp).toLowerCase().split(' ') ); }) +// console.log(synonyms); process.exit(); +// console.log(synonym_kbs); process.exit(); + // significant terms /////////////////////////////////////////////////////////// // ----------------------------------------------------------------------------- @@ -240,69 +245,74 @@ significantExceptios = sanitize_GR('7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' // --- -- -- - - - replaces = []; replaceSource = [ - '3 ΑΛΦΑ ;3-ΑΛΦΑ ', - 'HEAD & SHOULDERS ;HEAD&SHOULDERS ', - 'HEAD N SHOULDERS ;HEAD&SHOULDERS ', - 'W.K Kellogg ; W-K-Kellogg ', - 'W.K Kellogg ; ', - '7 DAYS ;7-DAYS ', - '7 UP ;7-UP ', - 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ ', - 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ ', - 'COCA COLA ;COCA-COLA', - 'COCACOLA ;COCA-COLA', - 'KELLOGS ;KELLOGG\'S', - 'NES CAFE ;NESCAFE', - 'NES-CAFE ;NESCAFE', - 'LE PETIT MARSEILLAIS ;LE-PETIT-MARSEILLAIS', - 'PETIT MARSEILLAIS ;PETIT-MARSEILLAIS', - 'Το Μάννα ;Το-Μάννα', - 'Χωρίς Γλουτένη ;Χωρίς-Γλουτένη', - 'Χωρίς Ζάχαρη ;Χωρίς-Ζάχαρη', - 'Χωρίς Αλάτι ;Χωρίς-Αλάτι', - 'Χωρίς Λακτόζη ;Χωρίς-Λακτόζη', - 'Χωρίς Συντηρητικά ;Χωρίς-Συντηρητικά', - 'Χωρίς Αλκοόλ ;Χωρίς-Αλκοόλ', - 'Χωρίς Kαφεϊνη ;Χωρίς-Kαφεϊνη', - 'Χωρίς Γλυκάνισο ;Χωρίς-Γλυκάνισο', - 'Χωρίς Ανθρακικό ;Χωρίς-Ανθρακικό', - 'Υψηλής Παστερίωσης ;Υψηλής-Παστερίωσης', - 'Ολικής Άλεσης ;Ολικής-Άλεσης', - 'Ολικής Aλέσεως ;Ολικής-Aλέσεως', - 'Χαρτί Υγείας ;Χαρτί-Υγείας', - 'ρολό υγείας ;ρολό-υγείας', - 'χαρτί τουαλέτας ;χαρτί-τουαλέτας', - 'Χαρτί Κουζίνας ;Χαρτί-Κουζίνας', - 'ρολό κουζίνας ;ρολό-κουζίνας', - 'Μπάρες Δημητριακών ;Μπάρες-Δημητριακών', - 'Aς Μαγειρέψουμε ;Aς-Μαγειρέψουμε', - 'ΚΡΙΣ ΚΡΙΣ ;ΚΡΙΣ-ΚΡΙΣ', - 'ΚΡΙΣΚΡΙΣ ;ΚΡΙΣ-ΚΡΙΣ', - 'ΚΡΙ ΚΡΙ ;ΚΡΙ-ΚΡΙ', - 'ΚΡΙΚΡΙ ;ΚΡΙ-ΚΡΙ', - 'ΕΛ ΓΚΡΕΚΟ ;ΕΛ-ΓΚΡΕΚΟ', - 'ΕΛΓΚΡΕΚΟ ;ΕΛ-ΓΚΡΕΚΟ', - 'FREE STEP ;FREE-STEP', - 'EL SABOR ;EL-SABOR', - 'ELSABOR ;EL-SABOR', - 'DOUWE EGBERTS ;DOUWE-EGBERTS', - 'ΕΝ ΕΛΛΑΔΙ ;ΕΝ-ΕΛΛΑΔΙ', - 'ΕΝΕΛΛΑΔΙ ;ΕΝ-ΕΛΛΑΔΙ', - 'SPIN SPAN ;SPIN-SPAN', - 'SPINSPAN ;SPIN-SPAN', - 'CRETA-FARMS ;CRETA-FARM', - 'CRETA-FARM ;CRETA-FARM', - 'CRETAFARM ;CRETA-FARM', - 'Ολες-τις-Χρήσεις ;Ολες-τις-Χρήσεις', - 'Χωρίς προσθήκη ζάχαρης ;Χωρίς-ζάχαρη' + '3 ΑΛΦΑ;3-ΑΛΦΑ', + 'HEAD & SHOULDERS;HEAD&SHOULDERS', + 'HEAD N SHOULDERS;HEAD&SHOULDERS', + 'W.K Kellogg; W-K-Kellogg', + 'W.K Kellogg;', + '7 DAYS;7-DAYS', + '7 UP;7-UP', + 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', + 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', + 'COCA COLA;COCA-COLA', + 'COCACOLA;COCA-COLA', + // 'KELLOGS;KELLOGG\'S', + 'NES CAFE;NESCAFE', + 'NES-CAFE;NESCAFE', + 'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS', + 'PETIT MARSEILLAIS;PETIT-MARSEILLAIS', + 'Το Μάννα;Το-Μάννα', + 'Χωρίς Γλουτένη;Χωρίς-Γλουτένη', + 'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη', + 'Χωρίς Αλάτι;Χωρίς-Αλάτι', + 'Χωρίς Λακτόζη;Χωρίς-Λακτόζη', + 'Χωρίς Συντηρητικά;Χωρίς-Συντηρητικά', + 'Χωρίς Αλκοόλ;Χωρίς-Αλκοόλ', + 'Χωρίς Kαφεϊνη;Χωρίς-Kαφεϊνη', + 'Χωρίς Γλυκάνισο;Χωρίς-Γλυκάνισο', + 'Χωρίς Ανθρακικό;Χωρίς-Ανθρακικό', + 'Υψηλής Παστερίωσης;Υψηλής-Παστερίωσης', + 'Ολικής Άλεσης;Ολικής-Άλεσης', + 'Ολικής Aλέσεως;Ολικής-Aλέσεως', + 'Χαρτί Υγείας;Χαρτί-Υγείας', + 'ρολό υγείας;ρολό-υγείας', + 'χαρτί τουαλέτας;χαρτί-τουαλέτας', + 'Χαρτί Κουζίνας;Χαρτί-Κουζίνας', + 'ρολό κουζίνας;ρολό-κουζίνας', + 'Μπάρες Δημητριακών;Μπάρες-Δημητριακών', + 'Ας Μαγειρέψουμε;Ας-Μαγειρέψουμε', + 'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', + 'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', + 'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ', + 'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ', + 'ΕΛ ΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', + 'ΕΛΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', + 'FREE STEP;FREE-STEP', + 'EL SABOR;EL-SABOR', + 'ELSABOR;EL-SABOR', + 'DOUWE EGBERTS;DOUWE-EGBERTS', + 'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', + 'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', + 'SPIN SPAN;SPIN-SPAN', + 'SPINSPAN;SPIN-SPAN', + 'CRETA-FARMS;CRETA-FARM', + 'CRETA-FARM;CRETA-FARM', + 'CRETAFARM;CRETA-FARM', + 'Ολες-τις-Χρήσεις;Ολες-τις-Χρήσεις', + 'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη' ]; replaceSource.forEach( it => { st = sanitize_GR( - it + it.toLowerCase() ).split(';'); - replaces.push({ src: st[0], trg: st[1] }); + replaces.push({ + src: ' '+ st[0] +' ', // encolse between spaces + trg: ' '+ st[1] +' ' // to separate from before/after words + }); }); +// console.log(replaces); process.exit(); + // words that shall not be searched first ////////////////////////////////////// // ----------------------------------------------------------------------------- @@ -474,17 +484,20 @@ function extract_linked_keywords(obj) { if (rec.img != 0) { - var description = preproccess_text(rec.txt); + //var description = preproccess_text(rec.txt); + + var description = clean_text(rec.txt); + var pid = rec.eys; var fq = freq_of(pid)+1; prods_.push({ id: pid, - w: description + w: description.trim() }) var keys = []; - var words = description.split(' '); + var words = preproccess_text(description).split(' '); // filter words; keep only significant words.forEach( w => { @@ -607,7 +620,7 @@ function connect_keys( a, b, id, f ) { w : b, kb : kbB, f : f, - p : [ i ] + p : [ id ] }); } return true; @@ -624,13 +637,13 @@ function connect_keys( a, b, id, f ) { /** clean text * --- -- -- - - - - * removes non keyword characters [\+, \., \'] and internal multiple-spaces + * removes non keyword characters [+ . , !] and internal multiple-spaces * @param txt (string): product description */ function clean_text(txt) { - return sanitize_GR(txt).replace('+',' ').replace('.',' ') - .replace(' ',' ') - .replace(' ',' '); + return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space + .replace('!','').replace('\"', '') // remove character + .replace(' ',' ').replace(' ',' '); // remove multiple spaces } @@ -662,7 +675,7 @@ function is_significant(str) { function synonym_keys(w) { w_kb = keyboardize(w); for (i=0 ; i < synonym_kbs.length ; i++) { - if (synonym_kbs.indexOf(w_kb) !== -1) + if (synonym_kbs[i].indexOf(w_kb) !== -1) return synonyms[i]; } return [ w ]; @@ -682,24 +695,24 @@ function synonym_keys(w) { // edit common mistakes // with suggested replaces function mark_explicit_links(str) { - - replaces.forEach( it => { str = str.replace(it.src, it.trg); }); - return str; + str = ' '+ str +' '; + replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); }); + return str.replaceAll(' ', ' ').trim(); } // preproccess description // --- -function preproccess_text(str) { +function preproccess_text(description) { - str = clean_text(str); // remove non-keyword characters + str = clean_text(description); // remove non-keyword characters - str = mark_explicit_links(sanitize_GR(str)); // mark explicit links + str = mark_explicit_links(sanitize_GR( str.toLowerCase() )); // mark explicit links - replaces.forEach( lw => { - reg = new RegExp( lw.src, "gi"); // prepare regex for case insensitive (i) replace all (g) - str = str.toLowerCase().replace(reg, lw.trg); - }) + // replaces.forEach( lw => { + // reg = new RegExp( lw.src, "gi"); // prepare regex for case insensitive (i) replace all (g) + // str = str.toLowerCase().replace(reg, lw.trg); + // }) // depricated: str = mark_linked_words(str); return str; } @@ -792,8 +805,6 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) { - - /** #5 * --- * MAIN function (exposed function to run the whole proccess) diff --git a/javascript/test-linkwords.js b/javascript/test-linkwords.js new file mode 100644 index 0000000..680be70 --- /dev/null +++ b/javascript/test-linkwords.js @@ -0,0 +1,781 @@ +/** keygen + * + * this script constucts a linked-wordkeys structure + * ----------------------------------------------------------------------------- + * + * Contents: + * #1 Requirements + * #2.1 Personalized constants and parametres + * #2.2 Purify string functions + * #2.3 Preloaded Data + * #3 Supporting functions + * #4 Output functions + * #5 Entry-point function main() + */ + + +/** #1 + * --- + * REQUIREMENTS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + const fs = require('fs'); + const os = require('os'); + const fetch = require('node-fetch'); + const {Storage} = require('@google-cloud/storage'); // Google Cloud Storage + + + + + /** #2.1 + * --- + * SETUP PERSONALIZED CONSTANTS AND PARAMETRES + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + + // get products FROM api/endpoint parametres + // --- -- -- - - - + const frequency_endpoint = 'https://storage.googleapis.com/pythia-files/uploads/json/freq.json'; + const products_endpoint = "https://emarket-laravel-dlqjpfxz5q-oa.a.run.app/api/v1/productsSearch"; + const request_settings = { method: "Get" }; + + + // google-storage parametres + // --- -- -- - - - + const projectId = 'pythia-251711'; + const keyFilename = '../auth/pythia-251711-047e3d5e6608.json'; + + + // temp files + // need to be created prior to save-to-google-storage proccess + // because upload_file() uploats an existing local file to G-Storage + // --- -- -- - - - + const temp_kw_file = '../results/keywords.json'; + const temp_prod_file = '../results/products.json'; + + + // Global Variables + // ----------------------------------------------------------------------------- + + // Global arrays + // keyword links (word-links dictionary; array of objects) + // ----------------------------------------------------------------------------- + var kwlinks_ = []; // keyword-links structure (structure to construct) + var prods_ = []; // products (array to construct) + var _fr =[]; // product frequencies array (as read from endpiont) + var _pr =[ + "ΚΡΙ ΚΡΙ Γιαούρτι Στραγγιστό & Αγαπημένο 2% 1kg", + "COCA COLA Zero Αναψυκτικό Χωρίς ζάχαρη 6x330ml", +// "COCA COLA Αναψυκτικό 2x1lt", +// "ΚΡΙΣ ΚΡΙΣ Τόστιμο Ψωμί Τοστ Σταρένιο 400gr", +// "ΚΡΙ ΚΡΙ Heartmade Παγωτό Βανίλια Κακάο & Φράουλα 1kg (2lt) 50% Δώρο", +// "ΚΡΙ ΚΡΙ Kids Peppa Επιδό… Παιδικό 2x140gr 1 Δώρο", +// "ΒΙΚΟΣ Αναψυκτικό Cola 330ml" + ]; // product array (as read from endpiont) + + + // Console Colors + // ----------------------------------------------------------------------------- + const Reset = "\x1b[0m"; + const FgRed = "\x1b[31m"; + const FgGreen = "\x1b[32m"; + + + + + + /** #2.2 + * --- + * PURIFY STRING FUNCTIONS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + + + /** keyboardize + * ----------------------------------------------------------------------------- + * translates string to keyboard-latin keys + * (the ones that used whan typing each letter of the word) + * + * @param str (string): original string (utf8 of latin or greek subgroups) + * @return (string): latin/ascii equivalent string + */ + + // cache (keeo in global) any-character to keyboard-latin mapping + // --- -- -- - - - + var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + const map = new Map(); + for (var i=0; i { + ap = pair.split(' '); + accented_vowels.push({ + a: ap[0], // accented + p: ap[1] // pure = non accended + }); + }) + + // the sanitize_GR function + // --- -- -- - - - + function sanitize_GR(str) { + + str = str.toLowerCase(); + + // replace accended vowels with pure ones + accented_vowels.forEach( v => { + str = str.replaceAll(v.a, v.p); + }); + + // replace sigma on the end of words + str = str + ' '; + str = str.replaceAll('σ-', 'ς-'); + str = str.replaceAll('σ ', 'ς '); + + // return str.trim(); + return str; + } + + + + /** #2.3 + * --- + * PRELOADED DATA + * in the future these data may be loaded from a database / api-endpoint + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + + // synonyms //////////////////////////////////////////////////////////////////// + // ----------------------------------------------------------------------------- + + var synonyms = []; // groups of synonyms + var synonym_kbs = []; // cache kb-formats for performance + var synonyms_Originals = [ + 'μπίρα μπύρα μπίρες μπύρες', + 'coca-cola cocacola coke', + 'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ', + 'Ντομάτα Ντομάτας', + 'Ελαφρύ Ελαφρά Light', + 'Εγχώρια Εγχώριες Ελληνικό Ελληνική Ελληνικά', + 'τριμμένη τριμμένο', + 'Τόνος Τόνου', + 'Κριθαρένια κρίθινα', + 'Χωρίς-Kαφεϊνη Decaffeine', + 'Το-Μάννα Μάννα', + 'Κράνμπερι Κράνμπερις', + 'Κρήτης Κρητικό', + 'Πέννες Πένες', + 'Μακαρόνια Ζυμαρικά Σπαγγέτι Σπαγγετίνι Σπαγγετόνι', + 'Καρτέλλα Καρτέλα Καρτέλλες' + ] + synonyms_Originals.forEach( grp => { + // remove accends so any replaces can be done flawlessly + grp = sanitize_GR(grp); + + // create array of synonyms; + // push it to sanitized and keyboardized arrays + synonyms.push( grp.split(' ') ); + synonym_kbs.push( keyboardize(grp).split(' ') ); + }) + + + // significant terms /////////////////////////////////////////////////////////// + // ----------------------------------------------------------------------------- + significantExceptios = sanitize_GR('7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ') + + + // replaces (correcting descriptions) + // = construct unequivocally liked words /////////////////////////////////////// + // ----------------------------------------------------------------------------- + + // NOTE: TODO: + // in future implementations multi-word keywords + // may use the non-breaking space as conecting character (\u00A0) instead of dush (-) + // (or maybe both of them) + // --- -- -- - - - + replaces = []; + replaceSource = [ + 'COCA COLA;COCA-COLA', + 'COCACOLA;COCA-COLA', + 'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', + 'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', + 'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη', + 'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ', + 'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ', + 'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη' + ]; + replaceSource.forEach( it => { + st = sanitize_GR( + it.toLowerCase() + ).split(';'); + replaces.push({ + src: ' '+ st[0] +' ', // encolse between spaces + trg: ' '+ st[1] +' ' // to separate from before/after words + }); + }); + + // console.log(replaces); process.exit(); + + + // words that shall not be searched first ////////////////////////////////////// + // ----------------------------------------------------------------------------- + var noRootKeywords = []; + noRoot = [ + 'χωρίς', + 'εισαγωγής', + 'δώρο', + 'γεύση', + 'γεύσεις', + 'φέτες', + 'Χωρίς-Γλουτένη', + 'Γλουτένη', + 'Λακτόζη', + 'Παστερίωσης', + 'Ανθρακικό', + 'Συντηρητικά', + 'Χωρίς-Ζάχαρη', + 'Άλεσης', + 'Χωρίς-Αλάτι', + 'Χωρίς-Λακτόζη', + 'Χωρίς-Συντηρητικά', + 'Χωρίς-Αλκοόλ', + 'Χωρίς-Kαφεϊνη', + 'Χωρίς-Γλυκάνισο', + 'Χωρίς-Ανθρακικό', + 'Υψηλής-Παστερίωσης', + 'Ολες-τις-Χρήσεις', + 'Ολικής-Άλεσης', + 'Ολικής-Aλέσεως', + 'Ολικής', + 'Σοκολάτας', + 'Υγείας', + 'Γαϊδούρας', + 'Δημητριακών', + 'Γαϊδάρου', + 'Ρούχων', + 'Πιάτων', + 'πλύσεις', + 'Πλυντηρίου', + 'Φύλλων', + 'Γάλακτος', + 'Χρήσης', + 'Τύπου', + 'Ολλανδίας', + 'Απορριμμάτων', + 'Medium', + 'Μαλλιά', + 'Μαλλιών', + 'Γενικής', + 'Plus', + 'Classic', + 'Έκπληξη', + 'Μάνης', + 'Ελάτου', + 'Άγριων', + 'Βοτάνων', + 'Κατσαρίδες', + 'Λακωνίας', + 'ΠΑΡΑΓΓΕΛΙΩΝ', + 'Λήμνου', + 'Αργινίνης', + 'Καλαθάκι', + 'Σκύλου', + 'Γάτας', + 'ΠΓΕ', + 'ΤΟ', + 'Μελί', + 'Σώματος', + 'Αδυνατίσματος', + 'Προστασίας', + 'Ακράτειας', + 'Προσώπου', + 'προσθήκη', + 'Βρεφικής', + 'Καθαρισμού', + 'Ρούχων' + ]; + noRoot.forEach( w => { noRootKeywords.push(keyboardize(w)); }); + + + // list of linked-words //////////////////////////////////////////////////////// + // ----------------------------------------------------------------------------- + linkedWords = [ + sanitize_GR('Χωρίς-Γλουτένη'), + sanitize_GR('Χωρίς-Ζάχαρη'), + sanitize_GR('Χωρίς-Αλάτι'), + sanitize_GR('Χωρίς-Λακτόζη'), + sanitize_GR('Χωρίς-Συντηρητικά'), + sanitize_GR('Χωρίς-Αλκοόλ'), + sanitize_GR('Χωρίς-Kαφεϊνη'), + sanitize_GR('Χωρίς-Γλυκάνισο'), + sanitize_GR('Χωρίς-Ανθρακικό'), + sanitize_GR('Υψηλής-Παστερίωσης'), + sanitize_GR('Ολικής-Άλεσης'), + sanitize_GR('Ολικής-Aλέσεως'), + sanitize_GR('Χαρτί-Υγείας'), + sanitize_GR('ρολό-υγείας'), + sanitize_GR('χαρτί-τουαλέτας'), + sanitize_GR('Χαρτί-Κουζίνας'), + sanitize_GR('Μπάρες-Δημητριακών'), + sanitize_GR('Μπαρμπα-Στάθης'), + sanitize_GR('COCA-COLA'), + sanitize_GR('Aς-Μαγειρέψουμε'), + sanitize_GR('ΚΡΙΣ-ΚΡΙΣ'), + sanitize_GR('ΚΡΙ-ΚΡΙ'), + sanitize_GR('ΕΛ-ΓΚΡΕΚΟ'), + sanitize_GR('FREE-STEP'), + sanitize_GR('EL-SABOR'), + sanitize_GR('LE-PETIT-MARSEILLAIS'), + sanitize_GR('DOUWE-EGBERTS'), + sanitize_GR('ΕΝ-ΕΛΛΑΔΙ'), + sanitize_GR('SPIN-SPAN'), + sanitize_GR('CRETA-FARMS'), + sanitize_GR('CRETA-FARM'), + sanitize_GR('NES-CAFE'), + sanitize_GR('Ολες-τις-Χρήσεις'), + sanitize_GR('Το-Μάννα'), + sanitize_GR('Χωρίς-προσθήκη-ζάχαρης') + ]; + + + // list of words to exclude from keywords ////////////////////////////////////// + // ----------------------------------------------------------------------------- + // NOTE: APPLIED in PER-WORD base -> after spliting description to words + removeList = [] + removeOriginals = sanitize_GR( + 'κατά παρά υπό από μετά προς μας με σε για του της των από στο το στον & r s ft l τ e g h k m n o p s x' + ) + .split(' ') + removeOriginals.forEach( w => { removeList.push( keyboardize(w)); }); + + + + + + + /** #3 + * --- + * SUPPORTING FUNCTIONS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + + /** freq + * + * @param id (int): product's eys_code + * @return frequency of product + */ + function freq_of(id) { + _fr.forEach( rec => { + if (rec.id == id) return rec.fq + }); + + return 0; + } + + + /** proccess data + * --- -- -- - - - + * + * central proccessing procedure + * ----------------------------------------------------------------------------- + */ + function extract_linked_keywords(obj) { + // console.log(JSON.stringify(obj, null, 2)); + + var i = 0; + + obj.forEach( rec => { + + i++; + //var description = preproccess_text(rec.txt); + + var description = clean_text(rec).trim(); + + var pid = i; + var fq = 1; + + prods_.push({ + id: pid, + w: description + }) + + var keys = []; + var words = preproccess_text(description).split(' '); + + // filter words; keep only significant + words.forEach( w => { + if (removeList.indexOf(keyboardize(w)) == -1) // if not excluded + if (is_significant(w)) // and significant + keys.push(w); // add it to keys + }); + // console.log(pid, description, keys); + + keys.forEach( w => { + var wl = synonym_keys(w); + + // if key CAN be a root word (not a no-Root-keyword) update root-node + if (noRootKeywords.indexOf(keyboardize(w)) == -1) { + + root_key(wl, fq); // update root keyword stats + + // if key is the only in the list of product's keywords + // connect it with a dummy key (to preserve the reference to the product) + if (keys.length == 1) connect_keys(wl, ['*'], pid, fq); + + // connect w with all the other product's keywords + keys.forEach( w2 => { + if (w2 != w) { + var w2syns = synonym_keys(w2); + connect_keys( wl, w2syns, pid, fq); + } + }); + } + + }); + + }); // main proccessing finished; + + // post proccess + // ------------------------------------------------------------------------- + + // remove cached keys from final array + kwlinks_.forEach( ro => { + delete ro.kb; + ro.c.forEach( ch => { delete ch.kb; }); + }); + + // sort root and child nodes by frequency descanding + kwlinks_.forEach( it => { + it.c = it.c.sort((a, b) => b.f - a.f ); + }); + kwlinks_ = kwlinks_.sort((a, b) => b.f - a.f ); + + } + + + // functions for linking words in keywords dictionary + //////////////////////////////////////////////////////////////////////////////// + + + /** root_key( wl, f ) + * --- -- -- - - - + * set root keyword: wl (if not exist) + * update frequency: f + * + * @param wl (list) :a list of synonym-words + * @param f (int): frequency of product + * + * ** comparison is based on the *keyboard* format + */ + function root_key ( wl, f ) { + var wkb = keyboardize(wl[0]) // cache kb format + + // check if exists in root keys already + // NOTE: you only need to check the 1st word of synonyms-list + for (i=0; i< kwlinks_.length ; i++) { + if (kwlinks_[i].kb == wkb) { + kwlinks_[i].f += f; + return true; + } + } + // if not exists, append keyword + kwlinks_.push({ + w : wl, + kb : wkb, + f : f, + c : [] + }); + return true; + } + + /** connect_keys + * --- -- -- - - - + * @param (list) : connect keyword a (synonyms list) + * @param (list) : with keyword b (synonyms list) + * @param (int) : id (eys code) of product where keywords a and b are linked + * @param (int) : frequency of product (now many times has been purchased) + * @returns true + */ + function connect_keys( a, b, id, f ) { + var kbA = keyboardize(a[0]); + var kbB = keyboardize(b[0]); + var bExists = false; + + if (kbA == kbB) return false; // exclude just-in-case + + for (i=0; i< kwlinks_.length ; i++) { + if (kwlinks_[i].kb == kbA) { // found: a; + // update connection to: b + bExists = false; + for (j=0 ; j < kwlinks_[i].c.length ; j++) { + if (kwlinks_[i].c[j].kb == kbB) { + bExists = true; + // update the connection's data + kwlinks_[i].c[j].f += f; + kwlinks_[i].c[j].p.push(id) + break; + } + } + // if connection not exist, init a new one + if (bExists == false) { + // create connection with: b + kwlinks_[i].c.push({ + w : b, + kb : kbB, + f : f, + p : [ i ] + }); + } + return true; + } + } + } + + + // other supplementary functions + //////////////////////////////////////////////////////////////////////////////// + + + + + /** clean text + * --- -- -- - - - + * removes non keyword characters [\+, \., \'] and internal multiple-spaces + * @param txt (string): product description + */ + function clean_text(txt) { + return txt.replace('+',' ').replace('.',' ') + .replace(' ',' ').replace(' ',' '); + } + + + /** is significant + * --- -- -- - - - + * check if term is significant (and needs to be indexed) + * (if not, the term will be excluded from keywords dicionary) + * + * @param str + * @return (boolean) + */ + function is_significant(str) { + if (str == '') return false; + if (significantExceptios.indexOf(str) !== -1) return true; + return !(/\d/.test(str)); + } + + + /** synonym_keys + * --- -- -- - - - + * check if word: w + * has synonyms; return list of synonyms + * + * @param w (string) + * @return list of synonyms + * + * NOTE: in no synonyms exist, retutn the word as a list + */ + function synonym_keys(w) { + w_kb = keyboardize(w); + for (i=0 ; i < synonym_kbs.length ; i++) { + if (synonym_kbs.indexOf(w_kb) !== -1) + return synonyms[i]; + } + return [ w ]; + } + + + + /** mark_explicit_links + * + * mark linked words + * shall be handled as one-(key)word + * + * @param str + * @return + */ + + // edit common mistakes + // with suggested replaces + function mark_explicit_links(str) { + str = ' '+ str +' '; + replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); }); + return str.replaceAll(' ', ' ').trim(); + } + + + // preproccess description + // --- + function preproccess_text(description) { + + str = clean_text(description); // remove non-keyword characters + + str = mark_explicit_links(sanitize_GR( str.toLowerCase() )); // mark explicit links + + console.log('>', str); + + // replaces.forEach( lw => { + // reg = new RegExp( lw.src, "gi"); // prepare regex for case insensitive (i) replace all (g) + // str = str.toLowerCase().replace(reg, lw.trg); + // }) + // depricated: str = mark_linked_words(str); + return str; + } + + + + /* --- + DEPRICATED: + // mark linked words (connect them with a dash) + // return new text after "all-links" are marked + // --- + function mark_linked_words(txt) { + linkedWords.forEach( lw => { + + txt = mark_link( lw, txt ); + }); + return txt + } + */ + + + + /* --- + DEPRICATED: + // mark a link (lws) to a text (source) + // conecting them with a dash/minus character + // --- + function mark_link(lws, source) { + var src_kb = keyboardize(source.replace(' ', '-')); + var lws_kb = keyboardize(lws.replace(' ', '-')); + var _left = src_kb.toLowerCase().indexOf(lws_kb.toLowerCase()) + if (_left !== -1 ) { + return source.slice(0, _left) + lws + source.slice(_left + lws.length); + } + else return source; + } + */ + + + + + /** #4 + * --- + * OUTPUT FUNCTIONS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + + + // Save Local + // ----------------------------------------------------------------------------- + async function save_local(jsonArray, filePath) { + let fileStr = JSON.stringify(jsonArray); // convert json to string + + // write string to file + fs.writeFileSync(filePath, fileStr, 'utf8', (err) => { + if (err) { + return err; + } else { + return true; + } + }); + } + + + + // Save to Google Cloud Storage + // ----------------------------------------------------------------------------- + async function upload_file( bucketName, srcFilePath, trgFilePath ) { + // Creates a client + const storage = new Storage({projectId, keyFilename}); + + try { + await storage.bucket(bucketName).upload(srcFilePath, { + destination: trgFilePath, + gzip: true, // serve compressed + metadata: { // cache for 8 hours + cacheControl: 'public, max-age=60' // production set: 28800 + } + }); + console.log(`${srcFilePath} uploaded to ${bucketName}`); + } + + catch(err) { + console.error('ERROR:', err); + } + + } + + + + + + + /** #5 + * --- + * MAIN function (exposed function to run the whole proccess) + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + + + /** main() + * + * nodejs's exported function + * used as entry-point function (in case of Google Cloud function) + */ + + // keep only 1 of next 2 lines + // exports.main = () => { // entry point functioon when google cloud function + var main = () => { // concise function when direct script + + // proccess data and extract ketwords + + // proccess data + extract_linked_keywords(_pr); + + + // console.log(kwlinks_) + + // console.log(prods_) + + } + + + // NOTE: + // if running the script directry you need to call the main function + // if running through google cloud-functions you do NOT need to call main() + // (you need define main() as the entry-point function instead) + main(); \ No newline at end of file -- cgit v1.2.3