diff options
| -rw-r--r-- | javascript/keygen.js | 134 |
1 files changed, 97 insertions, 37 deletions
diff --git a/javascript/keygen.js b/javascript/keygen.js index 5d1b1cf..af69a52 100644 --- a/javascript/keygen.js +++ b/javascript/keygen.js @@ -4,13 +4,45 @@ * ----------------------------------------------------------------------------- * * Contents: + * --- * #1 Requirements + * -* fs + * -* os + * -* node-fetch + * -* @google-cloud/storage + * * #2.1 Personalized constants and parametres + * -* frequency endpoin + * -* products endpoint + * * #2.2 Purify string functions - * #2.3 Preloaded Data + * -* keyboardize() + * -* sanitize() + * -* clean() + * + * #2.3 Preloaded Data (TODO: load via endpoint) + * -* synonyms + * -* significant exceptions + * -* replaces (linked words) + * -* noRootKeywords + * -* removeList + * * #3 Supporting functions + * -* freq_of( product's eys-code) + * -* extract_linked_keywords -> NOTE: MAIN PROCEDURE to do the job + * -* root_key( word_list, frequency ) + * -* connect_keys( wordlist_#1, wordlist_#2 , product_id, frequency ) + * -* is_significant + * -* synonym_keys( word ) + * -* mark_explicit_links() + * -* preproccess_text() + * * #4 Output functions + * -* save_local() + * -* upload_file + * * #5 Entry-point function main() + * -* main() NOTE: entry-point function */ @@ -107,6 +139,7 @@ for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]); function keyboardize(str) { str = str.replace('\'',''); var out = ''; + // [map]'s implementation is 40x faster than [for]'s for (var i=0 ; i< str.length; i++) out += map.get(str[i]); return out; } @@ -128,7 +161,7 @@ function keyboardize(str) { // --- -- -- - - - accented_vowels = []; [ - 'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω', + 'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ΐ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω', 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω' ].forEach( pair => { ap = pair.split(' '); @@ -158,6 +191,18 @@ function sanitize_GR(str) { } +/** clean text + * --- -- -- - - - + * removes non keyword characters [+ . , !] and internal multiple-spaces + * @param txt (string): product description + */ +function clean_text(txt) { + return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space + .replace('!','').replace('\"', '') // remove character + .replace(' ',' ').replace(' ',' '); // remove multiple spaces +} + + /** #2.3 * --- @@ -167,8 +212,9 @@ function sanitize_GR(str) { *////////////////////////////////////////////////////////////////////////////// -// synonyms //////////////////////////////////////////////////////////////////// -// ----------------------------------------------------------------------------- +/** synonyms /////////////////////////////////////////////////////////////////// + * ----------------------------------------------------------------------------- + */ var synonyms = []; // groups of synonyms var synonym_kbs = []; // cache kb-formats for performance @@ -212,6 +258,7 @@ var synonyms_Originals = [ 'Πέννες Πένες', 'σοκολατένιος σοκολατένια σοκολατένιο', 'Μακαρόνια Ζυμαρικά Σπαγγέτι Σπαγγετίνι Σπαγγετόνι', + 'Λαζάνια Λιγκουίνι Ματσάτα Linguine', 'Καρτέλλα Καρτέλα Καρτέλλες' ] synonyms_Originals.forEach( grp => { @@ -229,34 +276,38 @@ synonyms_Originals.forEach( grp => { // console.log(synonym_kbs); process.exit(); -// significant terms /////////////////////////////////////////////////////////// -// ----------------------------------------------------------------------------- -significantExceptios = sanitize_GR('7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ') +/** significant terms ////////////////////////////////////////////////////////// + * ----------------------------------------------------------------------------- + */ +significantExceptios = sanitize_GR('7-UP 7UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ') -// replaces (correcting descriptions) -// = construct unequivocally liked words /////////////////////////////////////// -// ----------------------------------------------------------------------------- -// NOTE: TODO: -// in future implementations multi-word keywords -// may use the non-breaking space as conecting character (\u00A0) instead of dush (-) -// (or maybe both of them) -// --- -- -- - - - +/** replaces (correcting descriptions) + * = construct unequivocally liked words /////////////////////////////////////// + * ----------------------------------------------------------------------------- + * + * NOTE: TODO: + * in future implementations multi-word keywords + * may use the non-breaking space as conecting character (\u00A0) instead of dush (-) + * (or maybe both of them) + * --- -- -- - - - + */ replaces = []; replaceSource = [ + '3Α;3-ΑΛΦΑ', '3 ΑΛΦΑ;3-ΑΛΦΑ', 'HEAD & SHOULDERS;HEAD&SHOULDERS', 'HEAD N SHOULDERS;HEAD&SHOULDERS', 'W.K Kellogg; W-K-Kellogg', 'W.K Kellogg;', '7 DAYS;7-DAYS', - '7 UP;7-UP', + '7 UP;7UP', + '7-UP;7UP', 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', 'COCA COLA;COCA-COLA', 'COCACOLA;COCA-COLA', - // 'KELLOGS;KELLOGG\'S', 'NES CAFE;NESCAFE', 'NES-CAFE;NESCAFE', 'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS', @@ -291,6 +342,7 @@ replaceSource = [ 'EL SABOR;EL-SABOR', 'ELSABOR;EL-SABOR', 'DOUWE EGBERTS;DOUWE-EGBERTS', + 'DOUWEEGBERTS;DOUWE-EGBERTS', 'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', 'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', 'SPIN SPAN;SPIN-SPAN', @@ -471,11 +523,25 @@ function freq_of(id) { } -/** proccess data +/** proccess data ///////////////////////////////////////////////////////////// * --- -- -- - - - * * central proccessing procedure * ----------------------------------------------------------------------------- + * + * Operation: + * --- + * for each title { + * + clean text, get frequency + * + preproccess and split into individual words + * + for each significant word { + * + update root keywords + * + update linked words + * } + * } + * post-process results + * + * ///////////////////////////////////////////////////////////////////////////// */ function extract_linked_keywords(obj) { // console.log(JSON.stringify(obj, null, 2)); @@ -635,18 +701,6 @@ function connect_keys( a, b, id, f ) { -/** clean text - * --- -- -- - - - - * removes non keyword characters [+ . , !] and internal multiple-spaces - * @param txt (string): product description - */ -function clean_text(txt) { - return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space - .replace('!','').replace('\"', '') // remove character - .replace(' ',' ').replace(' ',' '); // remove multiple spaces -} - - /** is significant * --- -- -- - - - * check if term is significant (and needs to be indexed) @@ -685,15 +739,12 @@ function synonym_keys(w) { /** mark_explicit_links * - * mark linked words - * shall be handled as one-(key)word + * mark linked words shall be handled as one-(key)word + * (also edit common mistakes with suggested replaces) * * @param str * @return */ - -// edit common mistakes -// with suggested replaces function mark_explicit_links(str) { str = ' '+ str +' '; replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); }); @@ -701,8 +752,12 @@ function mark_explicit_links(str) { } -// preproccess description -// --- +/** preproccess description + * == do... + * + clean + * + sanitize + * + mark lined words + */ function preproccess_text(description) { str = clean_text(description); // remove non-keyword characters @@ -817,6 +872,11 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) { * * nodejs's exported function * used as entry-point function (in case of Google Cloud function) + * + * Operation + * + 1: load products and frequencies + * + 2: forward to extract_linked_keywords() + * + 3: Save results (localy, then to cloud) */ // keep only 1 of next 2 lines |