new keygen impelementation completed for local machine

1 files changed, 97 insertions, 37 deletions

diff --git a/javascript/keygen.js b/javascript/keygen.js
index 5d1b1cf..af69a52 100644
--- a/javascript/keygen.js
+++ b/javascript/keygen.js
@@ -4,13 +4,45 @@
  * -----------------------------------------------------------------------------
  * 
  * Contents:
+ * ---
  * #1 Requirements
+ *  -* fs
+ *  -* os
+ *  -* node-fetch
+ *  -* @google-cloud/storage
+ * 
  * #2.1 Personalized constants and parametres
+ *  -* frequency endpoin
+ *  -* products endpoint
+ * 
  * #2.2 Purify string functions
- * #2.3 Preloaded Data
+ *  -* keyboardize()
+ *  -* sanitize()
+ *  -* clean()
+ * 
+ * #2.3 Preloaded Data  (TODO: load via endpoint)
+ *  -* synonyms
+ *  -* significant exceptions
+ *  -* replaces (linked words)
+ *  -* noRootKeywords
+ *  -* removeList
+ * 
  * #3 Supporting functions
+ *  -* freq_of( product's eys-code)
+ *  -* extract_linked_keywords -> NOTE: MAIN PROCEDURE to do the job
+ *  -* root_key( word_list, frequency )
+ *  -* connect_keys( wordlist_#1, wordlist_#2 , product_id, frequency )
+ *  -* is_significant
+ *  -* synonym_keys( word )
+ *  -* mark_explicit_links()
+ *  -* preproccess_text()
+ * 
  * #4 Output functions
+ *  -* save_local()
+ *  -* upload_file
+ * 
  * #5 Entry-point function main()
+ * -* main()    NOTE: entry-point function
  */
 
 
@@ -107,6 +139,7 @@ for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
 function keyboardize(str) {
     str = str.replace('\'','');
     var out = '';
+    // [map]'s implementation is 40x faster than [for]'s
     for (var i=0 ; i< str.length; i++)  out += map.get(str[i]);
     return out;
 }
@@ -128,7 +161,7 @@ function keyboardize(str) {
 // --- -- -- - - -
 accented_vowels = [];   
 [
-    'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω',
+    'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ΐ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω',
     'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω'
 ].forEach( pair => {
     ap = pair.split(' ');
@@ -158,6 +191,18 @@ function sanitize_GR(str) {
 }
 
 
+/** clean text
+ * --- -- -- - - -
+ * removes non keyword characters [+ . , !] and internal multiple-spaces
+ * @param txt (string): product description
+ */
+function clean_text(txt) {
+    return txt.replace('+',' ').replace('.',' ').replace(',',' ')   // change to space
+        .replace('!','').replace('\"', '')                  // remove character
+        .replace('   ',' ').replace('  ',' ');          // remove multiple spaces
+}
+
+
 
 /** #2.3
  * ---
@@ -167,8 +212,9 @@ function sanitize_GR(str) {
  *//////////////////////////////////////////////////////////////////////////////
 
 
-// synonyms ////////////////////////////////////////////////////////////////////
-// -----------------------------------------------------------------------------
+/** synonyms ///////////////////////////////////////////////////////////////////
+ * -----------------------------------------------------------------------------
+ */
 
 var synonyms = [];      // groups of synonyms
 var synonym_kbs = [];   // cache kb-formats for performance
@@ -212,6 +258,7 @@ var synonyms_Originals = [
     'Πέννες Πένες',
     'σοκολατένιος σοκολατένια σοκολατένιο',
     'Μακαρόνια Ζυμαρικά Σπαγγέτι Σπαγγετίνι Σπαγγετόνι',
+    'Λαζάνια Λιγκουίνι Ματσάτα Linguine',
     'Καρτέλλα Καρτέλα Καρτέλλες'
 ]
 synonyms_Originals.forEach( grp => {
@@ -229,34 +276,38 @@ synonyms_Originals.forEach( grp => {
 // console.log(synonym_kbs); process.exit();
 
 
-// significant terms ///////////////////////////////////////////////////////////
-// -----------------------------------------------------------------------------
-significantExceptios = sanitize_GR('7-UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ')
 
+/** significant terms //////////////////////////////////////////////////////////
+ * -----------------------------------------------------------------------------
+ */
+significantExceptios = sanitize_GR('7-UP 7UP 3-ΑΛΦΑ 17 3Π 7-DAYS K2R').split(' ')
 
-// replaces (correcting descriptions)
-// = construct unequivocally liked words ///////////////////////////////////////
-// -----------------------------------------------------------------------------
 
-// NOTE: TODO:
-// in future implementations multi-word keywords
-// may use the non-breaking space as conecting character (\u00A0) instead of dush (-)
-// (or maybe both of them)
-// --- -- -- - - -
+/** replaces (correcting descriptions)
+ * = construct unequivocally liked words ///////////////////////////////////////
+ * -----------------------------------------------------------------------------
+ * 
+ * NOTE: TODO:
+ * in future implementations multi-word keywords
+ * may use the non-breaking space as conecting character (\u00A0) instead of dush (-)
+ * (or maybe both of them)
+ * --- -- -- - - -
+ */
 replaces = [];
 replaceSource = [
+    '3Α;3-ΑΛΦΑ',
     '3 ΑΛΦΑ;3-ΑΛΦΑ',
     'HEAD & SHOULDERS;HEAD&SHOULDERS',
     'HEAD N SHOULDERS;HEAD&SHOULDERS',
     'W.K Kellogg; W-K-Kellogg',
     'W.K Kellogg;',
     '7 DAYS;7-DAYS',
-    '7 UP;7-UP',
+    '7 UP;7UP',
+    '7-UP;7UP',
     'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ',
     'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ',
     'COCA COLA;COCA-COLA',
     'COCACOLA;COCA-COLA',
-    // 'KELLOGS;KELLOGG\'S',
     'NES CAFE;NESCAFE',
     'NES-CAFE;NESCAFE',
     'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS',
@@ -291,6 +342,7 @@ replaceSource = [
     'EL SABOR;EL-SABOR',
     'ELSABOR;EL-SABOR',
     'DOUWE EGBERTS;DOUWE-EGBERTS',
+    'DOUWEEGBERTS;DOUWE-EGBERTS',
     'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ',
     'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ',
     'SPIN SPAN;SPIN-SPAN',
@@ -471,11 +523,25 @@ function freq_of(id) {
 }
 
 
-/**  proccess data
+/**  proccess data /////////////////////////////////////////////////////////////
  * --- -- -- - - -
  * 
  * central proccessing procedure
  * -----------------------------------------------------------------------------
+ * 
+ * Operation:
+ * ---
+ * for each title {
+ *   + clean text, get frequency
+ *   + preproccess and split into individual words
+ *   + for each significant word {
+ *       + update root keywords
+ *       + update linked words
+ *   }
+ * }
+ * post-process results
+ * 
+ * /////////////////////////////////////////////////////////////////////////////
  */
 function extract_linked_keywords(obj) {   
     // console.log(JSON.stringify(obj, null, 2));
@@ -635,18 +701,6 @@ function connect_keys( a, b, id, f ) {
 
 
 
-/** clean text
- * --- -- -- - - -
- * removes non keyword characters [+ . , !] and internal multiple-spaces
- * @param txt (string): product description
- */
-function clean_text(txt) {
-    return txt.replace('+',' ').replace('.',' ').replace(',',' ')   // change to space
-        .replace('!','').replace('\"', '')                  // remove character
-        .replace('   ',' ').replace('  ',' ');          // remove multiple spaces
-}
-
-
 /** is significant
  * --- -- -- - - -
  * check if term is significant (and needs to be indexed)
@@ -685,15 +739,12 @@ function synonym_keys(w) {
 
 /** mark_explicit_links
  * 
- * mark linked words
- * shall be handled as one-(key)word
+ * mark linked words shall be handled as one-(key)word
+ * (also edit common mistakes with suggested replaces)
  * 
  * @param str 
  * @return 
  */
-
-// edit common mistakes
-// with suggested replaces
 function mark_explicit_links(str) {
     str = ' '+ str +' ';
     replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); });
@@ -701,8 +752,12 @@ function mark_explicit_links(str) {
 }
 
 
-// preproccess description
-// ---
+/** preproccess description
+ * == do...
+ * + clean
+ * + sanitize
+ * + mark lined words
+ */ 
 function preproccess_text(description) {
 
     str = clean_text(description);  // remove non-keyword characters
@@ -817,6 +872,11 @@ async function upload_file( bucketName, srcFilePath, trgFilePath ) {
  * 
  * nodejs's exported function
  * used as entry-point function (in case of Google Cloud function)
+ * 
+ * Operation
+ * + 1: load products and frequencies
+ * + 2: forward to extract_linked_keywords()
+ * + 3: Save results (localy, then to cloud)
  */
 
 // keep only 1 of next 2 lines