working on a custom fs-cache system

6 files changed, 66 insertions, 95 deletions

diff --git a/.gitignore b/.gitignore
index c2658d7..902b281 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 node_modules/
+data/
diff --git a/benchmark/product-from-id.js b/benchmark/product-from-id.js
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/benchmark/product-from-id.js
diff --git a/paths.js b/paths.js
index 2653ff3..7d33877 100644
--- a/paths.js
+++ b/paths.js
@@ -1,5 +1,7 @@
 const Router = require('koa-router');
 const kb = require('./pieces/kb-util.js');
+const data = require('./pieces/prepare-streams');
+const prod = require('./data/j.json');
 
 // Prefix all routes with: /items
 const router = new Router({
@@ -39,6 +41,10 @@ const router = new Router({
 */
 
 
+// NOTE: set timeout per route (how-to)
+// https://stackoverflow.com/questions/66634123/how-to-add-individual-timeout-value-per-specific-route-using-node-js
+
+
 // Routes
 
 router.get('/search', (ctx, next) => {
@@ -68,4 +74,17 @@ router.get('/test', (ctx, next) => {    // easy test route
     next();
 });
 
+router.get('/test/do-data', (ctx, next) => {    // easy test route
+    // test anything ...
+    data.create();
+    ctx.body = { success: true, operation: 'create new data' };
+    next();
+});
+
+router.get('/test/json', (ctx, next) => {    // easy test route
+    // test anything ...
+    ctx.body = { t: + new Date(), p: prod };
+    next();
+});
+
 module.exports = router;
\ No newline at end of file
diff --git a/pieces/prepare-streams.js b/pieces/prepare-streams.js
index 0e10e27..9a33604 100644
--- a/pieces/prepare-streams.js
+++ b/pieces/prepare-streams.js
@@ -1,7 +1,9 @@
-const fs = require("fs");
+// const fs = require("fs");
+const fs = require('node:fs');
 // const https = require("https");
-var request = require('request-promise');
+// var request = require('request-promise');
 
+/*
 var calls = [
     request({
         url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json',
@@ -20,17 +22,47 @@ Promise.all(calls).then(function(results) {
   // do something with results[1]
   // ...
 });
-
+*/
 
 
 // write files into local fs; use them cached
+/*
+let file = fs.createWriteStream('../data/emarket-products.json');
 
-let file = fs.createWriteStream("data.txt");
-
-https.get("https://www.w3.org/TR/PNG/iso_8859-1.txt", response => {
+https.get('https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', response => {
     var stream = response.pipe(file);
 
     stream.on("finish", function() {
         console.log("done");
     });
-});
\ No newline at end of file
+});
+
+var response = {
+    success: true,
+    products: 100,
+    ttl: + new Date()
+}
+
+*/
+
+function create() {
+    var data = [];
+
+    for( let i = 0; i < (Math.floor(Math.random() * 100) +1) ; i++) {
+        data.push({ id: i, x5: i*5 });
+    }
+    console.log(data);
+
+    try {
+        fs.writeFileSync(__dirname + '/../data/j.json', JSON.stringify(data), 'utf-8');
+        // file written successfully
+    } catch (err) {
+        console.error(err);
+    }
+}
+
+
+
+module.exports = {
+    create
+}
\ No newline at end of file
diff --git a/pieces/search.js b/pieces/search.js
index 6606272..4bfb48c 100644
--- a/pieces/search.js
+++ b/pieces/search.js
@@ -1,3 +1,5 @@
+
+
 /** Search Engine
  * ---------------------------------------------------------------------------
  * 
@@ -81,92 +83,7 @@ function retrosearch_module(options) {
     
     
 
-    /** 2.PURIFY STRING FUNCTIONS
-       * -------------------------------------------------------------------------
-       *//////////////////////////////////////////////////////////////////////////
-  
-  
-    /** keyboardize
-     * -------------------------------------------------------------------------
-     * translates string to keyboard-latin keys
-     * (the ones that used whan typing each letter of the word)
-     * 
-     * @param str (string): original string (utf8 of latin or greek subgroups)
-     * @return (string): latin/ascii equivalent string
-     */
-
-    // cache (keeo in global) any-character to keyboard-latin mapping
-    // --- -- -- - - -
-    var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
-    var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
-    const map = new Map();
-    for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
-
-    // "keyboardize" function
-    // --- -- -- - - -
-    function keyboardize(str) {
-        str = str.replace('\'','');
-        var out = '';
-        // [map]'s implementation is 40x faster than [for]'s
-        for (var i=0 ; i< str.length; i++)  out += map.get(str[i]);
-        return out;
-    }
-  
-    /** sanitize_GR
-     * -------------------------------------------------------------------------
-     * replaces greek accended vowels with non accended ones
-     * takes care of sigma on the end of words
-     * 
-     * @param {string} str
-     * @return {string} sanitized
-     */
-  
-    // first cache (=create a global array)
-    // of accended to non-accended vowels mapping
-    // --- -- -- - - -
-    accented_vowels = [];   
-    [
-        'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ΐ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω',
-        'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω'
-    ].forEach( pair => {
-        ap = pair.split(' ');
-        accented_vowels.push({
-            a: ap[0],   // accented
-            p: ap[1]    // pure = non accended
-        });
-    })
-
-    // the actual `sanitize_GR` function code
-    // --- -- -- - - -
-    function sanitize_GR(str) {
-
-        str = str.toLowerCase();
 
-        // replace accended vowels with pure ones
-        accented_vowels.forEach( v => {
-            str = str.replaceAll(v.a, v.p);
-        });
-
-        // replace sigma on the end of words
-        str = str + ' ';
-        str = str.replaceAll('σ-', 'ς-');
-        str = str.replaceAll('σ ', 'ς ');
-        
-        return str;
-    }
-  
-  
-    /** clean text
-     * --- -- -- - - -
-     * removes non keyword characters [+ . , !] and internal multiple-spaces
-     * @param txt (string): product description
-     */
-    function clean_text(txt) {
-        return txt.replace('+',' ').replace('.',' ').replace(',',' ')   // change to space
-            .replace('!','').replace('\"', '')                  // remove character
-            .replace('   ',' ').replace('  ',' ');          // remove multiple spaces
-    }
-  
  
   
     /** 3. SUPPLEMENTARY FUNCTIONS
diff --git a/pieces/suggest.js b/pieces/suggest.js
index ab62cfd..a420edd 100644
--- a/pieces/suggest.js
+++ b/pieces/suggest.js
@@ -1,7 +1,10 @@
 const kb = require('./kb-util.js');
 const match = require('./match-util.js');
 
-var n = 2;    // Ngram base
+var _allowFuzzy = true;     // enable|disable fuzzy search
+var n = 2;                  // Ngram base
+var _fuzzyLimit = .5;       // minimum bigram score for being considered a match
+
 
 /** (Search) SUGGESTIONS ENGINE
  * ---------------------------------------------------------------------------
@@ -75,8 +78,7 @@ function suggestions_module(options) {
     var _maxResults = 24;     // limit suggestions
     var _blendProds = 4;      // minimum final-produncts to blend with next-word suggestions
     var _timeout_ms = 100;    // time (in ms) for the search engine to find matches (before rendering)
-    var _allowFuzzy = true;   // enable|disable fuzzy search
-    var _fuzzyLimit = .5;     // minimum bigram score for being considered a match
+    
     var _Ngram_base = 2;      // number of N in Ngram spliting algorithm
     var _isReady = false;     // whether the searchbox is ready to be used