compare supplamentary search core implementations (for vs find, etc)

2 files changed, 67 insertions, 36 deletions

diff --git a/pieces/prepare-streams.js b/pieces/prepare-streams.js
index 9a33604..4fa0a7e 100644
--- a/pieces/prepare-streams.js
+++ b/pieces/prepare-streams.js
@@ -1,10 +1,14 @@
-// const fs = require("fs");
-const fs = require('node:fs');
+const fs = require('fs');
+const kb = require('./kb-util.js');
+
+var request = require('request');
+
 // const https = require("https");
-// var request = require('request-promise');
 
-/*
-var calls = [
+
+/* TODO: ?? parallel read
+  // var request = require('request-promise');
+  var calls = [
     request({
         url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json',
         // headers: { ... }
@@ -15,46 +19,73 @@ var calls = [
     }),
     // + linked-terms and replaces
     // and more..
-];
-
-Promise.all(calls).then(function(results) {
-  // do something with results[0]
-  // do something with results[1]
-  // ...
-});
+  ];
+  
+  Promise.all(calls).then(function(results) {
+    // do something with results[0]
+    // do something with results[1]
+    // ...
+  });
 */
 
 
-// write files into local fs; use them cached
-/*
-let file = fs.createWriteStream('../data/emarket-products.json');
-
-https.get('https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', response => {
-    var stream = response.pipe(file);
+/** create entity file from url
+ * @param {string} entity: use the entity name 
+ * @param {string} url: prepared entity json-file in remote server
+ */
+function create(entity, url) {
+    request(url,
+        function (error, response, body) {
+            if (!error && response.statusCode == 200) {
+                // body is a ready json-string; no need to parse and (re-)stringify
+                try {
+                    fs.writeFileSync(`${__dirname}/../data/${entity}.json`, body, 'utf-8');
+                    // file written successfully
+                } catch (err) {
+                    console.error(err);
+                }
+            }
+        }
+    );
+}
 
-    stream.on("finish", function() {
-        console.log("done");
-    });
-});
 
-var response = {
-    success: true,
-    products: 100,
-    ttl: + new Date()
+function load_products(url) {
+    request(url,
+        function (error, response, body) {
+            if (!error && response.statusCode == 200) {
+                // body is a ready json-string; no need to parse and (re-)stringify
+                json = JSON.parse(body);
+                newJson = [];
+                json.forEach( p => {
+                    newJson.push({
+                        id: p.id,
+                        w: p.w,
+                        kb: kb.keyboardize(kb.clean(p.w))
+                    });
+                });
+                try {
+                    fs.writeFileSync(`${__dirname}/../data/products.json`, JSON.stringify(newJson), 'utf-8');
+                    // file written successfully
+                } catch (err) {
+                    console.error(err);
+                }
+            }
+        }
+    );
 }
 
-*/
 
-function create() {
+// create data
+// testing an implemenatatin of a custom local-fs-cache system
+function random() {
     var data = [];
-
-    for( let i = 0; i < (Math.floor(Math.random() * 100) +1) ; i++) {
+    for( let i = 0; i < (Math.floor(Math.random() * 100) +15) ; i++) {
         data.push({ id: i, x5: i*5 });
     }
-    console.log(data);
-
+    
     try {
-        fs.writeFileSync(__dirname + '/../data/j.json', JSON.stringify(data), 'utf-8');
+        fs.writeFileSync(__dirname + '/../data/random.json', JSON.stringify(data), 'utf-8');
         // file written successfully
     } catch (err) {
         console.error(err);
@@ -62,7 +93,8 @@ function create() {
 }
 
 
-
 module.exports = {
-    create
+    create,
+    random,
+    load_products
 }
\ No newline at end of file
diff --git a/pieces/search.js b/pieces/search.js
index 4bfb48c..52ce4e3 100644
--- a/pieces/search.js
+++ b/pieces/search.js
@@ -1,5 +1,4 @@
 
-
 /** Search Engine
  * ---------------------------------------------------------------------------
  *