1 files changed, 128 insertions, 0 deletions
diff --git a/app/pieces/prepare.js b/app/pieces/prepare.js
new file mode 100644
index 0000000..20825cd
--- /dev/null
+++ b/app/pieces/prepare.js
@@ -0,0 +1,128 @@
+const fs = require('fs');
+var request = require('request');
+
+const kb = require('../utils/kb-util.js');
+
+
+// const https = require("https");
+
+
+/* TODO: ?? parallel read
+  // var request = require('request-promise');
+  var calls = [
+    request({
+        url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json',
+        // headers: { ... }
+    }),
+    request({
+        url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json',
+        // headers: { ... }
+    }),
+    // + linked-terms and replaces
+    // and more..
+  ];
+  
+  Promise.all(calls).then(function(results) {
+    // do something with results[0]
+    // do something with results[1]
+    // ...
+  });
+*/
+
+
+/** create entity file from url
+ * used for entities like 'linked-words', 'synonyms', etc.
+ * 
+ * @param {string} entity: use the entity name 
+ * @param {string} url: prepared entity json-file in remote server
+ */
+function create(entity, url) {
+    request(url,
+        function (error, response, body) {
+            if (!error && response.statusCode == 200) {
+                // body is a ready json-string; no need to parse and (re-)stringify
+                try {
+                    fs.writeFileSync(`${__dirname}/../../data/${entity}.json`, body, 'utf-8');
+                    // file written successfully
+                } catch (err) {
+                    console.error(err);
+                }
+            }
+        }
+    );
+}
+
+function get_data_structure() {
+    // data structure is an array of `{ url:.., expiration:.., path:.. }` objects
+}
+
+function set_data_structure(node) {
+
+}
+
+
+function load_products(url) {
+    request(url,
+        function (error, response, body) {
+            if (!error && response.statusCode == 200) {
+                // body is a ready json-string; no need to parse and (re-)stringify
+                json = JSON.parse(body);
+                newJson = [];
+
+                // TODO:
+                // + attach handle synonyms
+                // + mark brand-names
+                // + attach category-names and SAP-categories
+                // + construct combo words
+                // + remove non-important words
+                // + normalize popularity
+
+                json.forEach( p => {
+                    newJson.push({
+                        id: p.id,
+                        w: p.w,
+                        kb: kb.keyboardize(kb.clean(p.w))
+                    });
+                });
+                
+                try {
+
+                    fs.writeFileSync(
+                        `${__dirname}/../../data/products.json`,
+                        JSON.stringify(newJson),
+                        'utf-8'
+                    );
+                    return true;    // file written successfully
+
+                } catch (err) {
+                    console.error(err);
+                    return false;
+                }
+            }
+        }
+    );
+}
+
+
+// create data
+// testing an implemenatatin of a custom local-fs-cache system
+function random() {
+    var data = [];
+    for( let i = 0; i < (Math.floor(Math.random() * 100) +15) ; i++) {
+        data.push({ id: i, x5: i*5 });
+    }
+    
+    try {
+        fs.writeFileSync(__dirname + '/../data/random.json', JSON.stringify(data), 'utf-8');
+        // file written successfully
+    } catch (err) {
+        console.error(err);
+    }
+}
+
+
+module.exports = {
+    create,
+    random,
+    load_products
+}
+\ No newline at end of file