From 6248d2cf7a2214ac32628bc58248694108b4d8bf Mon Sep 17 00:00:00 2001 From: George Halkiadakis Date: Tue, 16 Apr 2024 01:55:02 +0300 Subject: optimize weighted_partial match --- pieces/prepare-streams.js | 101 ------------------------------------ pieces/prepare.js | 128 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 101 deletions(-) delete mode 100644 pieces/prepare-streams.js create mode 100644 pieces/prepare.js (limited to 'pieces') diff --git a/pieces/prepare-streams.js b/pieces/prepare-streams.js deleted file mode 100644 index ee7ba8e..0000000 --- a/pieces/prepare-streams.js +++ /dev/null @@ -1,101 +0,0 @@ -const fs = require('fs'); -var request = require('request'); - -const kb = require('../utils/kb-util.js'); - - -// const https = require("https"); - - -/* TODO: ?? parallel read - // var request = require('request-promise'); - var calls = [ - request({ - url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', - // headers: { ... } - }), - request({ - url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', - // headers: { ... } - }), - // + linked-terms and replaces - // and more.. - ]; - - Promise.all(calls).then(function(results) { - // do something with results[0] - // do something with results[1] - // ... - }); -*/ - - -/** create entity file from url - * @param {string} entity: use the entity name - * @param {string} url: prepared entity json-file in remote server - */ -function create(entity, url) { - request(url, - function (error, response, body) { - if (!error && response.statusCode == 200) { - // body is a ready json-string; no need to parse and (re-)stringify - try { - fs.writeFileSync(`${__dirname}/../data/${entity}.json`, body, 'utf-8'); - // file written successfully - } catch (err) { - console.error(err); - } - } - } - ); -} - - -function load_products(url) { - request(url, - function (error, response, body) { - if (!error && response.statusCode == 200) { - // body is a ready json-string; no need to parse and (re-)stringify - json = JSON.parse(body); - newJson = []; - json.forEach( p => { - newJson.push({ - id: p.id, - w: p.w, - kb: kb.keyboardize(kb.clean(p.w)) - }); - }); - try { - fs.writeFileSync(`${__dirname}/../data/products.json`, JSON.stringify(newJson), 'utf-8'); - // file written successfully - } catch (err) { - console.error(err); - } - } - } - ); -} - - -// create data -// testing an implemenatatin of a custom local-fs-cache system -function random() { - var data = []; - for( let i = 0; i < (Math.floor(Math.random() * 100) +15) ; i++) { - data.push({ id: i, x5: i*5 }); - } - - try { - fs.writeFileSync(__dirname + '/../data/random.json', JSON.stringify(data), 'utf-8'); - // file written successfully - } catch (err) { - console.error(err); - } -} - - -module.exports = { - create, - random, - load_products -} \ No newline at end of file diff --git a/pieces/prepare.js b/pieces/prepare.js new file mode 100644 index 0000000..65ccee1 --- /dev/null +++ b/pieces/prepare.js @@ -0,0 +1,128 @@ +const fs = require('fs'); +var request = require('request'); + +const kb = require('../utils/kb-util.js'); + + +// const https = require("https"); + + +/* TODO: ?? parallel read + // var request = require('request-promise'); + var calls = [ + request({ + url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', + // headers: { ... } + }), + request({ + url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', + // headers: { ... } + }), + // + linked-terms and replaces + // and more.. + ]; + + Promise.all(calls).then(function(results) { + // do something with results[0] + // do something with results[1] + // ... + }); +*/ + + +/** create entity file from url + * used for entities like 'linked-words', 'synonyms', etc. + * + * @param {string} entity: use the entity name + * @param {string} url: prepared entity json-file in remote server + */ +function create(entity, url) { + request(url, + function (error, response, body) { + if (!error && response.statusCode == 200) { + // body is a ready json-string; no need to parse and (re-)stringify + try { + fs.writeFileSync(`${__dirname}/../data/${entity}.json`, body, 'utf-8'); + // file written successfully + } catch (err) { + console.error(err); + } + } + } + ); +} + +function get_data_structure() { + // data structure is an array of `{ url:.., expiration:.., path:.. }` objects +} + +function set_data_structure(node) { + +} + + +function load_products(url) { + request(url, + function (error, response, body) { + if (!error && response.statusCode == 200) { + // body is a ready json-string; no need to parse and (re-)stringify + json = JSON.parse(body); + newJson = []; + + // TODO: + // + attach handle synonyms + // + mark brand-names + // + attach category-names and SAP-categories + // + construct combo words + // + remove non-important words + // + normalize popularity + + json.forEach( p => { + newJson.push({ + id: p.id, + w: p.w, + kb: kb.keyboardize(kb.clean(p.w)) + }); + }); + + try { + + fs.writeFileSync( + `${__dirname}/../data/products.json`, + JSON.stringify(newJson), + 'utf-8' + ); + return true; // file written successfully + + } catch (err) { + console.error(err); + return false; + } + } + } + ); +} + + +// create data +// testing an implemenatatin of a custom local-fs-cache system +function random() { + var data = []; + for( let i = 0; i < (Math.floor(Math.random() * 100) +15) ; i++) { + data.push({ id: i, x5: i*5 }); + } + + try { + fs.writeFileSync(__dirname + '/../data/random.json', JSON.stringify(data), 'utf-8'); + // file written successfully + } catch (err) { + console.error(err); + } +} + + +module.exports = { + create, + random, + load_products +} \ No newline at end of file -- cgit v1.2.3