From 2676484041e8771257bf99ed223bd9321fe72fa8 Mon Sep 17 00:00:00 2001 From: Geo Halkiadakis Date: Wed, 10 Apr 2024 13:29:01 +0300 Subject: working on a custom fs-cache system --- .gitignore | 1 + benchmark/product-from-id.js | 0 paths.js | 19 ++++++++++ pieces/prepare-streams.js | 46 +++++++++++++++++++---- pieces/search.js | 87 +------------------------------------------- pieces/suggest.js | 8 ++-- 6 files changed, 66 insertions(+), 95 deletions(-) create mode 100644 benchmark/product-from-id.js diff --git a/.gitignore b/.gitignore index c2658d7..902b281 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules/ +data/ diff --git a/benchmark/product-from-id.js b/benchmark/product-from-id.js new file mode 100644 index 0000000..e69de29 diff --git a/paths.js b/paths.js index 2653ff3..7d33877 100644 --- a/paths.js +++ b/paths.js @@ -1,5 +1,7 @@ const Router = require('koa-router'); const kb = require('./pieces/kb-util.js'); +const data = require('./pieces/prepare-streams'); +const prod = require('./data/j.json'); // Prefix all routes with: /items const router = new Router({ @@ -39,6 +41,10 @@ const router = new Router({ */ +// NOTE: set timeout per route (how-to) +// https://stackoverflow.com/questions/66634123/how-to-add-individual-timeout-value-per-specific-route-using-node-js + + // Routes router.get('/search', (ctx, next) => { @@ -68,4 +74,17 @@ router.get('/test', (ctx, next) => { // easy test route next(); }); +router.get('/test/do-data', (ctx, next) => { // easy test route + // test anything ... + data.create(); + ctx.body = { success: true, operation: 'create new data' }; + next(); +}); + +router.get('/test/json', (ctx, next) => { // easy test route + // test anything ... + ctx.body = { t: + new Date(), p: prod }; + next(); +}); + module.exports = router; \ No newline at end of file diff --git a/pieces/prepare-streams.js b/pieces/prepare-streams.js index 0e10e27..9a33604 100644 --- a/pieces/prepare-streams.js +++ b/pieces/prepare-streams.js @@ -1,7 +1,9 @@ -const fs = require("fs"); +// const fs = require("fs"); +const fs = require('node:fs'); // const https = require("https"); -var request = require('request-promise'); +// var request = require('request-promise'); +/* var calls = [ request({ url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', @@ -20,17 +22,47 @@ Promise.all(calls).then(function(results) { // do something with results[1] // ... }); - +*/ // write files into local fs; use them cached +/* +let file = fs.createWriteStream('../data/emarket-products.json'); -let file = fs.createWriteStream("data.txt"); - -https.get("https://www.w3.org/TR/PNG/iso_8859-1.txt", response => { +https.get('https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', response => { var stream = response.pipe(file); stream.on("finish", function() { console.log("done"); }); -}); \ No newline at end of file +}); + +var response = { + success: true, + products: 100, + ttl: + new Date() +} + +*/ + +function create() { + var data = []; + + for( let i = 0; i < (Math.floor(Math.random() * 100) +1) ; i++) { + data.push({ id: i, x5: i*5 }); + } + console.log(data); + + try { + fs.writeFileSync(__dirname + '/../data/j.json', JSON.stringify(data), 'utf-8'); + // file written successfully + } catch (err) { + console.error(err); + } +} + + + +module.exports = { + create +} \ No newline at end of file diff --git a/pieces/search.js b/pieces/search.js index 6606272..4bfb48c 100644 --- a/pieces/search.js +++ b/pieces/search.js @@ -1,3 +1,5 @@ + + /** Search Engine * --------------------------------------------------------------------------- * @@ -81,92 +83,7 @@ function retrosearch_module(options) { - /** 2.PURIFY STRING FUNCTIONS - * ------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////// - - - /** keyboardize - * ------------------------------------------------------------------------- - * translates string to keyboard-latin keys - * (the ones that used whan typing each letter of the word) - * - * @param str (string): original string (utf8 of latin or greek subgroups) - * @return (string): latin/ascii equivalent string - */ - - // cache (keeo in global) any-character to keyboard-latin mapping - // --- -- -- - - - - var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); - var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); - const map = new Map(); - for (var i=0; i { - ap = pair.split(' '); - accented_vowels.push({ - a: ap[0], // accented - p: ap[1] // pure = non accended - }); - }) - - // the actual `sanitize_GR` function code - // --- -- -- - - - - function sanitize_GR(str) { - - str = str.toLowerCase(); - // replace accended vowels with pure ones - accented_vowels.forEach( v => { - str = str.replaceAll(v.a, v.p); - }); - - // replace sigma on the end of words - str = str + ' '; - str = str.replaceAll('σ-', 'ς-'); - str = str.replaceAll('σ ', 'ς '); - - return str; - } - - - /** clean text - * --- -- -- - - - - * removes non keyword characters [+ . , !] and internal multiple-spaces - * @param txt (string): product description - */ - function clean_text(txt) { - return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space - .replace('!','').replace('\"', '') // remove character - .replace(' ',' ').replace(' ',' '); // remove multiple spaces - } - /** 3. SUPPLEMENTARY FUNCTIONS diff --git a/pieces/suggest.js b/pieces/suggest.js index ab62cfd..a420edd 100644 --- a/pieces/suggest.js +++ b/pieces/suggest.js @@ -1,7 +1,10 @@ const kb = require('./kb-util.js'); const match = require('./match-util.js'); -var n = 2; // Ngram base +var _allowFuzzy = true; // enable|disable fuzzy search +var n = 2; // Ngram base +var _fuzzyLimit = .5; // minimum bigram score for being considered a match + /** (Search) SUGGESTIONS ENGINE * --------------------------------------------------------------------------- @@ -75,8 +78,7 @@ function suggestions_module(options) { var _maxResults = 24; // limit suggestions var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions var _timeout_ms = 100; // time (in ms) for the search engine to find matches (before rendering) - var _allowFuzzy = true; // enable|disable fuzzy search - var _fuzzyLimit = .5; // minimum bigram score for being considered a match + var _Ngram_base = 2; // number of N in Ngram spliting algorithm var _isReady = false; // whether the searchbox is ready to be used -- cgit v1.2.3