From d9364679a51ff80db8e5948ab089d749da36a6b2 Mon Sep 17 00:00:00 2001 From: Geo Halkiadakis Date: Wed, 17 Apr 2024 18:32:13 +0300 Subject: dockerize the app --- .gitignore | 2 +- Dockerfile | 16 + README.md | 69 +++- app.js | 20 -- app/app.js | 21 ++ app/benchmark/find.js | 112 +++++++ app/benchmark/match-str.js | 80 +++++ app/pieces/prepare.js | 128 ++++++++ app/pieces/retro-search.js | 305 ++++++++++++++++++ app/pieces/suggest.js | 786 +++++++++++++++++++++++++++++++++++++++++++++ app/routes/dev.js | 103 ++++++ app/routes/index.js | 26 ++ app/routes/v1.js | 68 ++++ app/utils/kb-util.js | 88 +++++ app/utils/match-util.js | 154 +++++++++ app/utils/mem-usage.js | 20 ++ app/utils/url-util.js | 23 ++ benchmark/find.js | 105 ------ benchmark/match-str.js | 74 ----- data/.gitkeep | 0 data/products.null | 1 + docker-compose.yml | 16 + package.json | 2 +- pieces/prepare.js | 128 -------- pieces/retro-search.js | 304 ------------------ pieces/suggest.js | 786 --------------------------------------------- routes/dev.js | 103 ------ routes/index.js | 26 -- routes/v1.js | 67 ---- utils/kb-util.js | 88 ----- utils/match-util.js | 154 --------- utils/mem-usage.js | 20 -- utils/url-util.js | 23 -- 33 files changed, 2015 insertions(+), 1903 deletions(-) create mode 100644 Dockerfile delete mode 100644 app.js create mode 100644 app/app.js create mode 100644 app/benchmark/find.js create mode 100644 app/benchmark/match-str.js create mode 100644 app/pieces/prepare.js create mode 100644 app/pieces/retro-search.js create mode 100644 app/pieces/suggest.js create mode 100644 app/routes/dev.js create mode 100644 app/routes/index.js create mode 100644 app/routes/v1.js create mode 100644 app/utils/kb-util.js create mode 100644 app/utils/match-util.js create mode 100644 app/utils/mem-usage.js create mode 100644 app/utils/url-util.js delete mode 100644 benchmark/find.js delete mode 100644 benchmark/match-str.js create mode 100644 data/.gitkeep create mode 100644 data/products.null create mode 100644 docker-compose.yml delete mode 100644 pieces/prepare.js delete mode 100644 pieces/retro-search.js delete mode 100644 pieces/suggest.js delete mode 100644 routes/dev.js delete mode 100644 routes/index.js delete mode 100644 routes/v1.js delete mode 100644 utils/kb-util.js delete mode 100644 utils/match-util.js delete mode 100644 utils/mem-usage.js delete mode 100644 utils/url-util.js diff --git a/.gitignore b/.gitignore index 1575068..5079808 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ node_modules/ -data/* +data/*.json .env diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9cf8f84 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM node:alpine +WORKDIR /home/node + +# using wildcard (*) to copy both package.json and package-lock.json +COPY package*.json /home/node/ + +# create a null-data json-files +COPY data/products.null /home/node/data/products.json +COPY package.json /home/node/package.json +RUN npm i + +# create and set app directory as current dir +WORKDIR /home/node/app +COPY app/ /home/node/app/ +EXPOSE 3000 +CMD ["node", "app.js"] \ No newline at end of file diff --git a/README.md b/README.md index 477d0cf..7f7c049 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# Local installation + clone from ... git clone git@code.roptron.gr:/var/www/git/oseine.git @@ -19,7 +21,68 @@ run npm start -prepare source for production: -* remove ``/bench`` routes from ``paths.js`` -* remove ``/bench`` folder + +# Docker container preparation + +create the container: + + docker build -t oseine . + +build and start the instance for first time: + + docker-compose up --build + +start instance (any other time): + + docker-compose up + + + +# Notes + +## 1 Web Server for a node-js app: Older vs newer approaces + +Old .NET applications, Java (for applets, oldschool) and PHP are the odd ones out in that they need a webserver to run. + +It’s not the case for most others: Node, Rust, Go, pure Java applications, C++ and so on. + +I mean that in the sense that a dedicated webserver is not necessary, though a reverse proxy might be a good idea in some cases - but in the general case, no, you don’t need a web server for Node. + + +## 2 Choosing between Node-js and Apache server (pros+cons) + +The most important thing to understand is that one is "generally" not better than the other. The selection depends on the case in hand. + +* Apache generally uses PHP as a scripting language, which is extremely easy for the beginners and it is built for Web. Node js is simply a javascript v8 engine which was not originally intended to serve web. + +* Apache and PHP are old and stable. Everything you need to develop a working web application is available with a lot of support. Node Js is the new kid in the block ( .. not so new.. but still..). It is not essentially built for developing Web applications. Http based Web application can be considered as one use of Node Js. + +* Many famous cms (Wordpress, Drupal, Joomla etc) and development frameworks (Yii, Laravel, Code Ignitor, Cake PHP) are build in PHP. These frameworks are very helpful in organizing the code base for a large application. Node Js also has web frameworks like Express and sails js but they are new and the support is not as readily available as compared to PHP frameworks. + +* Apache is thread and process based i.e each request is handled by a separate thread or process (depending upon configuration), which means if the process is waiting for the I/O, whole thread is blocked. Node JS has asynchronous, event driven I/O. Every nodejs instance runs in a single thread and due to its asynchronous nature, it can handle far more number of concurrent requests as compared to apache. + +* Apache is thread based, so if an unexpected problem occurs while processing a request, only that particular thread will crash leaving rest of the requests and threads intact. Node Js handles multiple requests using one single thread. If a problem occurs whole node js instance will crash along with any global data that was stored in javascript variables or arrays. It can be automatically restarted using "forever" or some other modules but the current requests and the data is gone. + +* When Apache gets a request which is CPU intensive, other request do not get blocked because of the context switching between the threads. when NodeJs gets a CPU intensive request, all the other requests get blocked till this CPU intensive request stops for an I/O. its a good idea to delegate CPU intensive requests to a worker or some other process while using Node Js. + +* PHP does not support web sockets natively (to my knowledge). There are libraries which can help implement web sockets, but again due to thread based model, a decent number of live web socket connections will eat up your server's resources. NodeJs is a perfect candidate for real time communication over internet. Combine it with Socket.IO and you will get a good scalable web socket server along with fallback communication mediums such as flash socket and long polling. Its light weight and it can scale pretty good because its all running in one single thread. + + +## 3. Reloading a module on demand on node-js + +See [this!](https://stackoverflow.com/questions/33546880/node-js-how-to-reload-module). + + function nocache(module) { + require("fs").watchFile(require("path").resolve(module), () => { + delete require.cache[require.resolve(module)] + }) + } + +The function will delete your module from the cache each time the file changes. To use it, just paste it in the REPL, call nocache("d:/myapp.js"), then use require normally + + nocache('path/myapp.js); + var myapp = require('path/myapp.js'); + // ... + myapp = require('path/myapp.js); + diff --git a/app.js b/app.js deleted file mode 100644 index b5fd29b..0000000 --- a/app.js +++ /dev/null @@ -1,20 +0,0 @@ -// app.js - -const Koa = require('koa'); -// const { koaBody } = require('koa-body'); - -// load parameters -require('dotenv').config(); - -// define app -const app = new Koa(); - -// middleware -// app.use(koaBody()); - -// load routes -let _r = require('./routes'); -app.use(_r.routes()).use(_r.allowedMethods()); - -// start server listening on APP_PORT -app.listen(process.env.APP_PORT); diff --git a/app/app.js b/app/app.js new file mode 100644 index 0000000..42ff8c1 --- /dev/null +++ b/app/app.js @@ -0,0 +1,21 @@ +// app.js + +const Koa = require('koa'); +// const { koaBody } = require('koa-body'); + +// load parameters +require('dotenv').config(); + +// define app +const app = new Koa(); + +// middleware +// app.use(koaBody()); + +// load routes +let _r = require('./routes'); +app.use(_r.routes()).use(_r.allowedMethods()); + +// start server listening on APP_PORT +// app.listen(process.env.APP_PORT); +app.listen(3000); diff --git a/app/benchmark/find.js b/app/benchmark/find.js new file mode 100644 index 0000000..1e86c24 --- /dev/null +++ b/app/benchmark/find.js @@ -0,0 +1,112 @@ +/** + * benchmark: find a product in product list + * using: for vs forEach vs find + */ + +const microtime = require('microtime'); + +var products = require('../../data/products.json'); +if (products.length == 0) { + const prepare = require('../pieces/prepare.js'); + prepare.load_products( + 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' + ); +} + + + +var selected = []; +products.forEach( pr => { + if (Math.floor(Math.random() * 100) > 85) { + selected.push(pr.id); + } +}); + + +function compare() { + + let n = 4; + + var f0 = microtime.nowDouble(); + for(i=0 ; i < n ; i++) byFor(); + var f1 = microtime.nowDouble(); + // + var e0 = microtime.nowDouble(); + for(i=0 ; i < n ; i++) byEach(); + var e1 = microtime.nowDouble(); + // + var b0 = microtime.nowDouble(); + for(i=0 ; i < n ; i++) byFind(); + var b1 = microtime.nowDouble(); + + return { + n: n, + + for: f1-f0, + items_for: byFor(), + + each: e1-e0, + items_each: byEach(), + + find: b1-b0, + items_find: byFind(), + + sel: selected, + } +} + + +function byFor() { + var items = []; + var notFound = []; + var found; + selected.forEach( id => { + found = false; + for(i = 0; i < products.length; i++) { + if (products[i].id == id) { + items.push(products[i]); + found = true; + break; + } + } + if (!found) notFound.push(id) + + }) + return {items: items, nf: notFound}; +} + + +function byEach() { + var items = []; + var notFound = []; + var found; + selected.forEach( id => { + found = false; + products.forEach( pr => { + if (pr.id == id) { + items.push(pr); + found = true; + } + }); + if (!found) notFound.push(id) + }); + return {items: items, nf: notFound}; +} + +function byFind() { + var items = []; + var notFound = []; + var result; + selected.forEach( id => { + found = false; + result = products.find((pr) => pr.id == id); + if (result === undefined) notFound.push(id) + else items.push(result); + }); + return {items: items, nf: notFound}; +} + +module.exports = { + compare, + byFor +} \ No newline at end of file diff --git a/app/benchmark/match-str.js b/app/benchmark/match-str.js new file mode 100644 index 0000000..225cbc2 --- /dev/null +++ b/app/benchmark/match-str.js @@ -0,0 +1,80 @@ +var microtime = require('microtime'); + +const match = require('../utils/match-util.js'); +const memory_usage = require('../utils/mem-usage.js'); +const kb = require('../utils/kb-util.js'); + +// const products = require('../../data/products.json'); +var products; +try { + products = require('../../data/products.json'); +} catch (e) { + products = []; +} + +// memory_usage.report(); + +// test runner +function run(query) { + + var f0 = microtime.nowDouble(); + let result; + // for(let i = 0 ; i < 20 ; i++) + result = matchQuery(query); + var f1 = microtime.nowDouble(); + + memory_usage.report(); + + return { + t: f1-f0, + q: query, + result: result + } +} + +// sample search implementation +function matchQuery(query) { + var result = []; + + let q = kb.keyboardize(query); + + for(let i = 0; i < products.length; i++) { + + let rate = 0; + let found = false; + let x; + let src = products[i].kb.split(' '); // (array) source of product (key-)words + if ((x = match.weighted_exact(q, src)) > 0) { + // console.log(products[i].kb, x); + rate = 5.0 + x; + found = true; + } + else if ((x = match.weighted_partial(q, src)) > 0) { + rate = 3.0 + x; + found = true; + } + else { + let similarity = 0; + src.forEach( w => { + let sim = match.resemblance(q, w, 2); + if (sim > 0.6) { + found = true; + similarity = (sim > similarity) ? sim : similarity; + } + }); + if (found) { + rate = 2 * similarity; + } + } + + if (found) { + products[i].rate = rate; + result.push(products[i]); + } + } + + return result.sort((a,b) => b.rate - a.rate).slice(0, 48);; +} + + +module.exports = { run } diff --git a/app/pieces/prepare.js b/app/pieces/prepare.js new file mode 100644 index 0000000..20825cd --- /dev/null +++ b/app/pieces/prepare.js @@ -0,0 +1,128 @@ +const fs = require('fs'); +var request = require('request'); + +const kb = require('../utils/kb-util.js'); + + +// const https = require("https"); + + +/* TODO: ?? parallel read + // var request = require('request-promise'); + var calls = [ + request({ + url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', + // headers: { ... } + }), + request({ + url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', + // headers: { ... } + }), + // + linked-terms and replaces + // and more.. + ]; + + Promise.all(calls).then(function(results) { + // do something with results[0] + // do something with results[1] + // ... + }); +*/ + + +/** create entity file from url + * used for entities like 'linked-words', 'synonyms', etc. + * + * @param {string} entity: use the entity name + * @param {string} url: prepared entity json-file in remote server + */ +function create(entity, url) { + request(url, + function (error, response, body) { + if (!error && response.statusCode == 200) { + // body is a ready json-string; no need to parse and (re-)stringify + try { + fs.writeFileSync(`${__dirname}/../../data/${entity}.json`, body, 'utf-8'); + // file written successfully + } catch (err) { + console.error(err); + } + } + } + ); +} + +function get_data_structure() { + // data structure is an array of `{ url:.., expiration:.., path:.. }` objects +} + +function set_data_structure(node) { + +} + + +function load_products(url) { + request(url, + function (error, response, body) { + if (!error && response.statusCode == 200) { + // body is a ready json-string; no need to parse and (re-)stringify + json = JSON.parse(body); + newJson = []; + + // TODO: + // + attach handle synonyms + // + mark brand-names + // + attach category-names and SAP-categories + // + construct combo words + // + remove non-important words + // + normalize popularity + + json.forEach( p => { + newJson.push({ + id: p.id, + w: p.w, + kb: kb.keyboardize(kb.clean(p.w)) + }); + }); + + try { + + fs.writeFileSync( + `${__dirname}/../../data/products.json`, + JSON.stringify(newJson), + 'utf-8' + ); + return true; // file written successfully + + } catch (err) { + console.error(err); + return false; + } + } + } + ); +} + + +// create data +// testing an implemenatatin of a custom local-fs-cache system +function random() { + var data = []; + for( let i = 0; i < (Math.floor(Math.random() * 100) +15) ; i++) { + data.push({ id: i, x5: i*5 }); + } + + try { + fs.writeFileSync(__dirname + '/../data/random.json', JSON.stringify(data), 'utf-8'); + // file written successfully + } catch (err) { + console.error(err); + } +} + + +module.exports = { + create, + random, + load_products +} \ No newline at end of file diff --git a/app/pieces/retro-search.js b/app/pieces/retro-search.js new file mode 100644 index 0000000..4e26720 --- /dev/null +++ b/app/pieces/retro-search.js @@ -0,0 +1,305 @@ +const kb = require('../utils/kb-util.js'); +const match = require('../utils/match-util.js'); + +const products = require('../../data/products.json'); + + +/** VARIABLES + * may passed as module arguments + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + +var _allowFuzzy = true; // enable|disable fuzzy search +var n = 2; // Ngram base +var _fuzzyLimit = .5; // minimum bigram score for being considered a match +var max_list = 24; +var tolerance = 42; +var STORE = { id: 904 }; + +var _kwlinks; // keyword links (word-connections; imported via ajax-get) +var _products = []; // all products (imported via ajax-get) + +// setup options +var _maxResults = options.max_list; // limit suggestions +var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions +var _Ngram_base = 2; // number of N in Ngram spliting algorithm +var _isReady = false; // whether the searchbox is ready to be used + +// product keywords +var keywordsURL = options.keywords_json; + +var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } + + + + + + +/** SUPPLEMENTARY FUNCTIONS + * ----------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////////// + + + +// callback function for sorting resulrs per r (=rating) property +function compare_rate(a,b) { + return (a.r < b.r); +} + + + +/** SEARCH ENGINE + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + + +function matchWordInList(q, list = false) { + + if (list !== false && list.length == 0) return []; // no results + + var result = []; + var firstPass = false; + + if (list === false) { + firstPass = true; // on first pass + list = products; // list is all products + } + + for(let i = 0 ; i < list.length ; i++) { + + // check exact + rate + + // else check partial + rate + + // else check similarity + rate + + let found = false; + let similarity = 0; + products[i].kb.split(' ').forEach( w => { + let sim = match.resemblance(q, w, 2); + if (sim > 0.5) { + found = true; + similarity = (sim > similarity) ? sim : similarity; + } + }); + if (found) { + products[i].similarity = similarity; + products[i].rating = (firstPass) + ? similarity * 2.0 + : products[i].rating + similarity * 2.0 + result.push(products[i]); + } + } + + return result +} + + + +// suggestions engine ////////////////////////////////////////////////// +// --- +function suggestions_engine(query) { + var results = []; + var pot = []; pot.length = 0; + + // clean and sanitize and mark links onto q(uery) string + var q = kb.keyboardize( kb.sanitizeGR( kb.clean(query.trim()) ) ).trim(); + + // TODO: + // construct direct-linked words + // = do unequivocally replaces + // steps: + // 1. replace accented vowels with non accented ones + // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'` + + var qAr = q.split(' '); // split to words + /// if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array + /// + /// if (qAr.slice(-1) == "") { + /// qAr.pop(); + /// } + + + pot = _products; // potential results // NOTE: CRITICAL: BY REFERENCE + + var wi = 0; // word index (from list) + var wc = qAr.length; + + qAr.forEach( w => { + + let sf = []; // (matches) so far + let mi; // position of match + wi++; + + pot.forEach( it => { + let matched = false; + let tester = ' '+ it.kb + ' '; + + // reset previous history and ratings + if (wi == 1) { + it.r = 0; + it.history = []; + } + + + // rate word-match > start-match > simple-match + // ... up to 8 points + + if (tester.indexOf(' '+ w +' ') != -1) { + it.r += 9; + it.history.push({ w: w, rate: 9 }); + matched = true; + } + else if (tester.indexOf(' '+ w) != -1) { + it.r += 5; + it.history.push({ w: w, rate: 5 }); + matched = true; + } + else if (tester.indexOf(w) != -1) { + it.r += 2; + it.history.push({ w: w, rate: 2 }); + matched = true; + } + + // rate `near-to-start` matching .. up to 7p + // rate `earlyness` of word in query .. up to 7p + + if ((mi = tester.indexOf(' '+w)) != -1) { + let fc1 = 100 - ((mi < 99) ? mi : 99); // near-to-start factor + let fc2 = wc - wi + 1; // query earlyness factor + let r1 = Math.floor(7*fc1/100); + let r2 = Math.floor(7*fc2/wc); + + it.r += (r1 + r2); + it.history.push({ w: w, left: [fc1, r1], early: [fc2, r2] }); + } + if (matched) sf.push(it); + }); + + if ((sf.length > (_maxResults + Math.floor(_maxResults/2))) + || (wi == 1) ) { + // ..if pot has a fair amount (= max + 50%) of results + // ..or these are results of '1st-query-word' + // set sf as new source + pot.lenght = 0; pot = []; + pot = JSON.parse(JSON.stringify(sf)); // copy by value + + } else { + // else.. keep the source list and increase of 'so-far rating' + // console.log('found small list', sf, pot) + pot.forEach( it => { + sf.forEach( si => { + if (it.id == si.id) { + it.r += 10; + it.history.push({ w: w, plus: '+10'}); + } + }); + }); + } + }); + + // sort results, get max-list of best rated + results = (pot.length > _maxResults) + ? pot.sort(compare_rate).slice(0, _maxResults) + : pot.sort(compare_rate) + + if (options.debug) console.log(results); + + return results; +} + + +// sub-module (start) +//////////////////////////////////////////////////////////////////////////// + +function update_common_search_results(q, results) { + let queries = getSessionObj('sr'); + let newSRlist = []; + let isnewQ = true; + if (queries === null) { + setSessionObj('sr', [{ + q: q, + result: result, + t: + new Date() + }]); + return true; + + } else { + + queries.forEach(it => { + if (it.q == q) { + newSRlist.push({ + q:q, + result: result, + t: + new Date() + }); + isnewQ = false; + } else { newSRlist.push(it); } + }); + + if (isnewQ) { + newSRlist.push({ + q:q, + result: result, + t: + new Date() + }); + } + + return true; + } +} + + + +//////////////////////////////////////////////////////////////////////////// +// sub-module (end) + + +function common_search(query) { + // clear ; sanitize ; split + var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' '); + + // if last item is empty, remove it + if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop() + + var results = _products; + + // for each key fitler results + qAr.forEach( key => { + results = key_sublist(key, results) + }); + + // echo products (and prepare list to POST) + var list_ = []; + results.forEach( item => { + if (options.debug) console.log(item.id, ':', item.w); + list_.push(item.id) + }) + + // *** TODO: keep results in local storage (or on session storage) + + // update_common_search_results(query, list_); + + let l = list_.join(','); + var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`); + + console.log('common search: search query > location = search') + window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`); + +} + +/** return from list only items that include 'key' + */ +function key_sublist(key, list) { + var result = []; + list.forEach( item => { + if (item.kb.includes(key)) { + result.push(item); + } + }); + + return result; +} diff --git a/app/pieces/suggest.js b/app/pieces/suggest.js new file mode 100644 index 0000000..c0a6e04 --- /dev/null +++ b/app/pieces/suggest.js @@ -0,0 +1,786 @@ +const kb = require('../utils/kb-util.js'); +const match = require('../utils/match-util.js'); + +var _allowFuzzy = true; // enable|disable fuzzy search +var n = 2; // Ngram base +var _fuzzyLimit = .5; // minimum bigram score for being considered a match + + +/** (Search) SUGGESTIONS ENGINE + * --------------------------------------------------------------------------- + * + * Operates in dual mode; + * -- suggestions engine (interactive) + * -- classic-like mode (passive) + * + * TODO: + * CRITICAL: (optimization) + * Search initialization uses quite a lot of network sources; + * thus it should be started in a later time; + * lets say ... after `x` seconds + * or... when document/core-ui-elements are ready + * + * @parametres (json) : options + * --- + * @var keywords_json (sring) : endpoint url of linked keywords structure + * @var products_json (string) : endpoint od product descriptions + * @var search_tag (string) : selector of field that shall act as typeahead-suggestions + * @var visualize_search_results_url (str) : url that will visualize the sended "results-page" + * @var debug (bool) : if true sends several debug console messages; if false mesagges are eliminated + */ + +suggestions_module({ + keywords_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', + products_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', + search_tag: '#tagsInput', + visualize_search_results_url: '/product_list', + debug: ((location.hostname == 'localhost') || (location.hostname == '127.0.0.1')) +}); + +function suggestions_module(options) { + + /** CONTENTS + * + * +1: Variables + * + * +2: Purify string functions + * + keyboardize + * + sanitize_GR + * + clean + * + mark_explicit_links + * + * +3: Supplementary function (vanilla js) + * + ajax_get(url, callback) + * + createNgram (fuzzy) + * + checkSimilarity (fuzzy) + * + check_match + * + is_exact_match + * + match_one + * + * +4: Actual data loading (async) + * + * +5: Suggestions Engine (jQuery) + * + suggestions_engine + * + */ + + console.log('executing suggestions...'); + + + /** 1. VARIABLES + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + var _kwlinks; // keyword links (word-connections; imported via ajax-get) + var _products = []; // all products (imported via ajax-get) + + // setup options + var _maxResults = 24; // limit suggestions + var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions + var _timeout_ms = 100; // time (in ms) for the search engine to find matches (before rendering) + + var _Ngram_base = 2; // number of N in Ngram spliting algorithm + var _isReady = false; // whether the searchbox is ready to be used + + // product keywords + var keywordsURL = options.keywords_json; + + var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } + + + /** replaces (correcting descriptions) + * == construct unequivocally liked words ////////////////////////////////// + * ------------------------------------------------------------------------- + * + * NOTE: TODO: + * in future implementations multi-word keywords + * may use the non-breaking space as conecting character (\u00A0) instead of dush (-) + * (or maybe both of them) + * + * also TODO: + * in future implementaions linked words may passed via some endpoint + */ + replaces = []; + replaceSource = [ + '3Α;3-ΑΛΦΑ', + '3 ΑΛΦΑ;3-ΑΛΦΑ', + 'HEAD & SHOULDERS;HEAD&SHOULDERS', + 'HEAD N SHOULDERS;HEAD&SHOULDERS', + 'W.K Kellogg; W-K-Kellogg', + 'W.K Kellogg;', + '7 DAYS;7-DAYS', + '7 UP;7UP', + '7-UP;7UP', + 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', + 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', + 'COCA COLA;COCA-COLA', + 'J P. CHENET; J.P.CHENET', + 'J.P. CHENET; J.P.CHENET', + 'COCACOLA;COCA-COLA', + 'NES CAFE;NESCAFE', + 'NES-CAFE;NESCAFE', + 'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS', + 'PETIT MARSEILLAIS;PETIT-MARSEILLAIS', + 'Το Μάννα;Το-Μάννα', + 'Χωρίς Γλουτένη;Χωρίς-Γλουτένη', + 'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη', + 'Χωρίς Αλάτι;Χωρίς-Αλάτι', + 'Χωρίς Λακτόζη;Χωρίς-Λακτόζη', + 'Χωρίς Συντηρητικά;Χωρίς-Συντηρητικά', + 'Χωρίς Αλκοόλ;Χωρίς-Αλκοόλ', + 'Χωρίς Kαφεϊνη;Χωρίς-Kαφεϊνη', + 'Χωρίς Kαφεΐνη;Χωρίς-Kαφεϊνη', + 'Χωρίς Γλυκάνισο;Χωρίς-Γλυκάνισο', + 'Χωρίς Ανθρακικό;Χωρίς-Ανθρακικό', + 'Υψηλής Παστερίωσης;Υψηλής-Παστερίωσης', + 'Ολικής Άλεσης;Ολικής-Άλεσης', + 'Ολικής Aλέσεως;Ολικής-Aλέσεως', + 'Χαρτί Υγείας;Χαρτί-Υγείας', + 'ρολό υγείας;ρολό-υγείας', + 'χαρτί τουαλέτας;χαρτί-τουαλέτας', + 'Χαρτί Κουζίνας;Χαρτί-Κουζίνας', + 'ρολό κουζίνας;ρολό-κουζίνας', + 'Μπάρες Δημητριακών;Μπάρες-Δημητριακών', + 'Ας Μαγειρέψουμε;Ας-Μαγειρέψουμε', + 'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', + 'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', + 'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ', + 'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ', + 'ΕΛ ΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', + 'ΕΛΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', + 'FREE STEP;FREE-STEP', + 'EL SABOR;EL-SABOR', + 'ELSABOR;EL-SABOR', + 'DOUWE EGBERTS;DOUWE-EGBERTS', + 'DOUWEEGBERTS;DOUWE-EGBERTS', + 'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', + 'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', + 'SPIN SPAN;SPIN-SPAN', + 'SPINSPAN;SPIN-SPAN', + 'CRETA-FARMS;CRETA-FARM', + 'CRETA-FARM;CRETA-FARM', + 'CRETAFARM;CRETA-FARM', + 'Ολες-τις-Χρήσεις;Ολες-τις-Χρήσεις', + 'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη', + 'φρουι ζελε, φρουί-ζελε', + 'DR BECKMANN, DR-BECKMANN' + ]; + replaceSource.forEach( it => { + st = sanitize_GR( + it.toLowerCase() + ).split(';'); + replaces.push({ + src: ' '+ st[0] +' ', // encolse between spaces + trg: ' '+ st[1] +' ' // to separate from before/after words + }); + }); + + /** mark_explicit_links + * + * mark linked words shall be handled as one-(key)word + * also, edit common mistakes with suggested replaces + * + * @param str + * @return + */ + function mark_explicit_links(str) { + str = ' '+ str +' '; + replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); }); + return str.replaceAll(' ', ' ').trim(); + } + + + + + + /** 3. SUPPLEMENTARY FUNCTIONS + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + // TODO: + // exclude some generic non-critial words when proccessing user's query + // ** example code to work with: + // var ignoredKeys_kb = []; // keywords to ignore (in kb-format) + // 'μας με σε για του της των από στο στον &'.split(' ').forEach(w => { ignoredKeys_kb.push(keyboardize(w)); }); + + // pure JS ajax GET request; return data as JSON + // no fancy things like UTF8; if needed use base64 + function ajax_get(url, callback) { + var xmlhttp = new XMLHttpRequest(); + xmlhttp.onreadystatechange = function() { + if (xmlhttp.readyState == 4 && xmlhttp.status == 200) { + try { + var data = JSON.parse(xmlhttp.responseText); + } catch(err) { + if (options.debug) console.log(err.message + " in " + xmlhttp.responseText); + return; + } + callback(data); + } + }; + xmlhttp.open("GET", url, true); + xmlhttp.send(); + } + + + + + // Search Endine's match functions + ////////////////////////////////////////////////////////////////////////////// + + /** check_match + * --- + * check if a searching string -> query (string/latin in kb-format) + * matches an item of the array of synonyms -> chkArr (array of utf-8/strings) + * + * + option to use fyzzy (bigram) match + * + * @return: matched string (utf-8) --or-- false (if not matched) + */ + function check_match( query, chkArr, fuzzy = false ) { + var result = ''; + var found = false; + + if (query == ' ') return chkArr[0]; + + chkArr.forEach( chk => { + if (!found) { + chk_kb = keyboardize(chk); + if ( (chk_kb.indexOf( query ) !== -1) + || (fuzzy && (match.similarity(chk_kb, query, n) > _fuzzyLimit)) ) { + found = true; + result = chk; + } + } + }); + return found ? result : false; + } + + /** is_exact_match + * + * check if a searching string -> query (string/latin in kb-format) + * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) + * + * @param query (string): searching string; string/latin in kb-format + * @param chkArr (array): array of synonyms; (array of utf-8/strings) + * @return (boolean): true|false + */ + function is_exact_match( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (keyboardize(w) == query) found = true }); + return found; + } + + /** match_one + * + * check if at-least-ONE item from an array of query-words -> qArr (array of string/latin in kb-format) + * matches any item of an array of synonyms -> chkArr (array of utf-8/strings) + * + * @return (boolean) true|false + */ + function match_one( qArr, chkArr ) { + found = false; + qArr.forEach( query => { + chkArr.forEach( w => { if (keyboardize(w) == query) found = true }); + }); + return found; + } + + + + + /** 4. ACTUAL DATA LOADING + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + + // TODO: + // control completion of async loads via .then() rather by this custom structrure + // Need to rewrite the folllowing code ............................. from here + // ........................................................................... + // ........................................................................... + + + /** workline + * ------------------------------------------------------------------------- + * custom object/data-structure + * to track status/completion of async svents + * (it does the job using a special set method) + */ + var workline = { + + trackerJL : 0, + set jsonLoaded(x) { + this.trackerJL = x; + + // fire event on certain values + if (x == 3) { + if (options.debug) console.info('suggestion-engine requirements fulfilled'); + _isReady = true; + // code to execute + // ... + } + }, + get jsonLoaded() { return this.trackerJL; } + }; + + + /** LOAD DATA (from endoints) + * --------------------------------------------------------------------------- + */ + + ajax_get( keywordsURL, function(data) { // get _kwlinks + _kwlinks = data; + if (options.debug) console.log('...keywords loaded;'); + workline.jsonLoaded++; + }); + + function load_store_products(storeID) { + ajax_get(options.products_json, function(data) { // get stor's _products + _products = data; + _products.forEach(p => {p.kb = keyboardize(p.w).toLowerCase()} ); + if (options.debug) console.log('...products loaded;'); + workline.jsonLoaded++; + CURRENT_STOREs_CATALOG = storeID; + }); + } + + var STORE = { id: 904 }; + + if (STORE.id != 0) load_store_products(STORE.id); + + // ........................................................................... + // ........................................................................... + // ................................................................ up to here + + + + + /** 5. SUGGESTIONS ENGINE + * ------------------------------------------------------------------------- + *////////////////////////////////////////////////////////////////////////// + + + workline.jsonLoaded++; // notify workline that jQuery is ready!! + + + // suggestions engine ////////////////////////////////////////////////// + // --- + function suggestions_engine(qOrig) { + var results = []; // suggestions to respond + var proList = []; // list of products (for all suggestions) + var commonL = []; // list of common products (for multiple suggestions) + var possibleNext = []; // list of possible next suggestions + + var root, last; + + var space_ended = (qOrig.slice(-1) == ' ') ? true : false; + + // clean and sanitize and mark links onto q(uery) string + var q = mark_explicit_links( sanitize_GR( clean_text(qOrig) ) ); + + // TODO: + // construct direct-linked words + // = do unequivocally replaces + // steps: + // 1. replace accented vowels with non accented ones + // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'` + // --- -- -- - - - + // test: + // oneliner: var str = 'αλφα βητα ΑΛΦΑ Βητα world'; var src = 'αλφα βητα'; var reg = new RegExp(src, "gi"); var replacedOnce = str.toLowerCase().replace(reg, 'α-β'); console.log(replacedOnce); + + var qAr = q.split(' '); // split to words + if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array + + if (qAr.slice(-1) == "") { + qAr.pop(); + } + + if (options.debug) console.log(qAr); + + if (qAr.length == 1) { // suggest 1st word ////////////////////// + + var kbq = kb.keyboardize(q); + + // loop through root-words + // ... to match all possible suggestions; + _kwlinks.forEach( it => { + chk = check_match(kbq, it.w, true); + + if ( chk != false ) { + results.push({ w: chk }); + + // ... also keep possible products in a list + it.c.forEach( wo => { // for every Word-Link-Node + if (proList.length < _maxResults + 2) { // NOTE: about +2 (bellow) + + // concatenate this-suggestion's product sub-list (wo.p) + // to all-suggestions posiible products-list (proList) + proList = proList.concat(wo.p); + // keep unique products in the products-list + proList = proList.filter((item, i, ar) => ar.indexOf(item) === i); + } + }); + } + + }); + } + + if (qAr.length == 2) { // suggest 2nd word ///////////////////////// + root = qAr[0]; + last = qAr[1]; + kbroot = kb.keyboardize(root); + kblast = kb.keyboardize(last); + + // TODO: + // change [_kwlinks.forEach] to for loop + // --- + _kwlinks.forEach( it => { // locate the ... + if (match.exact(kbroot, kb.keyb_array(it.w))) { // exact match of root-word + + it.c.forEach ( wo => { // loop the word-links ... + chk = check_match(kblast, wo.w, true); + if ( chk !== false ) { // if a match is found + results.push({ // keep suggestion + w: root +' '+ chk, + f: wo.f + }); + if (proList.length < _maxResults + 2) { // plus... + proList = proList.concat(wo.p); // keep products-list + proList = proList.filter((item, i, ar) => ar.indexOf(item) === i); + } + } + }); + + } + }); + } + + if (qAr.length > 2) { // suggest N-th word (N>2) /////////////// + root = qAr.shift(); // isolate first item of qAr + last = qAr.pop(); // isolate last item too + // now qAr includes only the items after root and before last; + // so qAr includes all already selected suggestions (but root) + + // prepare/cache kb-formated string for any key we're going to use + var kb_qAr = []; // array of selected suggestions in kb-format + qAr.forEach( w => { kb_qAr.push(keyboardize(w)); }) + kbroot = keyboardize(root); + kblast = keyboardize(last); + + // TODO: + // change [_kwlinks.forEach] to for loop + // --- + _kwlinks.forEach( it => { + if (is_exact_match(kbroot, it.w)) { // locate root-word + + // calculate list of common items/products (commonL) + // for selected suggestions + is1stOcc = true; // 1st occurance flag + + it.c.forEach ( swo => { + if (match_one(kb_qAr, swo.w)) { + // swo is one of the already selected suggestions + // so... update the common-(products)-L(ist) + if (is1stOcc) { + commonL = swo.p; // init list (on 1st occurance) + is1stOcc = false; + } + else { + // caclulate list of common products + // = intersection of (so-far) commonL and swo.p + // commonL = commonL.filter(value => swo.p.includes(value)); + commonL = commonL.filter(function(n) { return swo.p.indexOf(n) !== -1; }); + } + } + else { // if swo is not already selected + // then This is a possible NEXT suggestion + possibleNext.push(swo); + } + }); + + // Now that we have all the possible next suggestions + // we'll match them with the last word of the query + + possibleNext.forEach( poss => { // for tthe possible next suggestions + // if last-word-of-query matches possible word(s) + // and list of word's products has commons with commonL + // then THiS is a Valid-Next-Suggestion + chk = check_match(kblast, poss.w, true); + if (chk !== false) { + // check intersection of commonL and suggestion's product-lists + tempL = commonL.filter(value => poss.p.includes(value)); + if (tempL.length > 0) { + results.push({ + w: root +' '+ qAr.join(' ') +' '+ chk, + f: poss.f + }); + // update proList too + proList = proList.concat(tempL); + } + } + }); + } + + }); + } + + // NOTE: about +2 (vs +1) + // after having calculated next suggestions and a banch of possible products + // the proccedure is going to decide what data will return; + // if pro(ducts)List includes less items than maximum suggestions + // ... this will be the array to return. So + // ... +2 ensures that this list will not become shorter than this limit + + // calculate unique products + // (credit: https://stackoverflow.com/questions/11246758/) + let unique = proList.filter((item, i, ar) => ar.indexOf(item) === i); + + + + if (unique.length < (_maxResults +1)) { // IF list is small ...... + + // serve products instead of suggestions + // ... + results = []; + notOnThisStore = [] + unique.forEach( pr => { + for (i=0 ; i< _products.length-1 ; i++) { // for makes things faster + pi = _products[i]; + if (pi.id == pr) { + results.push(pi); + break; + } + } + }); + + } else { // remove forced link character '-' from suggestions ..... + var dirty_results = results; + results = []; + for (i=0 ; i< dirty_results.length ; i++) { + results.push({ w: dirty_results[i].w.replaceAll('-', ' ') }) + } + } + + return results; + } + + // simple, fast search products by numeric code property + // --- + function search_by_code( num ) { + const str = num.toString(); + var results = []; + var p; + for (i=0 ; i< _products.length-1 ; i++) { + p = _products[i]; + if (results.length > _maxResults) { break; } + if ( (p.bp+'-'+p.bc).indexOf(str) !== -1 ) { results.push(p) } + } + return results; + } + + // suggestions caller (router function) + // arguments: + // * qOrig : original query string + // ** list : callback array structure to host results + // --- + var isuggest = function(qOrig, list) { + var results; + + qSanit = sanitize_GR(qOrig); // sanitize greel accended chars + + // TODO: + // check if all source-lists are ready + // if not you need to wait ... + // via async promishes or synced timouts + + // if query seems to be some king of 'code/id' + if (qSanit.length>2 && qSanit.match(/^[0-9]+$/) != null) { + results = search_by_code(qOrig); + + } else { + // string match procedure with suggestions engine + results = suggestions_engine(qOrig); + + } + list(results); + } + + /** + jQuery(function() { // on document ready code ////////////////////////// + /** + + // UI-dependent code + // uses reference to specific document element (passed via options) + // ----------------------------------------------------------------------- + const searchBox = $(options.search_tag); + searchBox.typeahead( + { + hint: true, + highlight: true, + minLength: 1 + }, + { + limit: _maxResults, + name: 'kwlinks', + displayKey: 'w', + source: isuggest, + templates: { + suggestion: function(data) { + if (data.id) + return '
'+ data.w +'
'; + return '
'+ data.w +'
'; + }, + empty: '
Δεν υπάρχει στο κωδικολόγιο του καταστήματος
' + } + } + ) + .bind("typeahead:selected", function(obj, datum, name) { + if (datum.hasOwnProperty('id')) { // ** selected: PRODUCT + + // $('#js-add-product-to-order').attr('disabled', false); + // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img); + if (options.debug) console.log('selected: ', datum.id, datum.w); + + // TODO: GOTO product page + window.location.href = 'product_list?product=id-' + datum.id; + + // $("#product-quantity").trigger('focus'); + + } + else { // selected: SUGGESTION + + $('.typeahead').typeahead('val','').trigger('blur'); + $('.typeahead').typeahead('val', datum.w +' ') + .trigger("query"); + // give some time to the engine to calculate results + // then fire focus again... + setTimeout(() => { $('.typeahead').trigger('focus'); }, _timeout_ms); + } + + }) + .bind("typeahead:cursorchange", function( event, obj) { + // track cursor-chane to handle special keys after a final product is selected + + // TODO: + // handle exception where obj is undefined; + // this occures... + // when cusror returns from suggestions list back to the search field + + if (typeof obj === 'undefined') cursor_on = { none: true }; + else if (obj.hasOwnProperty('id')) cursor_on = obj; + else cursor_on = { none: true }; + }); + + // TRACK user search attempt /////////////////////////////////////////////// + $('.typeahead').on('keyup', function(e) { + if (_isReady) { + // console.log('on:', cursor_on); + // console.log('key:', e.key); + if ((e.key == ' ') && cursor_on.hasOwnProperty('id')) { + + // product is actually selected + var datum = cursor_on; + + // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img); + $("#product-quantity").trigger('focus'); + e.preventDefault(); + + } else { + if (e.key === "Enter") { + + // if curson is not on some option + if ((typeof curson_on === 'undefined') || (curson_on.none == true)) { + + // do a common search + if (options.debug) console.log('Do a Non-Suggestions search', searchBox.val()); + + // DEPRICATED: var location = encodeURI('/product_list?productSearch=%'+ searchBox.val() +'%'); + + search_results = common_search(searchBox.val()); + + + } else { + // launch a product page + var location = '/product_list?product=id-'+ cursor_on.id; + if (options.debug) console.log('product', cursor_on.id, cursor_on.w); + + console.log('go to product', location) + } + // window.location.href = location; + } + } + } + }); + + }); + */ + + + /* + function common_search(query) { + // clear ; sanitize ; split + var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' '); + + // if last item is empty, remove it + if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop() + + var results = _products; + + // for each key fitler results + qAr.forEach( key => { + results = key_sublist(key, results) + }); + + // echo products (and prepare list to POST) + var list_ = []; + results.forEach( item => { + if (options.debug) console.log(item.id, ':', item.w); + list_.push(item.id) + }) + + let l = list_.join(','); + var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`); + + if ($('.js-categories.selected').length + || $('.js-checkout.selected').length + ) { + url = encodeURI(`api/v1/products?eys_code=${l}&q=${query}`); + console.log('common search : fetch results > history.push', url, query); + fetchSearchProducts(url, query); + window.history.pushState('search', null, '/product_list?search='+query); + } else { + console.log('common search: search query > location = search') + window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`); + } + + // Redirect with POST (template) + // --- -- -- - - - + // var form = $([ + // `
`, + // '', + // '
' + // ].join('')); + // $('body').append(form); + // form.submit(); + + } + */ + + /** return from list only items that include 'key' + */ + function key_sublist(key, list) { + var result = []; + list.forEach( item => { + if (item.kb.includes(key)) { + result.push(item); + } + }); + + return result; + } +} \ No newline at end of file diff --git a/app/routes/dev.js b/app/routes/dev.js new file mode 100644 index 0000000..cd40822 --- /dev/null +++ b/app/routes/dev.js @@ -0,0 +1,103 @@ +/** + * defines routes + * exports router + */ + +const Router = require('koa-router'); +const urler = require('../utils/url-util.js'); + +const data = require('../pieces/prepare.js'); + +const bench = require('../benchmark/find.js'); +const matchStr = require('../benchmark/match-str.js'); + + +// Prefix all routes with: /items +const router = new Router(); + + + +// Routes + +/* simple route example + + let items = [ + { id: 100, iname: 'Quartz Analog Wrist Watch', price: 'US $4.99'}, + { id: 101, iname: 'Leather Peep Pump Heels', price: 'US $33.56'}, + { id: 102, iname: 'Apple iPod', price: 'US $219.99'}, + { id: 103, iname: 'Prince Phantom 97P Tennnis Racket', price: 'US $50.00'}, + ]; + + router.get('/items', (ctx, next) => { + ctx.body = items; + next(); + }); + + router.get('/items/:id', (ctx, next) => { + let getCurrentItem = items.filter(function(item) { + if (item.id == ctx.params.id) { + return true; + } + }); + if (getCurrentItem.length) { + ctx.body = getCurrentItem[0]; + } else { + ctx.response.status = 404; + ctx.body = 'Item Not Found'; + } + next(); + }); +*/ + + +// NOTE: set timeout per route (how-to) +// https://stackoverflow.com/questions/66634123/how-to-add-individual-timeout-value-per-specific-route-using-node-js + + +// TEST routes +//////////////////////////////////////////////////////////////////////////////// + +router.get('/test', (ctx) => { // easy test route + // test anything ... + ctx.body = { params: urler.struct(ctx.request, ctx.url), ctx: ctx } +}); + +router.get('/test/do-data', (ctx) => { // easy test route + // test anything ... + // data.create( + // 'products', + // 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' + // ); + data.load_products( + 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' + ); + ctx.body = { success: true, operation: 'create new data' }; +}); + +// benchmark routes; +// shall be removed from production +//////////////////////////////////////////////////////////////////////////////// + +/// router.get('/test/json', (ctx, next) => { // easy test route +/// // test anything ... +/// ctx.body = { t: + new Date(), p: products }; +/// next(); +/// }); + +router.get('/bench/find', (ctx) => { // easy test route + // test anything ... + ctx.body = bench.compare(); + // next(); +}); + +router.get('/bench/match/:title', (ctx) => { // easy test route + // test anything ... [ query = 'solokata' ] + // ctx.body = bench.compare(); + ctx.body = matchStr.run(ctx.params.title); + // next(); +}); + + +// export routes + +module.exports = router; diff --git a/app/routes/index.js b/app/routes/index.js new file mode 100644 index 0000000..e3d6230 --- /dev/null +++ b/app/routes/index.js @@ -0,0 +1,26 @@ +/** + * routes (index) + */ +const Router = require('koa-router'); + +const router = new Router(); + +// Require grouped routes + +let v1 = require('./v1.js'); +router.use(v1.routes()); + +let dev_paths = require('./dev.js'); +router.use(dev_paths.routes()); + +// define default route +router.get('/', (ctx) => { + ctx.body = { + success: true, + title: 'oseine', + description: 'oseine search engine is not elastic', + message: 'where are you now?' + } +}) + +module.exports = router; diff --git a/app/routes/v1.js b/app/routes/v1.js new file mode 100644 index 0000000..394dde6 --- /dev/null +++ b/app/routes/v1.js @@ -0,0 +1,68 @@ +/** + * api v1 routes + */ + +const Router = require('koa-router'); +const prepare = require('../pieces/prepare.js'); + + +// Prefix all routes with: /items +const router = new Router({ + prefix: '/v1' +}); + + + +// Routes + +router.get('/prepare/:title', (ctx) => { + let result = false; + switch (ctx.params.title) { + case 'products': + result = prepare.load_products( + 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' + ); + break; + + default: + break; + + } + console.log(result); + ctx.body = { success: result }; +}) + + +router.get('/search', (ctx) => { + ctx.body = []; +}); + +router.get('/search/:title', (ctx) => { + let result = [ + { id: 10, w: 'Ένα Προϊόν' }, + { id: 15, w: 'Άλλο Προϊόν' }, + { id: 20, w: 'Προϊόν 3' }, + { id: 25, w: 'Προϊόν 4' } + ] + // console.log(ctx); + /// const kb = require('./utils/kb-util.js'); + /// let words = kb.keyboardize(kb.clean(ctx.params.title)).split(' '); + /// ctx.body = { + /// params: ctx.params, + /// results: words, + /// nxt: next + /// }; + ctx.body = result; +}); + + +/** TODO: ?? other routes ?? + * ///////////////////////////////////////////////////////////////////////////// + * + /suggest/ + * + /prepare (products, keywords, linked-terms etc) + * + /stats + */ + + +// export routes +module.exports = router; diff --git a/app/utils/kb-util.js b/app/utils/kb-util.js new file mode 100644 index 0000000..aac88f9 --- /dev/null +++ b/app/utils/kb-util.js @@ -0,0 +1,88 @@ +/** + * fast string manipulation utilities + * for bi-lingual (EL/EN) words/phrases + * based on the keyboard layout + */ + +// suplamentary arrays (mostly for cache) +// --- -- -- - - - + +var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + +var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + +var map = new Map(); +for (var i=0; i { + ap = pair.split(' '); + accented_vowels.push({ + a: ap[0], // accented + p: ap[1] // pure = non accended + }); +}); + + +// translates string to keyboard-latin keys +// (the ones that used whan typing each letter of the word) +const keyboardize = (str) => { + str = str.replace('\'',''); + var out = ''; + // [map]'s implementation is 40x faster than [for]'s + for (var i=0 ; i< str.length; i++) out += map.get(str[i]); + return out; +} + +// keyboardize an array of strings +const keyb_array = (arr) => { + kb_arr = []; + arr.forEach( w => { + kb_arr.push(keyboardize(w)); + }); + return kb_arr; +} + + +// transforms to lowercase; handles sigma-teliko +const sanitizeGR = (str) => { + str = str.toLowerCase(); + + // replace accended vowels with pure ones + accented_vowels.forEach( v => { + str = str.replaceAll(v.a, v.p); + }); + + // replace sigma on the end of words + str = str + ' '; + str = str.replaceAll('σ-', 'ς-'); + str = str.replaceAll('σ ', 'ς '); + + return str; +} + +// removes non keyword characters [+ . , !] and internal multiple-spaces +// @param txt (string): product description +const clean = (txt) => { + return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space + .replace('!','').replace('\"', '') // remove character + .replace(' ',' ').replace(' ',' '); // remove multiple spaces +} + + +// exports +// --- -- -- - - - + +module.exports = { + keyboardize, + keyb_array, + sanitizeGR, + clean +}; \ No newline at end of file diff --git a/app/utils/match-util.js b/app/utils/match-util.js new file mode 100644 index 0000000..5e8c648 --- /dev/null +++ b/app/utils/match-util.js @@ -0,0 +1,154 @@ +/** + * match utility; + * includes fuzzy and partial match functions too; + * many of them return a match-rate + */ + + +// fuzzy match +//////////////////////////////////////////////////////////////////////////////// + +/** Ngram fuzzy match algorithm + * (simple and fast) + */ +const createNgram = (word, n) => { // Ngram creation + if (word.length <3) return word; + const vector = []; + for (let i = 0; i < word.length-n+1; ++i) { + vector.push(word.slice(i, i + n)); + } + return vector; +}; + +/** similarity + * rates similarity between 2 words + * based on Ngram matches of N = n letters; + * implements a 2-dim check (all a-Ngrams vs all all b-Ngrams) + * + * @param {string} a : first word + * @param {string} b : second word + * @param {int} n : Ngram base + * @returns {float} : match percentage as a float in [0, 1] + */ +const similarity = (a, b, n) => { // Ngram match score + if (a.length > 0 && b.length > 0) { + const aNgram = createNgram(a, n); + const bNgram = createNgram(b, n); + let hits = 0; + for (let x = 0; x < aNgram.length; ++x) { + for (let y = 0; y < bNgram.length; ++y) { + if (aNgram[x] === bNgram[y]) { + hits += 1; + } + } + } + if (hits > 0) { + const union = aNgram.length + bNgram.length; + return (2.0 * hits) / union; + } + } + return 0; +}; + +/** resemblance + * is an alternative similarity rating; + * implements an 1-dim Ngram similarity check + * and it's much faster than similarity() + */ +const resemblance = (a, b, n) => { + if (a.length > n && b.length >= a.length) { + const aNgram = createNgram(a, n); + let hits = 0; + for (let i = 0; i < aNgram.length; ++i) { + if (b.includes(aNgram[i])) { + hits++; + } + } + if (hits > 0) { + // rate resemblance based on hits and length-similarity + return (hits / aNgram.length) * (a.length / b.length); + } + } + return 0; +} + + +// exact and partial match +//////////////////////////////////////////////////////////////////////////////// + +/** is_exact_match + * check if a searching string -> query (string/latin in kb-format) + * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) + * + * @param {string} query: searching string; string/latin in kb-format + * @param {array} chkArr: array of synonyms; (array of utf-8/strings) + * @return {boolean}: true|false + */ +function exact( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (w == query) found = true }); + return found; +} + +function partial( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (w.includes(query)) found = true }); + return found; +} + +/** is exact match + weight rating + * @returns {float} weight rates depth of array when a match is found + */ +function weighted_exact( query, chkArr ) { + let weight = 0; // closer to left/begin rating + let len = chkArr.length; + for(let i = 0; i < len ; i++) { // i ~ depth + if (chkArr[i] == query) { + // weights array depth + weight = (len - i + 1.0) / len; + break; + } + } + return weight; +} + + +/** is partial match + weight rating + * + * @param query (string): searching string; string/latin in kb-format + * @param chkArr (array): array of synonyms; (array of utf-8/strings) + * @returns {float} weight rates both match position and depth of match + * + * (*) optimization NOTE: + * Given the weight `W` and the depth `i`, + * the best weight for next `i` shall be: `(L - (i+1)) / L` + * To be imposibbe to have a better weight, should: + * W > (L - (i+1)) / L => ... => i > (L - L*W - 1) + */ +function weighted_partial( query, chkArr ) { + let rate = 0; + let weight = 0; + let len = chkArr.length; + for( let i = 0 ; i < len ; i++ ) { + let chk = chkArr[i].indexOf(query) + if (chk != -1) { + rate = (len - i) / (len + 2.0 * chk); + weight = rate > weight ? rate : weight; + } + if (i > (len - len * weight - 1)) { + break; // better rating is not possible (*) + } + } + return weight; +} + + +// exports +module.exports = { + exact, + partial, + weighted_exact, + weighted_partial, + similarity, + resemblance +} diff --git a/app/utils/mem-usage.js b/app/utils/mem-usage.js new file mode 100644 index 0000000..766a93e --- /dev/null +++ b/app/utils/mem-usage.js @@ -0,0 +1,20 @@ +/** + * memory usage report utility + */ + +const formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`; + +function report() { + let memoryData = process.memoryUsage(); + + let memoryUsage = { + rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`, + heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`, + heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`, + external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`, + }; + + console.log(memoryUsage); +} + +module.exports = { report } diff --git a/app/utils/url-util.js b/app/utils/url-util.js new file mode 100644 index 0000000..e86ad9b --- /dev/null +++ b/app/utils/url-util.js @@ -0,0 +1,23 @@ +/** + * url utility + */ + +const querystring = require('querystring'); + +function struct(req, url) { + let url_parts = url.split('?'); + let query = (url_parts.length > 1) + ? querystring.decode(url_parts[1]) + : {}; + return { + method: req.method, + host: req.host, + path: url_parts[0], + query: query + } +} + +/** + * exports + */ +module.exports = { struct } \ No newline at end of file diff --git a/benchmark/find.js b/benchmark/find.js deleted file mode 100644 index 2d18657..0000000 --- a/benchmark/find.js +++ /dev/null @@ -1,105 +0,0 @@ -/** - * benchmark: find a product in product list - * using: for vs forEach vs find - */ - -const microtime = require('microtime'); - -const products = require('../data/products.json'); - - -var selected = []; -products.forEach( pr => { - if (Math.floor(Math.random() * 100) > 85) { - selected.push(pr.id); - } -}); - - -function compare() { - - let n = 4; - - var f0 = microtime.nowDouble(); - for(i=0 ; i < n ; i++) byFor(); - var f1 = microtime.nowDouble(); - // - var e0 = microtime.nowDouble(); - for(i=0 ; i < n ; i++) byEach(); - var e1 = microtime.nowDouble(); - // - var b0 = microtime.nowDouble(); - for(i=0 ; i < n ; i++) byFind(); - var b1 = microtime.nowDouble(); - - return { - n: n, - - for: f1-f0, - items_for: byFor(), - - each: e1-e0, - items_each: byEach(), - - find: b1-b0, - items_find: byFind(), - - sel: selected, - } -} - - -function byFor() { - var items = []; - var notFound = []; - var found; - selected.forEach( id => { - found = false; - for(i = 0; i < products.length; i++) { - if (products[i].id == id) { - items.push(products[i]); - found = true; - break; - } - } - if (!found) notFound.push(id) - - }) - return {items: items, nf: notFound}; -} - - -function byEach() { - var items = []; - var notFound = []; - var found; - selected.forEach( id => { - found = false; - products.forEach( pr => { - if (pr.id == id) { - items.push(pr); - found = true; - } - }); - if (!found) notFound.push(id) - }); - return {items: items, nf: notFound}; -} - -function byFind() { - var items = []; - var notFound = []; - var result; - selected.forEach( id => { - found = false; - result = products.find((pr) => pr.id == id); - if (result === undefined) notFound.push(id) - else items.push(result); - }); - return {items: items, nf: notFound}; -} - -module.exports = { - compare, - byFor -} \ No newline at end of file diff --git a/benchmark/match-str.js b/benchmark/match-str.js deleted file mode 100644 index 2114f41..0000000 --- a/benchmark/match-str.js +++ /dev/null @@ -1,74 +0,0 @@ -var microtime = require('microtime'); - -const match = require('../utils/match-util.js'); -const memory_usage = require('../utils/mem-usage.js'); -const kb = require('../utils/kb-util.js'); - -const products = require('../data/products.json'); - -// memory_usage.report(); - -// test runner -function run(query) { - - var f0 = microtime.nowDouble(); - let result; - // for(let i = 0 ; i < 20 ; i++) - result = matchQuery(query); - var f1 = microtime.nowDouble(); - - memory_usage.report(); - - return { - t: f1-f0, - q: query, - result: result - } -} - -// sample search implementation -function matchQuery(query) { - var result = []; - - let q = kb.keyboardize(query); - - for(let i = 0; i < products.length; i++) { - - let rate = 0; - let found = false; - let x; - let src = products[i].kb.split(' '); // (array) source of product (key-)words - if ((x = match.weighted_exact(q, src)) > 0) { - // console.log(products[i].kb, x); - rate = 5.0 + x; - found = true; - } - else if ((x = match.weighted_partial(q, src)) > 0) { - rate = 3.0 + x; - found = true; - } - else { - let similarity = 0; - src.forEach( w => { - let sim = match.resemblance(q, w, 2); - if (sim > 0.6) { - found = true; - similarity = (sim > similarity) ? sim : similarity; - } - }); - if (found) { - rate = 2 * similarity; - } - } - - if (found) { - products[i].rate = rate; - result.push(products[i]); - } - } - - return result.sort((a,b) => b.rate - a.rate).slice(0, 48);; -} - - -module.exports = { run } diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/products.null b/data/products.null new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/data/products.null @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..25c3319 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +services: + app: + image: node:alpine + user: "node" + working_dir: /home/node/app + environment: + - NODE_ENV=production + ports: + - "3000:3000" + volumes: + - ./app:/home/node/app + - ./data:/home/node/data + - ./node_modules:/home/node/node_modules + expose: + - "3000" + command: "node app.js" \ No newline at end of file diff --git a/package.json b/package.json index 2ffe852..84a5010 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "oseine search engine is not elastic", "main": "index.js", "scripts": { - "start": "nodemon app.js", + "start": "nodemon ./app/app.js", "test": "echo \"Error: no test specified\" && exit 1" }, "author": "Geo-Xalkiadakis@Sklavenitis-SA", diff --git a/pieces/prepare.js b/pieces/prepare.js deleted file mode 100644 index 65ccee1..0000000 --- a/pieces/prepare.js +++ /dev/null @@ -1,128 +0,0 @@ -const fs = require('fs'); -var request = require('request'); - -const kb = require('../utils/kb-util.js'); - - -// const https = require("https"); - - -/* TODO: ?? parallel read - // var request = require('request-promise'); - var calls = [ - request({ - url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', - // headers: { ... } - }), - request({ - url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', - // headers: { ... } - }), - // + linked-terms and replaces - // and more.. - ]; - - Promise.all(calls).then(function(results) { - // do something with results[0] - // do something with results[1] - // ... - }); -*/ - - -/** create entity file from url - * used for entities like 'linked-words', 'synonyms', etc. - * - * @param {string} entity: use the entity name - * @param {string} url: prepared entity json-file in remote server - */ -function create(entity, url) { - request(url, - function (error, response, body) { - if (!error && response.statusCode == 200) { - // body is a ready json-string; no need to parse and (re-)stringify - try { - fs.writeFileSync(`${__dirname}/../data/${entity}.json`, body, 'utf-8'); - // file written successfully - } catch (err) { - console.error(err); - } - } - } - ); -} - -function get_data_structure() { - // data structure is an array of `{ url:.., expiration:.., path:.. }` objects -} - -function set_data_structure(node) { - -} - - -function load_products(url) { - request(url, - function (error, response, body) { - if (!error && response.statusCode == 200) { - // body is a ready json-string; no need to parse and (re-)stringify - json = JSON.parse(body); - newJson = []; - - // TODO: - // + attach handle synonyms - // + mark brand-names - // + attach category-names and SAP-categories - // + construct combo words - // + remove non-important words - // + normalize popularity - - json.forEach( p => { - newJson.push({ - id: p.id, - w: p.w, - kb: kb.keyboardize(kb.clean(p.w)) - }); - }); - - try { - - fs.writeFileSync( - `${__dirname}/../data/products.json`, - JSON.stringify(newJson), - 'utf-8' - ); - return true; // file written successfully - - } catch (err) { - console.error(err); - return false; - } - } - } - ); -} - - -// create data -// testing an implemenatatin of a custom local-fs-cache system -function random() { - var data = []; - for( let i = 0; i < (Math.floor(Math.random() * 100) +15) ; i++) { - data.push({ id: i, x5: i*5 }); - } - - try { - fs.writeFileSync(__dirname + '/../data/random.json', JSON.stringify(data), 'utf-8'); - // file written successfully - } catch (err) { - console.error(err); - } -} - - -module.exports = { - create, - random, - load_products -} \ No newline at end of file diff --git a/pieces/retro-search.js b/pieces/retro-search.js deleted file mode 100644 index ba4d937..0000000 --- a/pieces/retro-search.js +++ /dev/null @@ -1,304 +0,0 @@ -const kb = require('../utils/kb-util.js'); -const match = require('../utils/match-util.js'); - -const products = require('../data/products.json'); - -/** VARIABLES - * may passed as module arguments - * ----------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////////// - - -var _allowFuzzy = true; // enable|disable fuzzy search -var n = 2; // Ngram base -var _fuzzyLimit = .5; // minimum bigram score for being considered a match -var max_list = 24; -var tolerance = 42; -var STORE = { id: 904 }; - -var _kwlinks; // keyword links (word-connections; imported via ajax-get) -var _products = []; // all products (imported via ajax-get) - -// setup options -var _maxResults = options.max_list; // limit suggestions -var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions -var _Ngram_base = 2; // number of N in Ngram spliting algorithm -var _isReady = false; // whether the searchbox is ready to be used - -// product keywords -var keywordsURL = options.keywords_json; - -var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } - - - - - - -/** SUPPLEMENTARY FUNCTIONS - * ----------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////////// - - - -// callback function for sorting resulrs per r (=rating) property -function compare_rate(a,b) { - return (a.r < b.r); -} - - - -/** SEARCH ENGINE - * ------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////// - - - - -function matchWordInList(q, list = false) { - - if (list !== false && list.length == 0) return []; // no results - - var result = []; - var firstPass = false; - - if (list === false) { - firstPass = true; // on first pass - list = products; // list is all products - } - - for(let i = 0 ; i < list.length ; i++) { - - // check exact + rate - - // else check partial + rate - - // else check similarity + rate - - let found = false; - let similarity = 0; - products[i].kb.split(' ').forEach( w => { - let sim = match.resemblance(q, w, 2); - if (sim > 0.5) { - found = true; - similarity = (sim > similarity) ? sim : similarity; - } - }); - if (found) { - products[i].similarity = similarity; - products[i].rating = (firstPass) - ? similarity * 2.0 - : products[i].rating + similarity * 2.0 - result.push(products[i]); - } - } - - return result -} - - - -// suggestions engine ////////////////////////////////////////////////// -// --- -function suggestions_engine(query) { - var results = []; - var pot = []; pot.length = 0; - - // clean and sanitize and mark links onto q(uery) string - var q = kb.keyboardize( kb.sanitizeGR( kb.clean(query.trim()) ) ).trim(); - - // TODO: - // construct direct-linked words - // = do unequivocally replaces - // steps: - // 1. replace accented vowels with non accented ones - // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'` - - var qAr = q.split(' '); // split to words - /// if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array - /// - /// if (qAr.slice(-1) == "") { - /// qAr.pop(); - /// } - - - pot = _products; // potential results // NOTE: CRITICAL: BY REFERENCE - - var wi = 0; // word index (from list) - var wc = qAr.length; - - qAr.forEach( w => { - - let sf = []; // (matches) so far - let mi; // position of match - wi++; - - pot.forEach( it => { - let matched = false; - let tester = ' '+ it.kb + ' '; - - // reset previous history and ratings - if (wi == 1) { - it.r = 0; - it.history = []; - } - - - // rate word-match > start-match > simple-match - // ... up to 8 points - - if (tester.indexOf(' '+ w +' ') != -1) { - it.r += 9; - it.history.push({ w: w, rate: 9 }); - matched = true; - } - else if (tester.indexOf(' '+ w) != -1) { - it.r += 5; - it.history.push({ w: w, rate: 5 }); - matched = true; - } - else if (tester.indexOf(w) != -1) { - it.r += 2; - it.history.push({ w: w, rate: 2 }); - matched = true; - } - - // rate `near-to-start` matching .. up to 7p - // rate `earlyness` of word in query .. up to 7p - - if ((mi = tester.indexOf(' '+w)) != -1) { - let fc1 = 100 - ((mi < 99) ? mi : 99); // near-to-start factor - let fc2 = wc - wi + 1; // query earlyness factor - let r1 = Math.floor(7*fc1/100); - let r2 = Math.floor(7*fc2/wc); - - it.r += (r1 + r2); - it.history.push({ w: w, left: [fc1, r1], early: [fc2, r2] }); - } - if (matched) sf.push(it); - }); - - if ((sf.length > (_maxResults + Math.floor(_maxResults/2))) - || (wi == 1) ) { - // ..if pot has a fair amount (= max + 50%) of results - // ..or these are results of '1st-query-word' - // set sf as new source - pot.lenght = 0; pot = []; - pot = JSON.parse(JSON.stringify(sf)); // copy by value - - } else { - // else.. keep the source list and increase of 'so-far rating' - // console.log('found small list', sf, pot) - pot.forEach( it => { - sf.forEach( si => { - if (it.id == si.id) { - it.r += 10; - it.history.push({ w: w, plus: '+10'}); - } - }); - }); - } - }); - - // sort results, get max-list of best rated - results = (pot.length > _maxResults) - ? pot.sort(compare_rate).slice(0, _maxResults) - : pot.sort(compare_rate) - - if (options.debug) console.log(results); - - return results; -} - - -// sub-module (start) -//////////////////////////////////////////////////////////////////////////// - -function update_common_search_results(q, results) { - let queries = getSessionObj('sr'); - let newSRlist = []; - let isnewQ = true; - if (queries === null) { - setSessionObj('sr', [{ - q: q, - result: result, - t: + new Date() - }]); - return true; - - } else { - - queries.forEach(it => { - if (it.q == q) { - newSRlist.push({ - q:q, - result: result, - t: + new Date() - }); - isnewQ = false; - } else { newSRlist.push(it); } - }); - - if (isnewQ) { - newSRlist.push({ - q:q, - result: result, - t: + new Date() - }); - } - - return true; - } -} - - - -//////////////////////////////////////////////////////////////////////////// -// sub-module (end) - - -function common_search(query) { - // clear ; sanitize ; split - var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' '); - - // if last item is empty, remove it - if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop() - - var results = _products; - - // for each key fitler results - qAr.forEach( key => { - results = key_sublist(key, results) - }); - - // echo products (and prepare list to POST) - var list_ = []; - results.forEach( item => { - if (options.debug) console.log(item.id, ':', item.w); - list_.push(item.id) - }) - - // *** TODO: keep results in local storage (or on session storage) - - // update_common_search_results(query, list_); - - let l = list_.join(','); - var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`); - - console.log('common search: search query > location = search') - window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`); - -} - -/** return from list only items that include 'key' - */ -function key_sublist(key, list) { - var result = []; - list.forEach( item => { - if (item.kb.includes(key)) { - result.push(item); - } - }); - - return result; -} diff --git a/pieces/suggest.js b/pieces/suggest.js deleted file mode 100644 index c0a6e04..0000000 --- a/pieces/suggest.js +++ /dev/null @@ -1,786 +0,0 @@ -const kb = require('../utils/kb-util.js'); -const match = require('../utils/match-util.js'); - -var _allowFuzzy = true; // enable|disable fuzzy search -var n = 2; // Ngram base -var _fuzzyLimit = .5; // minimum bigram score for being considered a match - - -/** (Search) SUGGESTIONS ENGINE - * --------------------------------------------------------------------------- - * - * Operates in dual mode; - * -- suggestions engine (interactive) - * -- classic-like mode (passive) - * - * TODO: - * CRITICAL: (optimization) - * Search initialization uses quite a lot of network sources; - * thus it should be started in a later time; - * lets say ... after `x` seconds - * or... when document/core-ui-elements are ready - * - * @parametres (json) : options - * --- - * @var keywords_json (sring) : endpoint url of linked keywords structure - * @var products_json (string) : endpoint od product descriptions - * @var search_tag (string) : selector of field that shall act as typeahead-suggestions - * @var visualize_search_results_url (str) : url that will visualize the sended "results-page" - * @var debug (bool) : if true sends several debug console messages; if false mesagges are eliminated - */ - -suggestions_module({ - keywords_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', - products_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', - search_tag: '#tagsInput', - visualize_search_results_url: '/product_list', - debug: ((location.hostname == 'localhost') || (location.hostname == '127.0.0.1')) -}); - -function suggestions_module(options) { - - /** CONTENTS - * - * +1: Variables - * - * +2: Purify string functions - * + keyboardize - * + sanitize_GR - * + clean - * + mark_explicit_links - * - * +3: Supplementary function (vanilla js) - * + ajax_get(url, callback) - * + createNgram (fuzzy) - * + checkSimilarity (fuzzy) - * + check_match - * + is_exact_match - * + match_one - * - * +4: Actual data loading (async) - * - * +5: Suggestions Engine (jQuery) - * + suggestions_engine - * - */ - - console.log('executing suggestions...'); - - - /** 1. VARIABLES - * ------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////// - - var _kwlinks; // keyword links (word-connections; imported via ajax-get) - var _products = []; // all products (imported via ajax-get) - - // setup options - var _maxResults = 24; // limit suggestions - var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions - var _timeout_ms = 100; // time (in ms) for the search engine to find matches (before rendering) - - var _Ngram_base = 2; // number of N in Ngram spliting algorithm - var _isReady = false; // whether the searchbox is ready to be used - - // product keywords - var keywordsURL = options.keywords_json; - - var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } - - - /** replaces (correcting descriptions) - * == construct unequivocally liked words ////////////////////////////////// - * ------------------------------------------------------------------------- - * - * NOTE: TODO: - * in future implementations multi-word keywords - * may use the non-breaking space as conecting character (\u00A0) instead of dush (-) - * (or maybe both of them) - * - * also TODO: - * in future implementaions linked words may passed via some endpoint - */ - replaces = []; - replaceSource = [ - '3Α;3-ΑΛΦΑ', - '3 ΑΛΦΑ;3-ΑΛΦΑ', - 'HEAD & SHOULDERS;HEAD&SHOULDERS', - 'HEAD N SHOULDERS;HEAD&SHOULDERS', - 'W.K Kellogg; W-K-Kellogg', - 'W.K Kellogg;', - '7 DAYS;7-DAYS', - '7 UP;7UP', - '7-UP;7UP', - 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', - 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ', - 'COCA COLA;COCA-COLA', - 'J P. CHENET; J.P.CHENET', - 'J.P. CHENET; J.P.CHENET', - 'COCACOLA;COCA-COLA', - 'NES CAFE;NESCAFE', - 'NES-CAFE;NESCAFE', - 'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS', - 'PETIT MARSEILLAIS;PETIT-MARSEILLAIS', - 'Το Μάννα;Το-Μάννα', - 'Χωρίς Γλουτένη;Χωρίς-Γλουτένη', - 'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη', - 'Χωρίς Αλάτι;Χωρίς-Αλάτι', - 'Χωρίς Λακτόζη;Χωρίς-Λακτόζη', - 'Χωρίς Συντηρητικά;Χωρίς-Συντηρητικά', - 'Χωρίς Αλκοόλ;Χωρίς-Αλκοόλ', - 'Χωρίς Kαφεϊνη;Χωρίς-Kαφεϊνη', - 'Χωρίς Kαφεΐνη;Χωρίς-Kαφεϊνη', - 'Χωρίς Γλυκάνισο;Χωρίς-Γλυκάνισο', - 'Χωρίς Ανθρακικό;Χωρίς-Ανθρακικό', - 'Υψηλής Παστερίωσης;Υψηλής-Παστερίωσης', - 'Ολικής Άλεσης;Ολικής-Άλεσης', - 'Ολικής Aλέσεως;Ολικής-Aλέσεως', - 'Χαρτί Υγείας;Χαρτί-Υγείας', - 'ρολό υγείας;ρολό-υγείας', - 'χαρτί τουαλέτας;χαρτί-τουαλέτας', - 'Χαρτί Κουζίνας;Χαρτί-Κουζίνας', - 'ρολό κουζίνας;ρολό-κουζίνας', - 'Μπάρες Δημητριακών;Μπάρες-Δημητριακών', - 'Ας Μαγειρέψουμε;Ας-Μαγειρέψουμε', - 'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', - 'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ', - 'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ', - 'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ', - 'ΕΛ ΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', - 'ΕΛΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ', - 'FREE STEP;FREE-STEP', - 'EL SABOR;EL-SABOR', - 'ELSABOR;EL-SABOR', - 'DOUWE EGBERTS;DOUWE-EGBERTS', - 'DOUWEEGBERTS;DOUWE-EGBERTS', - 'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', - 'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ', - 'SPIN SPAN;SPIN-SPAN', - 'SPINSPAN;SPIN-SPAN', - 'CRETA-FARMS;CRETA-FARM', - 'CRETA-FARM;CRETA-FARM', - 'CRETAFARM;CRETA-FARM', - 'Ολες-τις-Χρήσεις;Ολες-τις-Χρήσεις', - 'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη', - 'φρουι ζελε, φρουί-ζελε', - 'DR BECKMANN, DR-BECKMANN' - ]; - replaceSource.forEach( it => { - st = sanitize_GR( - it.toLowerCase() - ).split(';'); - replaces.push({ - src: ' '+ st[0] +' ', // encolse between spaces - trg: ' '+ st[1] +' ' // to separate from before/after words - }); - }); - - /** mark_explicit_links - * - * mark linked words shall be handled as one-(key)word - * also, edit common mistakes with suggested replaces - * - * @param str - * @return - */ - function mark_explicit_links(str) { - str = ' '+ str +' '; - replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); }); - return str.replaceAll(' ', ' ').trim(); - } - - - - - - /** 3. SUPPLEMENTARY FUNCTIONS - * ------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////// - - - // TODO: - // exclude some generic non-critial words when proccessing user's query - // ** example code to work with: - // var ignoredKeys_kb = []; // keywords to ignore (in kb-format) - // 'μας με σε για του της των από στο στον &'.split(' ').forEach(w => { ignoredKeys_kb.push(keyboardize(w)); }); - - // pure JS ajax GET request; return data as JSON - // no fancy things like UTF8; if needed use base64 - function ajax_get(url, callback) { - var xmlhttp = new XMLHttpRequest(); - xmlhttp.onreadystatechange = function() { - if (xmlhttp.readyState == 4 && xmlhttp.status == 200) { - try { - var data = JSON.parse(xmlhttp.responseText); - } catch(err) { - if (options.debug) console.log(err.message + " in " + xmlhttp.responseText); - return; - } - callback(data); - } - }; - xmlhttp.open("GET", url, true); - xmlhttp.send(); - } - - - - - // Search Endine's match functions - ////////////////////////////////////////////////////////////////////////////// - - /** check_match - * --- - * check if a searching string -> query (string/latin in kb-format) - * matches an item of the array of synonyms -> chkArr (array of utf-8/strings) - * - * + option to use fyzzy (bigram) match - * - * @return: matched string (utf-8) --or-- false (if not matched) - */ - function check_match( query, chkArr, fuzzy = false ) { - var result = ''; - var found = false; - - if (query == ' ') return chkArr[0]; - - chkArr.forEach( chk => { - if (!found) { - chk_kb = keyboardize(chk); - if ( (chk_kb.indexOf( query ) !== -1) - || (fuzzy && (match.similarity(chk_kb, query, n) > _fuzzyLimit)) ) { - found = true; - result = chk; - } - } - }); - return found ? result : false; - } - - /** is_exact_match - * - * check if a searching string -> query (string/latin in kb-format) - * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) - * - * @param query (string): searching string; string/latin in kb-format - * @param chkArr (array): array of synonyms; (array of utf-8/strings) - * @return (boolean): true|false - */ - function is_exact_match( query, chkArr ) { - found = false; - chkArr.forEach( w => { if (keyboardize(w) == query) found = true }); - return found; - } - - /** match_one - * - * check if at-least-ONE item from an array of query-words -> qArr (array of string/latin in kb-format) - * matches any item of an array of synonyms -> chkArr (array of utf-8/strings) - * - * @return (boolean) true|false - */ - function match_one( qArr, chkArr ) { - found = false; - qArr.forEach( query => { - chkArr.forEach( w => { if (keyboardize(w) == query) found = true }); - }); - return found; - } - - - - - /** 4. ACTUAL DATA LOADING - * ------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////// - - - - // TODO: - // control completion of async loads via .then() rather by this custom structrure - // Need to rewrite the folllowing code ............................. from here - // ........................................................................... - // ........................................................................... - - - /** workline - * ------------------------------------------------------------------------- - * custom object/data-structure - * to track status/completion of async svents - * (it does the job using a special set method) - */ - var workline = { - - trackerJL : 0, - set jsonLoaded(x) { - this.trackerJL = x; - - // fire event on certain values - if (x == 3) { - if (options.debug) console.info('suggestion-engine requirements fulfilled'); - _isReady = true; - // code to execute - // ... - } - }, - get jsonLoaded() { return this.trackerJL; } - }; - - - /** LOAD DATA (from endoints) - * --------------------------------------------------------------------------- - */ - - ajax_get( keywordsURL, function(data) { // get _kwlinks - _kwlinks = data; - if (options.debug) console.log('...keywords loaded;'); - workline.jsonLoaded++; - }); - - function load_store_products(storeID) { - ajax_get(options.products_json, function(data) { // get stor's _products - _products = data; - _products.forEach(p => {p.kb = keyboardize(p.w).toLowerCase()} ); - if (options.debug) console.log('...products loaded;'); - workline.jsonLoaded++; - CURRENT_STOREs_CATALOG = storeID; - }); - } - - var STORE = { id: 904 }; - - if (STORE.id != 0) load_store_products(STORE.id); - - // ........................................................................... - // ........................................................................... - // ................................................................ up to here - - - - - /** 5. SUGGESTIONS ENGINE - * ------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////// - - - workline.jsonLoaded++; // notify workline that jQuery is ready!! - - - // suggestions engine ////////////////////////////////////////////////// - // --- - function suggestions_engine(qOrig) { - var results = []; // suggestions to respond - var proList = []; // list of products (for all suggestions) - var commonL = []; // list of common products (for multiple suggestions) - var possibleNext = []; // list of possible next suggestions - - var root, last; - - var space_ended = (qOrig.slice(-1) == ' ') ? true : false; - - // clean and sanitize and mark links onto q(uery) string - var q = mark_explicit_links( sanitize_GR( clean_text(qOrig) ) ); - - // TODO: - // construct direct-linked words - // = do unequivocally replaces - // steps: - // 1. replace accented vowels with non accented ones - // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'` - // --- -- -- - - - - // test: - // oneliner: var str = 'αλφα βητα ΑΛΦΑ Βητα world'; var src = 'αλφα βητα'; var reg = new RegExp(src, "gi"); var replacedOnce = str.toLowerCase().replace(reg, 'α-β'); console.log(replacedOnce); - - var qAr = q.split(' '); // split to words - if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array - - if (qAr.slice(-1) == "") { - qAr.pop(); - } - - if (options.debug) console.log(qAr); - - if (qAr.length == 1) { // suggest 1st word ////////////////////// - - var kbq = kb.keyboardize(q); - - // loop through root-words - // ... to match all possible suggestions; - _kwlinks.forEach( it => { - chk = check_match(kbq, it.w, true); - - if ( chk != false ) { - results.push({ w: chk }); - - // ... also keep possible products in a list - it.c.forEach( wo => { // for every Word-Link-Node - if (proList.length < _maxResults + 2) { // NOTE: about +2 (bellow) - - // concatenate this-suggestion's product sub-list (wo.p) - // to all-suggestions posiible products-list (proList) - proList = proList.concat(wo.p); - // keep unique products in the products-list - proList = proList.filter((item, i, ar) => ar.indexOf(item) === i); - } - }); - } - - }); - } - - if (qAr.length == 2) { // suggest 2nd word ///////////////////////// - root = qAr[0]; - last = qAr[1]; - kbroot = kb.keyboardize(root); - kblast = kb.keyboardize(last); - - // TODO: - // change [_kwlinks.forEach] to for loop - // --- - _kwlinks.forEach( it => { // locate the ... - if (match.exact(kbroot, kb.keyb_array(it.w))) { // exact match of root-word - - it.c.forEach ( wo => { // loop the word-links ... - chk = check_match(kblast, wo.w, true); - if ( chk !== false ) { // if a match is found - results.push({ // keep suggestion - w: root +' '+ chk, - f: wo.f - }); - if (proList.length < _maxResults + 2) { // plus... - proList = proList.concat(wo.p); // keep products-list - proList = proList.filter((item, i, ar) => ar.indexOf(item) === i); - } - } - }); - - } - }); - } - - if (qAr.length > 2) { // suggest N-th word (N>2) /////////////// - root = qAr.shift(); // isolate first item of qAr - last = qAr.pop(); // isolate last item too - // now qAr includes only the items after root and before last; - // so qAr includes all already selected suggestions (but root) - - // prepare/cache kb-formated string for any key we're going to use - var kb_qAr = []; // array of selected suggestions in kb-format - qAr.forEach( w => { kb_qAr.push(keyboardize(w)); }) - kbroot = keyboardize(root); - kblast = keyboardize(last); - - // TODO: - // change [_kwlinks.forEach] to for loop - // --- - _kwlinks.forEach( it => { - if (is_exact_match(kbroot, it.w)) { // locate root-word - - // calculate list of common items/products (commonL) - // for selected suggestions - is1stOcc = true; // 1st occurance flag - - it.c.forEach ( swo => { - if (match_one(kb_qAr, swo.w)) { - // swo is one of the already selected suggestions - // so... update the common-(products)-L(ist) - if (is1stOcc) { - commonL = swo.p; // init list (on 1st occurance) - is1stOcc = false; - } - else { - // caclulate list of common products - // = intersection of (so-far) commonL and swo.p - // commonL = commonL.filter(value => swo.p.includes(value)); - commonL = commonL.filter(function(n) { return swo.p.indexOf(n) !== -1; }); - } - } - else { // if swo is not already selected - // then This is a possible NEXT suggestion - possibleNext.push(swo); - } - }); - - // Now that we have all the possible next suggestions - // we'll match them with the last word of the query - - possibleNext.forEach( poss => { // for tthe possible next suggestions - // if last-word-of-query matches possible word(s) - // and list of word's products has commons with commonL - // then THiS is a Valid-Next-Suggestion - chk = check_match(kblast, poss.w, true); - if (chk !== false) { - // check intersection of commonL and suggestion's product-lists - tempL = commonL.filter(value => poss.p.includes(value)); - if (tempL.length > 0) { - results.push({ - w: root +' '+ qAr.join(' ') +' '+ chk, - f: poss.f - }); - // update proList too - proList = proList.concat(tempL); - } - } - }); - } - - }); - } - - // NOTE: about +2 (vs +1) - // after having calculated next suggestions and a banch of possible products - // the proccedure is going to decide what data will return; - // if pro(ducts)List includes less items than maximum suggestions - // ... this will be the array to return. So - // ... +2 ensures that this list will not become shorter than this limit - - // calculate unique products - // (credit: https://stackoverflow.com/questions/11246758/) - let unique = proList.filter((item, i, ar) => ar.indexOf(item) === i); - - - - if (unique.length < (_maxResults +1)) { // IF list is small ...... - - // serve products instead of suggestions - // ... - results = []; - notOnThisStore = [] - unique.forEach( pr => { - for (i=0 ; i< _products.length-1 ; i++) { // for makes things faster - pi = _products[i]; - if (pi.id == pr) { - results.push(pi); - break; - } - } - }); - - } else { // remove forced link character '-' from suggestions ..... - var dirty_results = results; - results = []; - for (i=0 ; i< dirty_results.length ; i++) { - results.push({ w: dirty_results[i].w.replaceAll('-', ' ') }) - } - } - - return results; - } - - // simple, fast search products by numeric code property - // --- - function search_by_code( num ) { - const str = num.toString(); - var results = []; - var p; - for (i=0 ; i< _products.length-1 ; i++) { - p = _products[i]; - if (results.length > _maxResults) { break; } - if ( (p.bp+'-'+p.bc).indexOf(str) !== -1 ) { results.push(p) } - } - return results; - } - - // suggestions caller (router function) - // arguments: - // * qOrig : original query string - // ** list : callback array structure to host results - // --- - var isuggest = function(qOrig, list) { - var results; - - qSanit = sanitize_GR(qOrig); // sanitize greel accended chars - - // TODO: - // check if all source-lists are ready - // if not you need to wait ... - // via async promishes or synced timouts - - // if query seems to be some king of 'code/id' - if (qSanit.length>2 && qSanit.match(/^[0-9]+$/) != null) { - results = search_by_code(qOrig); - - } else { - // string match procedure with suggestions engine - results = suggestions_engine(qOrig); - - } - list(results); - } - - /** - jQuery(function() { // on document ready code ////////////////////////// - /** - - // UI-dependent code - // uses reference to specific document element (passed via options) - // ----------------------------------------------------------------------- - const searchBox = $(options.search_tag); - searchBox.typeahead( - { - hint: true, - highlight: true, - minLength: 1 - }, - { - limit: _maxResults, - name: 'kwlinks', - displayKey: 'w', - source: isuggest, - templates: { - suggestion: function(data) { - if (data.id) - return '
'+ data.w +'
'; - return '
'+ data.w +'
'; - }, - empty: '
Δεν υπάρχει στο κωδικολόγιο του καταστήματος
' - } - } - ) - .bind("typeahead:selected", function(obj, datum, name) { - if (datum.hasOwnProperty('id')) { // ** selected: PRODUCT - - // $('#js-add-product-to-order').attr('disabled', false); - // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img); - if (options.debug) console.log('selected: ', datum.id, datum.w); - - // TODO: GOTO product page - window.location.href = 'product_list?product=id-' + datum.id; - - // $("#product-quantity").trigger('focus'); - - } - else { // selected: SUGGESTION - - $('.typeahead').typeahead('val','').trigger('blur'); - $('.typeahead').typeahead('val', datum.w +' ') - .trigger("query"); - // give some time to the engine to calculate results - // then fire focus again... - setTimeout(() => { $('.typeahead').trigger('focus'); }, _timeout_ms); - } - - }) - .bind("typeahead:cursorchange", function( event, obj) { - // track cursor-chane to handle special keys after a final product is selected - - // TODO: - // handle exception where obj is undefined; - // this occures... - // when cusror returns from suggestions list back to the search field - - if (typeof obj === 'undefined') cursor_on = { none: true }; - else if (obj.hasOwnProperty('id')) cursor_on = obj; - else cursor_on = { none: true }; - }); - - // TRACK user search attempt /////////////////////////////////////////////// - $('.typeahead').on('keyup', function(e) { - if (_isReady) { - // console.log('on:', cursor_on); - // console.log('key:', e.key); - if ((e.key == ' ') && cursor_on.hasOwnProperty('id')) { - - // product is actually selected - var datum = cursor_on; - - // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img); - $("#product-quantity").trigger('focus'); - e.preventDefault(); - - } else { - if (e.key === "Enter") { - - // if curson is not on some option - if ((typeof curson_on === 'undefined') || (curson_on.none == true)) { - - // do a common search - if (options.debug) console.log('Do a Non-Suggestions search', searchBox.val()); - - // DEPRICATED: var location = encodeURI('/product_list?productSearch=%'+ searchBox.val() +'%'); - - search_results = common_search(searchBox.val()); - - - } else { - // launch a product page - var location = '/product_list?product=id-'+ cursor_on.id; - if (options.debug) console.log('product', cursor_on.id, cursor_on.w); - - console.log('go to product', location) - } - // window.location.href = location; - } - } - } - }); - - }); - */ - - - /* - function common_search(query) { - // clear ; sanitize ; split - var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' '); - - // if last item is empty, remove it - if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop() - - var results = _products; - - // for each key fitler results - qAr.forEach( key => { - results = key_sublist(key, results) - }); - - // echo products (and prepare list to POST) - var list_ = []; - results.forEach( item => { - if (options.debug) console.log(item.id, ':', item.w); - list_.push(item.id) - }) - - let l = list_.join(','); - var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`); - - if ($('.js-categories.selected').length - || $('.js-checkout.selected').length - ) { - url = encodeURI(`api/v1/products?eys_code=${l}&q=${query}`); - console.log('common search : fetch results > history.push', url, query); - fetchSearchProducts(url, query); - window.history.pushState('search', null, '/product_list?search='+query); - } else { - console.log('common search: search query > location = search') - window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`); - } - - // Redirect with POST (template) - // --- -- -- - - - - // var form = $([ - // `
`, - // '', - // '
' - // ].join('')); - // $('body').append(form); - // form.submit(); - - } - */ - - /** return from list only items that include 'key' - */ - function key_sublist(key, list) { - var result = []; - list.forEach( item => { - if (item.kb.includes(key)) { - result.push(item); - } - }); - - return result; - } -} \ No newline at end of file diff --git a/routes/dev.js b/routes/dev.js deleted file mode 100644 index cd40822..0000000 --- a/routes/dev.js +++ /dev/null @@ -1,103 +0,0 @@ -/** - * defines routes - * exports router - */ - -const Router = require('koa-router'); -const urler = require('../utils/url-util.js'); - -const data = require('../pieces/prepare.js'); - -const bench = require('../benchmark/find.js'); -const matchStr = require('../benchmark/match-str.js'); - - -// Prefix all routes with: /items -const router = new Router(); - - - -// Routes - -/* simple route example - - let items = [ - { id: 100, iname: 'Quartz Analog Wrist Watch', price: 'US $4.99'}, - { id: 101, iname: 'Leather Peep Pump Heels', price: 'US $33.56'}, - { id: 102, iname: 'Apple iPod', price: 'US $219.99'}, - { id: 103, iname: 'Prince Phantom 97P Tennnis Racket', price: 'US $50.00'}, - ]; - - router.get('/items', (ctx, next) => { - ctx.body = items; - next(); - }); - - router.get('/items/:id', (ctx, next) => { - let getCurrentItem = items.filter(function(item) { - if (item.id == ctx.params.id) { - return true; - } - }); - if (getCurrentItem.length) { - ctx.body = getCurrentItem[0]; - } else { - ctx.response.status = 404; - ctx.body = 'Item Not Found'; - } - next(); - }); -*/ - - -// NOTE: set timeout per route (how-to) -// https://stackoverflow.com/questions/66634123/how-to-add-individual-timeout-value-per-specific-route-using-node-js - - -// TEST routes -//////////////////////////////////////////////////////////////////////////////// - -router.get('/test', (ctx) => { // easy test route - // test anything ... - ctx.body = { params: urler.struct(ctx.request, ctx.url), ctx: ctx } -}); - -router.get('/test/do-data', (ctx) => { // easy test route - // test anything ... - // data.create( - // 'products', - // 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' - // ); - data.load_products( - 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' - ); - ctx.body = { success: true, operation: 'create new data' }; -}); - -// benchmark routes; -// shall be removed from production -//////////////////////////////////////////////////////////////////////////////// - -/// router.get('/test/json', (ctx, next) => { // easy test route -/// // test anything ... -/// ctx.body = { t: + new Date(), p: products }; -/// next(); -/// }); - -router.get('/bench/find', (ctx) => { // easy test route - // test anything ... - ctx.body = bench.compare(); - // next(); -}); - -router.get('/bench/match/:title', (ctx) => { // easy test route - // test anything ... [ query = 'solokata' ] - // ctx.body = bench.compare(); - ctx.body = matchStr.run(ctx.params.title); - // next(); -}); - - -// export routes - -module.exports = router; diff --git a/routes/index.js b/routes/index.js deleted file mode 100644 index e3d6230..0000000 --- a/routes/index.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * routes (index) - */ -const Router = require('koa-router'); - -const router = new Router(); - -// Require grouped routes - -let v1 = require('./v1.js'); -router.use(v1.routes()); - -let dev_paths = require('./dev.js'); -router.use(dev_paths.routes()); - -// define default route -router.get('/', (ctx) => { - ctx.body = { - success: true, - title: 'oseine', - description: 'oseine search engine is not elastic', - message: 'where are you now?' - } -}) - -module.exports = router; diff --git a/routes/v1.js b/routes/v1.js deleted file mode 100644 index 9690973..0000000 --- a/routes/v1.js +++ /dev/null @@ -1,67 +0,0 @@ -/** - * api v1 routes - */ - -const Router = require('koa-router'); -const prepare = require('../pieces/prepare.js'); - - -// Prefix all routes with: /items -const router = new Router({ - prefix: '/v1' -}); - - - -// Routes - -router.get('/prepare/:title', (ctx) => { - let result = false; - switch (ctx.params.title) { - case 'products': - result = prepare.load_products( - 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' - ); - break; - - default: - break; - - } - return { success: result }; -}) - - -router.get('/search', (ctx) => { - ctx.body = []; -}); - -router.get('/search/:title', (ctx) => { - let result = [ - { id: 10, w: 'Ένα Προϊόν' }, - { id: 15, w: 'Άλλο Προϊόν' }, - { id: 20, w: 'Προϊόν 3' }, - { id: 25, w: 'Προϊόν 4' } - ] - // console.log(ctx); - /// const kb = require('./utils/kb-util.js'); - /// let words = kb.keyboardize(kb.clean(ctx.params.title)).split(' '); - /// ctx.body = { - /// params: ctx.params, - /// results: words, - /// nxt: next - /// }; - ctx.body = result; -}); - - -/** TODO: ?? other routes ?? - * ///////////////////////////////////////////////////////////////////////////// - * + /suggest/ - * + /prepare (products, keywords, linked-terms etc) - * + /stats - */ - - -// export routes -module.exports = router; diff --git a/utils/kb-util.js b/utils/kb-util.js deleted file mode 100644 index aac88f9..0000000 --- a/utils/kb-util.js +++ /dev/null @@ -1,88 +0,0 @@ -/** - * fast string manipulation utilities - * for bi-lingual (EL/EN) words/phrases - * based on the keyboard layout - */ - -// suplamentary arrays (mostly for cache) -// --- -- -- - - - - -var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); - -var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); - -var map = new Map(); -for (var i=0; i { - ap = pair.split(' '); - accented_vowels.push({ - a: ap[0], // accented - p: ap[1] // pure = non accended - }); -}); - - -// translates string to keyboard-latin keys -// (the ones that used whan typing each letter of the word) -const keyboardize = (str) => { - str = str.replace('\'',''); - var out = ''; - // [map]'s implementation is 40x faster than [for]'s - for (var i=0 ; i< str.length; i++) out += map.get(str[i]); - return out; -} - -// keyboardize an array of strings -const keyb_array = (arr) => { - kb_arr = []; - arr.forEach( w => { - kb_arr.push(keyboardize(w)); - }); - return kb_arr; -} - - -// transforms to lowercase; handles sigma-teliko -const sanitizeGR = (str) => { - str = str.toLowerCase(); - - // replace accended vowels with pure ones - accented_vowels.forEach( v => { - str = str.replaceAll(v.a, v.p); - }); - - // replace sigma on the end of words - str = str + ' '; - str = str.replaceAll('σ-', 'ς-'); - str = str.replaceAll('σ ', 'ς '); - - return str; -} - -// removes non keyword characters [+ . , !] and internal multiple-spaces -// @param txt (string): product description -const clean = (txt) => { - return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space - .replace('!','').replace('\"', '') // remove character - .replace(' ',' ').replace(' ',' '); // remove multiple spaces -} - - -// exports -// --- -- -- - - - - -module.exports = { - keyboardize, - keyb_array, - sanitizeGR, - clean -}; \ No newline at end of file diff --git a/utils/match-util.js b/utils/match-util.js deleted file mode 100644 index 5e8c648..0000000 --- a/utils/match-util.js +++ /dev/null @@ -1,154 +0,0 @@ -/** - * match utility; - * includes fuzzy and partial match functions too; - * many of them return a match-rate - */ - - -// fuzzy match -//////////////////////////////////////////////////////////////////////////////// - -/** Ngram fuzzy match algorithm - * (simple and fast) - */ -const createNgram = (word, n) => { // Ngram creation - if (word.length <3) return word; - const vector = []; - for (let i = 0; i < word.length-n+1; ++i) { - vector.push(word.slice(i, i + n)); - } - return vector; -}; - -/** similarity - * rates similarity between 2 words - * based on Ngram matches of N = n letters; - * implements a 2-dim check (all a-Ngrams vs all all b-Ngrams) - * - * @param {string} a : first word - * @param {string} b : second word - * @param {int} n : Ngram base - * @returns {float} : match percentage as a float in [0, 1] - */ -const similarity = (a, b, n) => { // Ngram match score - if (a.length > 0 && b.length > 0) { - const aNgram = createNgram(a, n); - const bNgram = createNgram(b, n); - let hits = 0; - for (let x = 0; x < aNgram.length; ++x) { - for (let y = 0; y < bNgram.length; ++y) { - if (aNgram[x] === bNgram[y]) { - hits += 1; - } - } - } - if (hits > 0) { - const union = aNgram.length + bNgram.length; - return (2.0 * hits) / union; - } - } - return 0; -}; - -/** resemblance - * is an alternative similarity rating; - * implements an 1-dim Ngram similarity check - * and it's much faster than similarity() - */ -const resemblance = (a, b, n) => { - if (a.length > n && b.length >= a.length) { - const aNgram = createNgram(a, n); - let hits = 0; - for (let i = 0; i < aNgram.length; ++i) { - if (b.includes(aNgram[i])) { - hits++; - } - } - if (hits > 0) { - // rate resemblance based on hits and length-similarity - return (hits / aNgram.length) * (a.length / b.length); - } - } - return 0; -} - - -// exact and partial match -//////////////////////////////////////////////////////////////////////////////// - -/** is_exact_match - * check if a searching string -> query (string/latin in kb-format) - * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) - * - * @param {string} query: searching string; string/latin in kb-format - * @param {array} chkArr: array of synonyms; (array of utf-8/strings) - * @return {boolean}: true|false - */ -function exact( query, chkArr ) { - found = false; - chkArr.forEach( w => { if (w == query) found = true }); - return found; -} - -function partial( query, chkArr ) { - found = false; - chkArr.forEach( w => { if (w.includes(query)) found = true }); - return found; -} - -/** is exact match + weight rating - * @returns {float} weight rates depth of array when a match is found - */ -function weighted_exact( query, chkArr ) { - let weight = 0; // closer to left/begin rating - let len = chkArr.length; - for(let i = 0; i < len ; i++) { // i ~ depth - if (chkArr[i] == query) { - // weights array depth - weight = (len - i + 1.0) / len; - break; - } - } - return weight; -} - - -/** is partial match + weight rating - * - * @param query (string): searching string; string/latin in kb-format - * @param chkArr (array): array of synonyms; (array of utf-8/strings) - * @returns {float} weight rates both match position and depth of match - * - * (*) optimization NOTE: - * Given the weight `W` and the depth `i`, - * the best weight for next `i` shall be: `(L - (i+1)) / L` - * To be imposibbe to have a better weight, should: - * W > (L - (i+1)) / L => ... => i > (L - L*W - 1) - */ -function weighted_partial( query, chkArr ) { - let rate = 0; - let weight = 0; - let len = chkArr.length; - for( let i = 0 ; i < len ; i++ ) { - let chk = chkArr[i].indexOf(query) - if (chk != -1) { - rate = (len - i) / (len + 2.0 * chk); - weight = rate > weight ? rate : weight; - } - if (i > (len - len * weight - 1)) { - break; // better rating is not possible (*) - } - } - return weight; -} - - -// exports -module.exports = { - exact, - partial, - weighted_exact, - weighted_partial, - similarity, - resemblance -} diff --git a/utils/mem-usage.js b/utils/mem-usage.js deleted file mode 100644 index 766a93e..0000000 --- a/utils/mem-usage.js +++ /dev/null @@ -1,20 +0,0 @@ -/** - * memory usage report utility - */ - -const formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`; - -function report() { - let memoryData = process.memoryUsage(); - - let memoryUsage = { - rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`, - heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`, - heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`, - external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`, - }; - - console.log(memoryUsage); -} - -module.exports = { report } diff --git a/utils/url-util.js b/utils/url-util.js deleted file mode 100644 index e86ad9b..0000000 --- a/utils/url-util.js +++ /dev/null @@ -1,23 +0,0 @@ -/** - * url utility - */ - -const querystring = require('querystring'); - -function struct(req, url) { - let url_parts = url.split('?'); - let query = (url_parts.length > 1) - ? querystring.decode(url_parts[1]) - : {}; - return { - method: req.method, - host: req.host, - path: url_parts[0], - query: query - } -} - -/** - * exports - */ -module.exports = { struct } \ No newline at end of file -- cgit v1.2.3