summaryrefslogtreecommitdiff
path: root/app
diff options
context:
space:
mode:
authorGeo Halkiadakis <gchalkiadakis@sklavenitis.co.gr>2024-04-17 18:32:13 +0300
committerGeo Halkiadakis <gchalkiadakis@sklavenitis.co.gr>2024-04-17 18:32:13 +0300
commitd9364679a51ff80db8e5948ab089d749da36a6b2 (patch)
treef847a03a017be8857923aaaae9f8dcf7177b2ba9 /app
parent6248d2cf7a2214ac32628bc58248694108b4d8bf (diff)
downloadoseine-master.tar.gz
oseine-master.tar.bz2
oseine-master.zip
dockerize the appHEADmasterdevelopment
Diffstat (limited to 'app')
-rw-r--r--app/app.js21
-rw-r--r--app/benchmark/find.js112
-rw-r--r--app/benchmark/match-str.js80
-rw-r--r--app/pieces/prepare.js128
-rw-r--r--app/pieces/retro-search.js305
-rw-r--r--app/pieces/suggest.js786
-rw-r--r--app/routes/dev.js103
-rw-r--r--app/routes/index.js26
-rw-r--r--app/routes/v1.js68
-rw-r--r--app/utils/kb-util.js88
-rw-r--r--app/utils/match-util.js154
-rw-r--r--app/utils/mem-usage.js20
-rw-r--r--app/utils/url-util.js23
13 files changed, 1914 insertions, 0 deletions
diff --git a/app/app.js b/app/app.js
new file mode 100644
index 0000000..42ff8c1
--- /dev/null
+++ b/app/app.js
@@ -0,0 +1,21 @@
+// app.js
+
+const Koa = require('koa');
+// const { koaBody } = require('koa-body');
+
+// load parameters
+require('dotenv').config();
+
+// define app
+const app = new Koa();
+
+// middleware
+// app.use(koaBody());
+
+// load routes
+let _r = require('./routes');
+app.use(_r.routes()).use(_r.allowedMethods());
+
+// start server listening on APP_PORT
+// app.listen(process.env.APP_PORT);
+app.listen(3000);
diff --git a/app/benchmark/find.js b/app/benchmark/find.js
new file mode 100644
index 0000000..1e86c24
--- /dev/null
+++ b/app/benchmark/find.js
@@ -0,0 +1,112 @@
+/**
+ * benchmark: find a product in product list
+ * using: for vs forEach vs find
+ */
+
+const microtime = require('microtime');
+
+var products = require('../../data/products.json');
+if (products.length == 0) {
+ const prepare = require('../pieces/prepare.js');
+ prepare.load_products(
+ 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json'
+ );
+}
+
+
+
+var selected = [];
+products.forEach( pr => {
+ if (Math.floor(Math.random() * 100) > 85) {
+ selected.push(pr.id);
+ }
+});
+
+
+function compare() {
+
+ let n = 4;
+
+ var f0 = microtime.nowDouble();
+ for(i=0 ; i < n ; i++) byFor();
+ var f1 = microtime.nowDouble();
+ //
+ var e0 = microtime.nowDouble();
+ for(i=0 ; i < n ; i++) byEach();
+ var e1 = microtime.nowDouble();
+ //
+ var b0 = microtime.nowDouble();
+ for(i=0 ; i < n ; i++) byFind();
+ var b1 = microtime.nowDouble();
+
+ return {
+ n: n,
+
+ for: f1-f0,
+ items_for: byFor(),
+
+ each: e1-e0,
+ items_each: byEach(),
+
+ find: b1-b0,
+ items_find: byFind(),
+
+ sel: selected,
+ }
+}
+
+
+function byFor() {
+ var items = [];
+ var notFound = [];
+ var found;
+ selected.forEach( id => {
+ found = false;
+ for(i = 0; i < products.length; i++) {
+ if (products[i].id == id) {
+ items.push(products[i]);
+ found = true;
+ break;
+ }
+ }
+ if (!found) notFound.push(id)
+
+ })
+ return {items: items, nf: notFound};
+}
+
+
+function byEach() {
+ var items = [];
+ var notFound = [];
+ var found;
+ selected.forEach( id => {
+ found = false;
+ products.forEach( pr => {
+ if (pr.id == id) {
+ items.push(pr);
+ found = true;
+ }
+ });
+ if (!found) notFound.push(id)
+ });
+ return {items: items, nf: notFound};
+}
+
+function byFind() {
+ var items = [];
+ var notFound = [];
+ var result;
+ selected.forEach( id => {
+ found = false;
+ result = products.find((pr) => pr.id == id);
+ if (result === undefined) notFound.push(id)
+ else items.push(result);
+ });
+ return {items: items, nf: notFound};
+}
+
+module.exports = {
+ compare,
+ byFor
+} \ No newline at end of file
diff --git a/app/benchmark/match-str.js b/app/benchmark/match-str.js
new file mode 100644
index 0000000..225cbc2
--- /dev/null
+++ b/app/benchmark/match-str.js
@@ -0,0 +1,80 @@
+var microtime = require('microtime');
+
+const match = require('../utils/match-util.js');
+const memory_usage = require('../utils/mem-usage.js');
+const kb = require('../utils/kb-util.js');
+
+// const products = require('../../data/products.json');
+var products;
+try {
+ products = require('../../data/products.json');
+} catch (e) {
+ products = [];
+}
+
+// memory_usage.report();
+
+// test runner
+function run(query) {
+
+ var f0 = microtime.nowDouble();
+ let result;
+ // for(let i = 0 ; i < 20 ; i++)
+ result = matchQuery(query);
+ var f1 = microtime.nowDouble();
+
+ memory_usage.report();
+
+ return {
+ t: f1-f0,
+ q: query,
+ result: result
+ }
+}
+
+// sample search implementation
+function matchQuery(query) {
+ var result = [];
+
+ let q = kb.keyboardize(query);
+
+ for(let i = 0; i < products.length; i++) {
+
+ let rate = 0;
+ let found = false;
+ let x;
+ let src = products[i].kb.split(' '); // (array) source of product (key-)words
+ if ((x = match.weighted_exact(q, src)) > 0) {
+ // console.log(products[i].kb, x);
+ rate = 5.0 + x;
+ found = true;
+ }
+ else if ((x = match.weighted_partial(q, src)) > 0) {
+ rate = 3.0 + x;
+ found = true;
+ }
+ else {
+ let similarity = 0;
+ src.forEach( w => {
+ let sim = match.resemblance(q, w, 2);
+ if (sim > 0.6) {
+ found = true;
+ similarity = (sim > similarity) ? sim : similarity;
+ }
+ });
+ if (found) {
+ rate = 2 * similarity;
+ }
+ }
+
+ if (found) {
+ products[i].rate = rate;
+ result.push(products[i]);
+ }
+ }
+
+ return result.sort((a,b) => b.rate - a.rate).slice(0, 48);;
+}
+
+
+module.exports = { run }
diff --git a/app/pieces/prepare.js b/app/pieces/prepare.js
new file mode 100644
index 0000000..20825cd
--- /dev/null
+++ b/app/pieces/prepare.js
@@ -0,0 +1,128 @@
+const fs = require('fs');
+var request = require('request');
+
+const kb = require('../utils/kb-util.js');
+
+
+// const https = require("https");
+
+
+/* TODO: ?? parallel read
+ // var request = require('request-promise');
+ var calls = [
+ request({
+ url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json',
+ // headers: { ... }
+ }),
+ request({
+ url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json',
+ // headers: { ... }
+ }),
+ // + linked-terms and replaces
+ // and more..
+ ];
+
+ Promise.all(calls).then(function(results) {
+ // do something with results[0]
+ // do something with results[1]
+ // ...
+ });
+*/
+
+
+/** create entity file from url
+ * used for entities like 'linked-words', 'synonyms', etc.
+ *
+ * @param {string} entity: use the entity name
+ * @param {string} url: prepared entity json-file in remote server
+ */
+function create(entity, url) {
+ request(url,
+ function (error, response, body) {
+ if (!error && response.statusCode == 200) {
+ // body is a ready json-string; no need to parse and (re-)stringify
+ try {
+ fs.writeFileSync(`${__dirname}/../../data/${entity}.json`, body, 'utf-8');
+ // file written successfully
+ } catch (err) {
+ console.error(err);
+ }
+ }
+ }
+ );
+}
+
+function get_data_structure() {
+ // data structure is an array of `{ url:.., expiration:.., path:.. }` objects
+}
+
+function set_data_structure(node) {
+
+}
+
+
+function load_products(url) {
+ request(url,
+ function (error, response, body) {
+ if (!error && response.statusCode == 200) {
+ // body is a ready json-string; no need to parse and (re-)stringify
+ json = JSON.parse(body);
+ newJson = [];
+
+ // TODO:
+ // + attach handle synonyms
+ // + mark brand-names
+ // + attach category-names and SAP-categories
+ // + construct combo words
+ // + remove non-important words
+ // + normalize popularity
+
+ json.forEach( p => {
+ newJson.push({
+ id: p.id,
+ w: p.w,
+ kb: kb.keyboardize(kb.clean(p.w))
+ });
+ });
+
+ try {
+
+ fs.writeFileSync(
+ `${__dirname}/../../data/products.json`,
+ JSON.stringify(newJson),
+ 'utf-8'
+ );
+ return true; // file written successfully
+
+ } catch (err) {
+ console.error(err);
+ return false;
+ }
+ }
+ }
+ );
+}
+
+
+// create data
+// testing an implemenatatin of a custom local-fs-cache system
+function random() {
+ var data = [];
+ for( let i = 0; i < (Math.floor(Math.random() * 100) +15) ; i++) {
+ data.push({ id: i, x5: i*5 });
+ }
+
+ try {
+ fs.writeFileSync(__dirname + '/../data/random.json', JSON.stringify(data), 'utf-8');
+ // file written successfully
+ } catch (err) {
+ console.error(err);
+ }
+}
+
+
+module.exports = {
+ create,
+ random,
+ load_products
+} \ No newline at end of file
diff --git a/app/pieces/retro-search.js b/app/pieces/retro-search.js
new file mode 100644
index 0000000..4e26720
--- /dev/null
+++ b/app/pieces/retro-search.js
@@ -0,0 +1,305 @@
+const kb = require('../utils/kb-util.js');
+const match = require('../utils/match-util.js');
+
+const products = require('../../data/products.json');
+
+
+/** VARIABLES
+ * may passed as module arguments
+ * -----------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////////
+
+
+var _allowFuzzy = true; // enable|disable fuzzy search
+var n = 2; // Ngram base
+var _fuzzyLimit = .5; // minimum bigram score for being considered a match
+var max_list = 24;
+var tolerance = 42;
+var STORE = { id: 904 };
+
+var _kwlinks; // keyword links (word-connections; imported via ajax-get)
+var _products = []; // all products (imported via ajax-get)
+
+// setup options
+var _maxResults = options.max_list; // limit suggestions
+var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions
+var _Ngram_base = 2; // number of N in Ngram spliting algorithm
+var _isReady = false; // whether the searchbox is ready to be used
+
+// product keywords
+var keywordsURL = options.keywords_json;
+
+var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true }
+
+
+
+
+
+
+/** SUPPLEMENTARY FUNCTIONS
+ * -----------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////////
+
+
+
+// callback function for sorting resulrs per r (=rating) property
+function compare_rate(a,b) {
+ return (a.r < b.r);
+}
+
+
+
+/** SEARCH ENGINE
+ * -------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////
+
+
+
+
+function matchWordInList(q, list = false) {
+
+ if (list !== false && list.length == 0) return []; // no results
+
+ var result = [];
+ var firstPass = false;
+
+ if (list === false) {
+ firstPass = true; // on first pass
+ list = products; // list is all products
+ }
+
+ for(let i = 0 ; i < list.length ; i++) {
+
+ // check exact + rate
+
+ // else check partial + rate
+
+ // else check similarity + rate
+
+ let found = false;
+ let similarity = 0;
+ products[i].kb.split(' ').forEach( w => {
+ let sim = match.resemblance(q, w, 2);
+ if (sim > 0.5) {
+ found = true;
+ similarity = (sim > similarity) ? sim : similarity;
+ }
+ });
+ if (found) {
+ products[i].similarity = similarity;
+ products[i].rating = (firstPass)
+ ? similarity * 2.0
+ : products[i].rating + similarity * 2.0
+ result.push(products[i]);
+ }
+ }
+
+ return result
+}
+
+
+
+// suggestions engine //////////////////////////////////////////////////
+// ---
+function suggestions_engine(query) {
+ var results = [];
+ var pot = []; pot.length = 0;
+
+ // clean and sanitize and mark links onto q(uery) string
+ var q = kb.keyboardize( kb.sanitizeGR( kb.clean(query.trim()) ) ).trim();
+
+ // TODO:
+ // construct direct-linked words
+ // = do unequivocally replaces
+ // steps:
+ // 1. replace accented vowels with non accented ones
+ // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'`
+
+ var qAr = q.split(' '); // split to words
+ /// if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array
+ ///
+ /// if (qAr.slice(-1) == "") {
+ /// qAr.pop();
+ /// }
+
+
+ pot = _products; // potential results // NOTE: CRITICAL: BY REFERENCE
+
+ var wi = 0; // word index (from list)
+ var wc = qAr.length;
+
+ qAr.forEach( w => {
+
+ let sf = []; // (matches) so far
+ let mi; // position of match
+ wi++;
+
+ pot.forEach( it => {
+ let matched = false;
+ let tester = ' '+ it.kb + ' ';
+
+ // reset previous history and ratings
+ if (wi == 1) {
+ it.r = 0;
+ it.history = [];
+ }
+
+
+ // rate word-match > start-match > simple-match
+ // ... up to 8 points
+
+ if (tester.indexOf(' '+ w +' ') != -1) {
+ it.r += 9;
+ it.history.push({ w: w, rate: 9 });
+ matched = true;
+ }
+ else if (tester.indexOf(' '+ w) != -1) {
+ it.r += 5;
+ it.history.push({ w: w, rate: 5 });
+ matched = true;
+ }
+ else if (tester.indexOf(w) != -1) {
+ it.r += 2;
+ it.history.push({ w: w, rate: 2 });
+ matched = true;
+ }
+
+ // rate `near-to-start` matching .. up to 7p
+ // rate `earlyness` of word in query .. up to 7p
+
+ if ((mi = tester.indexOf(' '+w)) != -1) {
+ let fc1 = 100 - ((mi < 99) ? mi : 99); // near-to-start factor
+ let fc2 = wc - wi + 1; // query earlyness factor
+ let r1 = Math.floor(7*fc1/100);
+ let r2 = Math.floor(7*fc2/wc);
+
+ it.r += (r1 + r2);
+ it.history.push({ w: w, left: [fc1, r1], early: [fc2, r2] });
+ }
+ if (matched) sf.push(it);
+ });
+
+ if ((sf.length > (_maxResults + Math.floor(_maxResults/2)))
+ || (wi == 1) ) {
+ // ..if pot has a fair amount (= max + 50%) of results
+ // ..or these are results of '1st-query-word'
+ // set sf as new source
+ pot.lenght = 0; pot = [];
+ pot = JSON.parse(JSON.stringify(sf)); // copy by value
+
+ } else {
+ // else.. keep the source list and increase of 'so-far rating'
+ // console.log('found small list', sf, pot)
+ pot.forEach( it => {
+ sf.forEach( si => {
+ if (it.id == si.id) {
+ it.r += 10;
+ it.history.push({ w: w, plus: '+10'});
+ }
+ });
+ });
+ }
+ });
+
+ // sort results, get max-list of best rated
+ results = (pot.length > _maxResults)
+ ? pot.sort(compare_rate).slice(0, _maxResults)
+ : pot.sort(compare_rate)
+
+ if (options.debug) console.log(results);
+
+ return results;
+}
+
+
+// sub-module (start)
+////////////////////////////////////////////////////////////////////////////
+
+function update_common_search_results(q, results) {
+ let queries = getSessionObj('sr');
+ let newSRlist = [];
+ let isnewQ = true;
+ if (queries === null) {
+ setSessionObj('sr', [{
+ q: q,
+ result: result,
+ t: + new Date()
+ }]);
+ return true;
+
+ } else {
+
+ queries.forEach(it => {
+ if (it.q == q) {
+ newSRlist.push({
+ q:q,
+ result: result,
+ t: + new Date()
+ });
+ isnewQ = false;
+ } else { newSRlist.push(it); }
+ });
+
+ if (isnewQ) {
+ newSRlist.push({
+ q:q,
+ result: result,
+ t: + new Date()
+ });
+ }
+
+ return true;
+ }
+}
+
+
+
+////////////////////////////////////////////////////////////////////////////
+// sub-module (end)
+
+
+function common_search(query) {
+ // clear ; sanitize ; split
+ var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' ');
+
+ // if last item is empty, remove it
+ if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop()
+
+ var results = _products;
+
+ // for each key fitler results
+ qAr.forEach( key => {
+ results = key_sublist(key, results)
+ });
+
+ // echo products (and prepare list to POST)
+ var list_ = [];
+ results.forEach( item => {
+ if (options.debug) console.log(item.id, ':', item.w);
+ list_.push(item.id)
+ })
+
+ // *** TODO: keep results in local storage (or on session storage)
+
+ // update_common_search_results(query, list_);
+
+ let l = list_.join(',');
+ var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`);
+
+ console.log('common search: search query > location = search')
+ window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`);
+
+}
+
+/** return from list only items that include 'key'
+ */
+function key_sublist(key, list) {
+ var result = [];
+ list.forEach( item => {
+ if (item.kb.includes(key)) {
+ result.push(item);
+ }
+ });
+
+ return result;
+}
diff --git a/app/pieces/suggest.js b/app/pieces/suggest.js
new file mode 100644
index 0000000..c0a6e04
--- /dev/null
+++ b/app/pieces/suggest.js
@@ -0,0 +1,786 @@
+const kb = require('../utils/kb-util.js');
+const match = require('../utils/match-util.js');
+
+var _allowFuzzy = true; // enable|disable fuzzy search
+var n = 2; // Ngram base
+var _fuzzyLimit = .5; // minimum bigram score for being considered a match
+
+
+/** (Search) SUGGESTIONS ENGINE
+ * ---------------------------------------------------------------------------
+ *
+ * Operates in dual mode;
+ * -- suggestions engine (interactive)
+ * -- classic-like mode (passive)
+ *
+ * TODO:
+ * CRITICAL: (optimization)
+ * Search initialization uses quite a lot of network sources;
+ * thus it should be started in a later time;
+ * lets say ... after `x` seconds
+ * or... when document/core-ui-elements are ready
+ *
+ * @parametres (json) : options
+ * ---
+ * @var keywords_json (sring) : endpoint url of linked keywords structure
+ * @var products_json (string) : endpoint od product descriptions
+ * @var search_tag (string) : selector of field that shall act as typeahead-suggestions
+ * @var visualize_search_results_url (str) : url that will visualize the sended "results-page"
+ * @var debug (bool) : if true sends several debug console messages; if false mesagges are eliminated
+ */
+
+suggestions_module({
+ keywords_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json',
+ products_json: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json',
+ search_tag: '#tagsInput',
+ visualize_search_results_url: '/product_list',
+ debug: ((location.hostname == 'localhost') || (location.hostname == '127.0.0.1'))
+});
+
+function suggestions_module(options) {
+
+ /** CONTENTS
+ *
+ * +1: Variables
+ *
+ * +2: Purify string functions
+ * + keyboardize
+ * + sanitize_GR
+ * + clean
+ * + mark_explicit_links
+ *
+ * +3: Supplementary function (vanilla js)
+ * + ajax_get(url, callback)
+ * + createNgram (fuzzy)
+ * + checkSimilarity (fuzzy)
+ * + check_match
+ * + is_exact_match
+ * + match_one
+ *
+ * +4: Actual data loading (async)
+ *
+ * +5: Suggestions Engine (jQuery)
+ * + suggestions_engine
+ *
+ */
+
+ console.log('executing suggestions...');
+
+
+ /** 1. VARIABLES
+ * -------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////
+
+ var _kwlinks; // keyword links (word-connections; imported via ajax-get)
+ var _products = []; // all products (imported via ajax-get)
+
+ // setup options
+ var _maxResults = 24; // limit suggestions
+ var _blendProds = 4; // minimum final-produncts to blend with next-word suggestions
+ var _timeout_ms = 100; // time (in ms) for the search engine to find matches (before rendering)
+
+ var _Ngram_base = 2; // number of N in Ngram spliting algorithm
+ var _isReady = false; // whether the searchbox is ready to be used
+
+ // product keywords
+ var keywordsURL = options.keywords_json;
+
+ var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true }
+
+
+ /** replaces (correcting descriptions)
+ * == construct unequivocally liked words //////////////////////////////////
+ * -------------------------------------------------------------------------
+ *
+ * NOTE: TODO:
+ * in future implementations multi-word keywords
+ * may use the non-breaking space as conecting character (\u00A0) instead of dush (-)
+ * (or maybe both of them)
+ *
+ * also TODO:
+ * in future implementaions linked words may passed via some endpoint
+ */
+ replaces = [];
+ replaceSource = [
+ '3Α;3-ΑΛΦΑ',
+ '3 ΑΛΦΑ;3-ΑΛΦΑ',
+ 'HEAD & SHOULDERS;HEAD&SHOULDERS',
+ 'HEAD N SHOULDERS;HEAD&SHOULDERS',
+ 'W.K Kellogg; W-K-Kellogg',
+ 'W.K Kellogg;',
+ '7 DAYS;7-DAYS',
+ '7 UP;7UP',
+ '7-UP;7UP',
+ 'ΜΠΑΡΜΠΑ ΣΤΑΘΗ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ',
+ 'ΜΠΑΡΜΠΑ ΣΤΑΘΗΣ;ΜΠΑΡΜΠΑ-ΣΤΑΘΗΣ',
+ 'COCA COLA;COCA-COLA',
+ 'J P. CHENET; J.P.CHENET',
+ 'J.P. CHENET; J.P.CHENET',
+ 'COCACOLA;COCA-COLA',
+ 'NES CAFE;NESCAFE',
+ 'NES-CAFE;NESCAFE',
+ 'LE PETIT MARSEILLAIS;LE-PETIT-MARSEILLAIS',
+ 'PETIT MARSEILLAIS;PETIT-MARSEILLAIS',
+ 'Το Μάννα;Το-Μάννα',
+ 'Χωρίς Γλουτένη;Χωρίς-Γλουτένη',
+ 'Χωρίς Ζάχαρη;Χωρίς-Ζάχαρη',
+ 'Χωρίς Αλάτι;Χωρίς-Αλάτι',
+ 'Χωρίς Λακτόζη;Χωρίς-Λακτόζη',
+ 'Χωρίς Συντηρητικά;Χωρίς-Συντηρητικά',
+ 'Χωρίς Αλκοόλ;Χωρίς-Αλκοόλ',
+ 'Χωρίς Kαφεϊνη;Χωρίς-Kαφεϊνη',
+ 'Χωρίς Kαφεΐνη;Χωρίς-Kαφεϊνη',
+ 'Χωρίς Γλυκάνισο;Χωρίς-Γλυκάνισο',
+ 'Χωρίς Ανθρακικό;Χωρίς-Ανθρακικό',
+ 'Υψηλής Παστερίωσης;Υψηλής-Παστερίωσης',
+ 'Ολικής Άλεσης;Ολικής-Άλεσης',
+ 'Ολικής Aλέσεως;Ολικής-Aλέσεως',
+ 'Χαρτί Υγείας;Χαρτί-Υγείας',
+ 'ρολό υγείας;ρολό-υγείας',
+ 'χαρτί τουαλέτας;χαρτί-τουαλέτας',
+ 'Χαρτί Κουζίνας;Χαρτί-Κουζίνας',
+ 'ρολό κουζίνας;ρολό-κουζίνας',
+ 'Μπάρες Δημητριακών;Μπάρες-Δημητριακών',
+ 'Ας Μαγειρέψουμε;Ας-Μαγειρέψουμε',
+ 'ΚΡΙΣ ΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ',
+ 'ΚΡΙΣΚΡΙΣ;ΚΡΙΣ-ΚΡΙΣ',
+ 'ΚΡΙ ΚΡΙ;ΚΡΙ-ΚΡΙ',
+ 'ΚΡΙΚΡΙ;ΚΡΙ-ΚΡΙ',
+ 'ΕΛ ΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ',
+ 'ΕΛΓΚΡΕΚΟ;ΕΛ-ΓΚΡΕΚΟ',
+ 'FREE STEP;FREE-STEP',
+ 'EL SABOR;EL-SABOR',
+ 'ELSABOR;EL-SABOR',
+ 'DOUWE EGBERTS;DOUWE-EGBERTS',
+ 'DOUWEEGBERTS;DOUWE-EGBERTS',
+ 'ΕΝ ΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ',
+ 'ΕΝΕΛΛΑΔΙ;ΕΝ-ΕΛΛΑΔΙ',
+ 'SPIN SPAN;SPIN-SPAN',
+ 'SPINSPAN;SPIN-SPAN',
+ 'CRETA-FARMS;CRETA-FARM',
+ 'CRETA-FARM;CRETA-FARM',
+ 'CRETAFARM;CRETA-FARM',
+ 'Ολες-τις-Χρήσεις;Ολες-τις-Χρήσεις',
+ 'Χωρίς προσθήκη ζάχαρης;Χωρίς-ζάχαρη',
+ 'φρουι ζελε, φρουί-ζελε',
+ 'DR BECKMANN, DR-BECKMANN'
+ ];
+ replaceSource.forEach( it => {
+ st = sanitize_GR(
+ it.toLowerCase()
+ ).split(';');
+ replaces.push({
+ src: ' '+ st[0] +' ', // encolse between spaces
+ trg: ' '+ st[1] +' ' // to separate from before/after words
+ });
+ });
+
+ /** mark_explicit_links
+ *
+ * mark linked words shall be handled as one-(key)word
+ * also, edit common mistakes with suggested replaces
+ *
+ * @param str
+ * @return
+ */
+ function mark_explicit_links(str) {
+ str = ' '+ str +' ';
+ replaces.forEach( it => { str = str.replaceAll(it.src, it.trg); });
+ return str.replaceAll(' ', ' ').trim();
+ }
+
+
+
+
+
+ /** 3. SUPPLEMENTARY FUNCTIONS
+ * -------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////
+
+
+ // TODO:
+ // exclude some generic non-critial words when proccessing user's query
+ // ** example code to work with:
+ // var ignoredKeys_kb = []; // keywords to ignore (in kb-format)
+ // 'μας με σε για του της των από στο στον &'.split(' ').forEach(w => { ignoredKeys_kb.push(keyboardize(w)); });
+
+ // pure JS ajax GET request; return data as JSON
+ // no fancy things like UTF8; if needed use base64
+ function ajax_get(url, callback) {
+ var xmlhttp = new XMLHttpRequest();
+ xmlhttp.onreadystatechange = function() {
+ if (xmlhttp.readyState == 4 && xmlhttp.status == 200) {
+ try {
+ var data = JSON.parse(xmlhttp.responseText);
+ } catch(err) {
+ if (options.debug) console.log(err.message + " in " + xmlhttp.responseText);
+ return;
+ }
+ callback(data);
+ }
+ };
+ xmlhttp.open("GET", url, true);
+ xmlhttp.send();
+ }
+
+
+
+
+ // Search Endine's match functions
+ //////////////////////////////////////////////////////////////////////////////
+
+ /** check_match
+ * ---
+ * check if a searching string -> query (string/latin in kb-format)
+ * matches an item of the array of synonyms -> chkArr (array of utf-8/strings)
+ *
+ * + option to use fyzzy (bigram) match
+ *
+ * @return: matched string (utf-8) --or-- false (if not matched)
+ */
+ function check_match( query, chkArr, fuzzy = false ) {
+ var result = '';
+ var found = false;
+
+ if (query == ' ') return chkArr[0];
+
+ chkArr.forEach( chk => {
+ if (!found) {
+ chk_kb = keyboardize(chk);
+ if ( (chk_kb.indexOf( query ) !== -1)
+ || (fuzzy && (match.similarity(chk_kb, query, n) > _fuzzyLimit)) ) {
+ found = true;
+ result = chk;
+ }
+ }
+ });
+ return found ? result : false;
+ }
+
+ /** is_exact_match
+ *
+ * check if a searching string -> query (string/latin in kb-format)
+ * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings)
+ *
+ * @param query (string): searching string; string/latin in kb-format
+ * @param chkArr (array): array of synonyms; (array of utf-8/strings)
+ * @return (boolean): true|false
+ */
+ function is_exact_match( query, chkArr ) {
+ found = false;
+ chkArr.forEach( w => { if (keyboardize(w) == query) found = true });
+ return found;
+ }
+
+ /** match_one
+ *
+ * check if at-least-ONE item from an array of query-words -> qArr (array of string/latin in kb-format)
+ * matches any item of an array of synonyms -> chkArr (array of utf-8/strings)
+ *
+ * @return (boolean) true|false
+ */
+ function match_one( qArr, chkArr ) {
+ found = false;
+ qArr.forEach( query => {
+ chkArr.forEach( w => { if (keyboardize(w) == query) found = true });
+ });
+ return found;
+ }
+
+
+
+
+ /** 4. ACTUAL DATA LOADING
+ * -------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////
+
+
+
+ // TODO:
+ // control completion of async loads via .then() rather by this custom structrure
+ // Need to rewrite the folllowing code ............................. from here
+ // ...........................................................................
+ // ...........................................................................
+
+
+ /** workline
+ * -------------------------------------------------------------------------
+ * custom object/data-structure
+ * to track status/completion of async svents
+ * (it does the job using a special set method)
+ */
+ var workline = {
+
+ trackerJL : 0,
+ set jsonLoaded(x) {
+ this.trackerJL = x;
+
+ // fire event on certain values
+ if (x == 3) {
+ if (options.debug) console.info('suggestion-engine requirements fulfilled');
+ _isReady = true;
+ // code to execute
+ // ...
+ }
+ },
+ get jsonLoaded() { return this.trackerJL; }
+ };
+
+
+ /** LOAD DATA (from endoints)
+ * ---------------------------------------------------------------------------
+ */
+
+ ajax_get( keywordsURL, function(data) { // get _kwlinks
+ _kwlinks = data;
+ if (options.debug) console.log('...keywords loaded;');
+ workline.jsonLoaded++;
+ });
+
+ function load_store_products(storeID) {
+ ajax_get(options.products_json, function(data) { // get stor's _products
+ _products = data;
+ _products.forEach(p => {p.kb = keyboardize(p.w).toLowerCase()} );
+ if (options.debug) console.log('...products loaded;');
+ workline.jsonLoaded++;
+ CURRENT_STOREs_CATALOG = storeID;
+ });
+ }
+
+ var STORE = { id: 904 };
+
+ if (STORE.id != 0) load_store_products(STORE.id);
+
+ // ...........................................................................
+ // ...........................................................................
+ // ................................................................ up to here
+
+
+
+
+ /** 5. SUGGESTIONS ENGINE
+ * -------------------------------------------------------------------------
+ *//////////////////////////////////////////////////////////////////////////
+
+
+ workline.jsonLoaded++; // notify workline that jQuery is ready!!
+
+
+ // suggestions engine //////////////////////////////////////////////////
+ // ---
+ function suggestions_engine(qOrig) {
+ var results = []; // suggestions to respond
+ var proList = []; // list of products (for all suggestions)
+ var commonL = []; // list of common products (for multiple suggestions)
+ var possibleNext = []; // list of possible next suggestions
+
+ var root, last;
+
+ var space_ended = (qOrig.slice(-1) == ' ') ? true : false;
+
+ // clean and sanitize and mark links onto q(uery) string
+ var q = mark_explicit_links( sanitize_GR( clean_text(qOrig) ) );
+
+ // TODO:
+ // construct direct-linked words
+ // = do unequivocally replaces
+ // steps:
+ // 1. replace accented vowels with non accented ones
+ // 2. replace `/some pattern/gi , 'SOME-REPLACE-PATTERN'`
+ // --- -- -- - - -
+ // test:
+ // oneliner: var str = 'αλφα βητα ΑΛΦΑ Βητα world'; var src = 'αλφα βητα'; var reg = new RegExp(src, "gi"); var replacedOnce = str.toLowerCase().replace(reg, 'α-β'); console.log(replacedOnce);
+
+ var qAr = q.split(' '); // split to words
+ if (space_ended) qAr.push(' '); // if space-end existed, push a space to query array
+
+ if (qAr.slice(-1) == "") {
+ qAr.pop();
+ }
+
+ if (options.debug) console.log(qAr);
+
+ if (qAr.length == 1) { // suggest 1st word //////////////////////
+
+ var kbq = kb.keyboardize(q);
+
+ // loop through root-words
+ // ... to match all possible suggestions;
+ _kwlinks.forEach( it => {
+ chk = check_match(kbq, it.w, true);
+
+ if ( chk != false ) {
+ results.push({ w: chk });
+
+ // ... also keep possible products in a list
+ it.c.forEach( wo => { // for every Word-Link-Node
+ if (proList.length < _maxResults + 2) { // NOTE: about +2 (bellow)
+
+ // concatenate this-suggestion's product sub-list (wo.p)
+ // to all-suggestions posiible products-list (proList)
+ proList = proList.concat(wo.p);
+ // keep unique products in the products-list
+ proList = proList.filter((item, i, ar) => ar.indexOf(item) === i);
+ }
+ });
+ }
+
+ });
+ }
+
+ if (qAr.length == 2) { // suggest 2nd word /////////////////////////
+ root = qAr[0];
+ last = qAr[1];
+ kbroot = kb.keyboardize(root);
+ kblast = kb.keyboardize(last);
+
+ // TODO:
+ // change [_kwlinks.forEach] to for loop
+ // ---
+ _kwlinks.forEach( it => { // locate the ...
+ if (match.exact(kbroot, kb.keyb_array(it.w))) { // exact match of root-word
+
+ it.c.forEach ( wo => { // loop the word-links ...
+ chk = check_match(kblast, wo.w, true);
+ if ( chk !== false ) { // if a match is found
+ results.push({ // keep suggestion
+ w: root +' '+ chk,
+ f: wo.f
+ });
+ if (proList.length < _maxResults + 2) { // plus...
+ proList = proList.concat(wo.p); // keep products-list
+ proList = proList.filter((item, i, ar) => ar.indexOf(item) === i);
+ }
+ }
+ });
+
+ }
+ });
+ }
+
+ if (qAr.length > 2) { // suggest N-th word (N>2) ///////////////
+ root = qAr.shift(); // isolate first item of qAr
+ last = qAr.pop(); // isolate last item too
+ // now qAr includes only the items after root and before last;
+ // so qAr includes all already selected suggestions (but root)
+
+ // prepare/cache kb-formated string for any key we're going to use
+ var kb_qAr = []; // array of selected suggestions in kb-format
+ qAr.forEach( w => { kb_qAr.push(keyboardize(w)); })
+ kbroot = keyboardize(root);
+ kblast = keyboardize(last);
+
+ // TODO:
+ // change [_kwlinks.forEach] to for loop
+ // ---
+ _kwlinks.forEach( it => {
+ if (is_exact_match(kbroot, it.w)) { // locate root-word
+
+ // calculate list of common items/products (commonL)
+ // for selected suggestions
+ is1stOcc = true; // 1st occurance flag
+
+ it.c.forEach ( swo => {
+ if (match_one(kb_qAr, swo.w)) {
+ // swo is one of the already selected suggestions
+ // so... update the common-(products)-L(ist)
+ if (is1stOcc) {
+ commonL = swo.p; // init list (on 1st occurance)
+ is1stOcc = false;
+ }
+ else {
+ // caclulate list of common products
+ // = intersection of (so-far) commonL and swo.p
+ // commonL = commonL.filter(value => swo.p.includes(value));
+ commonL = commonL.filter(function(n) { return swo.p.indexOf(n) !== -1; });
+ }
+ }
+ else { // if swo is not already selected
+ // then This is a possible NEXT suggestion
+ possibleNext.push(swo);
+ }
+ });
+
+ // Now that we have all the possible next suggestions
+ // we'll match them with the last word of the query
+
+ possibleNext.forEach( poss => { // for tthe possible next suggestions
+ // if last-word-of-query matches possible word(s)
+ // and list of word's products has commons with commonL
+ // then THiS is a Valid-Next-Suggestion
+ chk = check_match(kblast, poss.w, true);
+ if (chk !== false) {
+ // check intersection of commonL and suggestion's product-lists
+ tempL = commonL.filter(value => poss.p.includes(value));
+ if (tempL.length > 0) {
+ results.push({
+ w: root +' '+ qAr.join(' ') +' '+ chk,
+ f: poss.f
+ });
+ // update proList too
+ proList = proList.concat(tempL);
+ }
+ }
+ });
+ }
+
+ });
+ }
+
+ // NOTE: about +2 (vs +1)
+ // after having calculated next suggestions and a banch of possible products
+ // the proccedure is going to decide what data will return;
+ // if pro(ducts)List includes less items than maximum suggestions
+ // ... this will be the array to return. So
+ // ... +2 ensures that this list will not become shorter than this limit
+
+ // calculate unique products
+ // (credit: https://stackoverflow.com/questions/11246758/)
+ let unique = proList.filter((item, i, ar) => ar.indexOf(item) === i);
+
+
+
+ if (unique.length < (_maxResults +1)) { // IF list is small ......
+
+ // serve products instead of suggestions
+ // ...
+ results = [];
+ notOnThisStore = []
+ unique.forEach( pr => {
+ for (i=0 ; i< _products.length-1 ; i++) { // for makes things faster
+ pi = _products[i];
+ if (pi.id == pr) {
+ results.push(pi);
+ break;
+ }
+ }
+ });
+
+ } else { // remove forced link character '-' from suggestions .....
+ var dirty_results = results;
+ results = [];
+ for (i=0 ; i< dirty_results.length ; i++) {
+ results.push({ w: dirty_results[i].w.replaceAll('-', ' ') })
+ }
+ }
+
+ return results;
+ }
+
+ // simple, fast search products by numeric code property
+ // ---
+ function search_by_code( num ) {
+ const str = num.toString();
+ var results = [];
+ var p;
+ for (i=0 ; i< _products.length-1 ; i++) {
+ p = _products[i];
+ if (results.length > _maxResults) { break; }
+ if ( (p.bp+'-'+p.bc).indexOf(str) !== -1 ) { results.push(p) }
+ }
+ return results;
+ }
+
+ // suggestions caller (router function)
+ // arguments:
+ // * qOrig : original query string
+ // ** list : callback array structure to host results
+ // ---
+ var isuggest = function(qOrig, list) {
+ var results;
+
+ qSanit = sanitize_GR(qOrig); // sanitize greel accended chars
+
+ // TODO:
+ // check if all source-lists are ready
+ // if not you need to wait ...
+ // via async promishes or synced timouts
+
+ // if query seems to be some king of 'code/id'
+ if (qSanit.length>2 && qSanit.match(/^[0-9]+$/) != null) {
+ results = search_by_code(qOrig);
+
+ } else {
+ // string match procedure with suggestions engine
+ results = suggestions_engine(qOrig);
+
+ }
+ list(results);
+ }
+
+ /**
+ jQuery(function() { // on document ready code //////////////////////////
+ /**
+
+ // UI-dependent code
+ // uses reference to specific document element (passed via options)
+ // -----------------------------------------------------------------------
+ const searchBox = $(options.search_tag);
+ searchBox.typeahead(
+ {
+ hint: true,
+ highlight: true,
+ minLength: 1
+ },
+ {
+ limit: _maxResults,
+ name: 'kwlinks',
+ displayKey: 'w',
+ source: isuggest,
+ templates: {
+ suggestion: function(data) {
+ if (data.id)
+ return '<div data-id="'+ data.id +'">'+ data.w +'</div>';
+ return '<div>'+ data.w +'<span class="icon icon-arrowRight--b"></span></div>';
+ },
+ empty: '<div class="-empty-">Δεν υπάρχει στο κωδικολόγιο του καταστήματος</div>'
+ }
+ }
+ )
+ .bind("typeahead:selected", function(obj, datum, name) {
+ if (datum.hasOwnProperty('id')) { // ** selected: PRODUCT
+
+ // $('#js-add-product-to-order').attr('disabled', false);
+ // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img);
+ if (options.debug) console.log('selected: ', datum.id, datum.w);
+
+ // TODO: GOTO product page
+ window.location.href = 'product_list?product=id-' + datum.id;
+
+ // $("#product-quantity").trigger('focus');
+
+ }
+ else { // selected: SUGGESTION
+
+ $('.typeahead').typeahead('val','').trigger('blur');
+ $('.typeahead').typeahead('val', datum.w +' ')
+ .trigger("query");
+ // give some time to the engine to calculate results
+ // then fire focus again...
+ setTimeout(() => { $('.typeahead').trigger('focus'); }, _timeout_ms);
+ }
+
+ })
+ .bind("typeahead:cursorchange", function( event, obj) {
+ // track cursor-chane to handle special keys after a final product is selected
+
+ // TODO:
+ // handle exception where obj is undefined;
+ // this occures...
+ // when cusror returns from suggestions list back to the search field
+
+ if (typeof obj === 'undefined') cursor_on = { none: true };
+ else if (obj.hasOwnProperty('id')) cursor_on = obj;
+ else cursor_on = { none: true };
+ });
+
+ // TRACK user search attempt ///////////////////////////////////////////////
+ $('.typeahead').on('keyup', function(e) {
+ if (_isReady) {
+ // console.log('on:', cursor_on);
+ // console.log('key:', e.key);
+ if ((e.key == ' ') && cursor_on.hasOwnProperty('id')) {
+
+ // product is actually selected
+ var datum = cursor_on;
+
+ // fill_fields(datum.id, datum.sc, datum.x, datum.w, 1, datum.bc, datum.eu, datum.stk, datum.img);
+ $("#product-quantity").trigger('focus');
+ e.preventDefault();
+
+ } else {
+ if (e.key === "Enter") {
+
+ // if curson is not on some option
+ if ((typeof curson_on === 'undefined') || (curson_on.none == true)) {
+
+ // do a common search
+ if (options.debug) console.log('Do a Non-Suggestions search', searchBox.val());
+
+ // DEPRICATED: var location = encodeURI('/product_list?productSearch=%'+ searchBox.val() +'%');
+
+ search_results = common_search(searchBox.val());
+
+
+ } else {
+ // launch a product page
+ var location = '/product_list?product=id-'+ cursor_on.id;
+ if (options.debug) console.log('product', cursor_on.id, cursor_on.w);
+
+ console.log('go to product', location)
+ }
+ // window.location.href = location;
+ }
+ }
+ }
+ });
+
+ });
+ */
+
+
+ /*
+ function common_search(query) {
+ // clear ; sanitize ; split
+ var qAr = keyboardize( sanitize_GR( clean_text(query) ) ).toLowerCase().split(' ');
+
+ // if last item is empty, remove it
+ if ((qAr.slice(-1) == ' ') || (qAr.slice(-1) == '')) qAr.pop()
+
+ var results = _products;
+
+ // for each key fitler results
+ qAr.forEach( key => {
+ results = key_sublist(key, results)
+ });
+
+ // echo products (and prepare list to POST)
+ var list_ = [];
+ results.forEach( item => {
+ if (options.debug) console.log(item.id, ':', item.w);
+ list_.push(item.id)
+ })
+
+ let l = list_.join(',');
+ var url = encodeURI(`${options.visualize_search_results_url}?search=${query}&eys_code=${l}`);
+
+ if ($('.js-categories.selected').length
+ || $('.js-checkout.selected').length
+ ) {
+ url = encodeURI(`api/v1/products?eys_code=${l}&q=${query}`);
+ console.log('common search : fetch results > history.push', url, query);
+ fetchSearchProducts(url, query);
+ window.history.pushState('search', null, '/product_list?search='+query);
+ } else {
+ console.log('common search: search query > location = search')
+ window.location.href = encodeURI(`${options.visualize_search_results_url}?search=${query}`);
+ }
+
+ // Redirect with POST (template)
+ // --- -- -- - - -
+ // var form = $([
+ // `<form action="${url}" method="post">`,
+ // '<textarea name="products_list">',
+ // list_.join(','),
+ // '</textarea>',
+ // '</form>'
+ // ].join(''));
+ // $('body').append(form);
+ // form.submit();
+
+ }
+ */
+
+ /** return from list only items that include 'key'
+ */
+ function key_sublist(key, list) {
+ var result = [];
+ list.forEach( item => {
+ if (item.kb.includes(key)) {
+ result.push(item);
+ }
+ });
+
+ return result;
+ }
+} \ No newline at end of file
diff --git a/app/routes/dev.js b/app/routes/dev.js
new file mode 100644
index 0000000..cd40822
--- /dev/null
+++ b/app/routes/dev.js
@@ -0,0 +1,103 @@
+/**
+ * defines routes
+ * exports router
+ */
+
+const Router = require('koa-router');
+const urler = require('../utils/url-util.js');
+
+const data = require('../pieces/prepare.js');
+
+const bench = require('../benchmark/find.js');
+const matchStr = require('../benchmark/match-str.js');
+
+
+// Prefix all routes with: /items
+const router = new Router();
+
+
+
+// Routes
+
+/* simple route example
+
+ let items = [
+ { id: 100, iname: 'Quartz Analog Wrist Watch', price: 'US $4.99'},
+ { id: 101, iname: 'Leather Peep Pump Heels', price: 'US $33.56'},
+ { id: 102, iname: 'Apple iPod', price: 'US $219.99'},
+ { id: 103, iname: 'Prince Phantom 97P Tennnis Racket', price: 'US $50.00'},
+ ];
+
+ router.get('/items', (ctx, next) => {
+ ctx.body = items;
+ next();
+ });
+
+ router.get('/items/:id', (ctx, next) => {
+ let getCurrentItem = items.filter(function(item) {
+ if (item.id == ctx.params.id) {
+ return true;
+ }
+ });
+ if (getCurrentItem.length) {
+ ctx.body = getCurrentItem[0];
+ } else {
+ ctx.response.status = 404;
+ ctx.body = 'Item Not Found';
+ }
+ next();
+ });
+*/
+
+
+// NOTE: set timeout per route (how-to)
+// https://stackoverflow.com/questions/66634123/how-to-add-individual-timeout-value-per-specific-route-using-node-js
+
+
+// TEST routes
+////////////////////////////////////////////////////////////////////////////////
+
+router.get('/test', (ctx) => { // easy test route
+ // test anything ...
+ ctx.body = { params: urler.struct(ctx.request, ctx.url), ctx: ctx }
+});
+
+router.get('/test/do-data', (ctx) => { // easy test route
+ // test anything ...
+ // data.create(
+ // 'products',
+ // 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json'
+ // );
+ data.load_products(
+ 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json'
+ );
+ ctx.body = { success: true, operation: 'create new data' };
+});
+
+// benchmark routes;
+// shall be removed from production
+////////////////////////////////////////////////////////////////////////////////
+
+/// router.get('/test/json', (ctx, next) => { // easy test route
+/// // test anything ...
+/// ctx.body = { t: + new Date(), p: products };
+/// next();
+/// });
+
+router.get('/bench/find', (ctx) => { // easy test route
+ // test anything ...
+ ctx.body = bench.compare();
+ // next();
+});
+
+router.get('/bench/match/:title', (ctx) => { // easy test route
+ // test anything ... [ query = 'solokata' ]
+ // ctx.body = bench.compare();
+ ctx.body = matchStr.run(ctx.params.title);
+ // next();
+});
+
+
+// export routes
+
+module.exports = router;
diff --git a/app/routes/index.js b/app/routes/index.js
new file mode 100644
index 0000000..e3d6230
--- /dev/null
+++ b/app/routes/index.js
@@ -0,0 +1,26 @@
+/**
+ * routes (index)
+ */
+const Router = require('koa-router');
+
+const router = new Router();
+
+// Require grouped routes
+
+let v1 = require('./v1.js');
+router.use(v1.routes());
+
+let dev_paths = require('./dev.js');
+router.use(dev_paths.routes());
+
+// define default route
+router.get('/', (ctx) => {
+ ctx.body = {
+ success: true,
+ title: 'oseine',
+ description: 'oseine search engine is not elastic',
+ message: 'where are you now?'
+ }
+})
+
+module.exports = router;
diff --git a/app/routes/v1.js b/app/routes/v1.js
new file mode 100644
index 0000000..394dde6
--- /dev/null
+++ b/app/routes/v1.js
@@ -0,0 +1,68 @@
+/**
+ * api v1 routes
+ */
+
+const Router = require('koa-router');
+const prepare = require('../pieces/prepare.js');
+
+
+// Prefix all routes with: /items
+const router = new Router({
+ prefix: '/v1'
+});
+
+
+
+// Routes
+
+router.get('/prepare/:title', (ctx) => {
+ let result = false;
+ switch (ctx.params.title) {
+ case 'products':
+ result = prepare.load_products(
+ 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json'
+ );
+ break;
+
+ default:
+ break;
+
+ }
+ console.log(result);
+ ctx.body = { success: result };
+})
+
+
+router.get('/search', (ctx) => {
+ ctx.body = [];
+});
+
+router.get('/search/:title', (ctx) => {
+ let result = [
+ { id: 10, w: 'Ένα Προϊόν' },
+ { id: 15, w: 'Άλλο Προϊόν' },
+ { id: 20, w: 'Προϊόν 3' },
+ { id: 25, w: 'Προϊόν 4' }
+ ]
+ // console.log(ctx);
+ /// const kb = require('./utils/kb-util.js');
+ /// let words = kb.keyboardize(kb.clean(ctx.params.title)).split(' ');
+ /// ctx.body = {
+ /// params: ctx.params,
+ /// results: words,
+ /// nxt: next
+ /// };
+ ctx.body = result;
+});
+
+
+/** TODO: ?? other routes ??
+ * /////////////////////////////////////////////////////////////////////////////
+ * + /suggest/<some search string>
+ * + /prepare (products, keywords, linked-terms etc)
+ * + /stats
+ */
+
+
+// export routes
+module.exports = router;
diff --git a/app/utils/kb-util.js b/app/utils/kb-util.js
new file mode 100644
index 0000000..aac88f9
--- /dev/null
+++ b/app/utils/kb-util.js
@@ -0,0 +1,88 @@
+/**
+ * fast string manipulation utilities
+ * for bi-lingual (EL/EN) words/phrases
+ * based on the keyboard layout
+ */
+
+// suplamentary arrays (mostly for cache)
+// --- -- -- - - -
+
+var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
+
+var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
+
+var map = new Map();
+for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
+
+
+// cache (=create a global array)
+// of accended to non-accended vowels mapping
+// --- -- -- - - -
+accented_vowels = [];
+[
+ 'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ΐ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω',
+ 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω'
+].forEach( pair => {
+ ap = pair.split(' ');
+ accented_vowels.push({
+ a: ap[0], // accented
+ p: ap[1] // pure = non accended
+ });
+});
+
+
+// translates string to keyboard-latin keys
+// (the ones that used whan typing each letter of the word)
+const keyboardize = (str) => {
+ str = str.replace('\'','');
+ var out = '';
+ // [map]'s implementation is 40x faster than [for]'s
+ for (var i=0 ; i< str.length; i++) out += map.get(str[i]);
+ return out;
+}
+
+// keyboardize an array of strings
+const keyb_array = (arr) => {
+ kb_arr = [];
+ arr.forEach( w => {
+ kb_arr.push(keyboardize(w));
+ });
+ return kb_arr;
+}
+
+
+// transforms to lowercase; handles sigma-teliko
+const sanitizeGR = (str) => {
+ str = str.toLowerCase();
+
+ // replace accended vowels with pure ones
+ accented_vowels.forEach( v => {
+ str = str.replaceAll(v.a, v.p);
+ });
+
+ // replace sigma on the end of words
+ str = str + ' ';
+ str = str.replaceAll('σ-', 'ς-');
+ str = str.replaceAll('σ ', 'ς ');
+
+ return str;
+}
+
+// removes non keyword characters [+ . , !] and internal multiple-spaces
+// @param txt (string): product description
+const clean = (txt) => {
+ return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space
+ .replace('!','').replace('\"', '') // remove character
+ .replace(' ',' ').replace(' ',' '); // remove multiple spaces
+}
+
+
+// exports
+// --- -- -- - - -
+
+module.exports = {
+ keyboardize,
+ keyb_array,
+ sanitizeGR,
+ clean
+}; \ No newline at end of file
diff --git a/app/utils/match-util.js b/app/utils/match-util.js
new file mode 100644
index 0000000..5e8c648
--- /dev/null
+++ b/app/utils/match-util.js
@@ -0,0 +1,154 @@
+/**
+ * match utility;
+ * includes fuzzy and partial match functions too;
+ * many of them return a match-rate
+ */
+
+
+// fuzzy match
+////////////////////////////////////////////////////////////////////////////////
+
+/** Ngram fuzzy match algorithm
+ * (simple and fast)
+ */
+const createNgram = (word, n) => { // Ngram creation
+ if (word.length <3) return word;
+ const vector = [];
+ for (let i = 0; i < word.length-n+1; ++i) {
+ vector.push(word.slice(i, i + n));
+ }
+ return vector;
+};
+
+/** similarity
+ * rates similarity between 2 words
+ * based on Ngram matches of N = n letters;
+ * implements a 2-dim check (all a-Ngrams vs all all b-Ngrams)
+ *
+ * @param {string} a : first word
+ * @param {string} b : second word
+ * @param {int} n : Ngram base
+ * @returns {float} : match percentage as a float in [0, 1]
+ */
+const similarity = (a, b, n) => { // Ngram match score
+ if (a.length > 0 && b.length > 0) {
+ const aNgram = createNgram(a, n);
+ const bNgram = createNgram(b, n);
+ let hits = 0;
+ for (let x = 0; x < aNgram.length; ++x) {
+ for (let y = 0; y < bNgram.length; ++y) {
+ if (aNgram[x] === bNgram[y]) {
+ hits += 1;
+ }
+ }
+ }
+ if (hits > 0) {
+ const union = aNgram.length + bNgram.length;
+ return (2.0 * hits) / union;
+ }
+ }
+ return 0;
+};
+
+/** resemblance
+ * is an alternative similarity rating;
+ * implements an 1-dim Ngram similarity check
+ * and it's much faster than similarity()
+ */
+const resemblance = (a, b, n) => {
+ if (a.length > n && b.length >= a.length) {
+ const aNgram = createNgram(a, n);
+ let hits = 0;
+ for (let i = 0; i < aNgram.length; ++i) {
+ if (b.includes(aNgram[i])) {
+ hits++;
+ }
+ }
+ if (hits > 0) {
+ // rate resemblance based on hits and length-similarity
+ return (hits / aNgram.length) * (a.length / b.length);
+ }
+ }
+ return 0;
+}
+
+
+// exact and partial match
+////////////////////////////////////////////////////////////////////////////////
+
+/** is_exact_match
+ * check if a searching string -> query (string/latin in kb-format)
+ * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings)
+ *
+ * @param {string} query: searching string; string/latin in kb-format
+ * @param {array} chkArr: array of synonyms; (array of utf-8/strings)
+ * @return {boolean}: true|false
+ */
+function exact( query, chkArr ) {
+ found = false;
+ chkArr.forEach( w => { if (w == query) found = true });
+ return found;
+}
+
+function partial( query, chkArr ) {
+ found = false;
+ chkArr.forEach( w => { if (w.includes(query)) found = true });
+ return found;
+}
+
+/** is exact match + weight rating
+ * @returns {float} weight rates depth of array when a match is found
+ */
+function weighted_exact( query, chkArr ) {
+ let weight = 0; // closer to left/begin rating
+ let len = chkArr.length;
+ for(let i = 0; i < len ; i++) { // i ~ depth
+ if (chkArr[i] == query) {
+ // weights array depth
+ weight = (len - i + 1.0) / len;
+ break;
+ }
+ }
+ return weight;
+}
+
+
+/** is partial match + weight rating
+ *
+ * @param query (string): searching string; string/latin in kb-format
+ * @param chkArr (array): array of synonyms; (array of utf-8/strings)
+ * @returns {float} weight rates both match position and depth of match
+ *
+ * (*) optimization NOTE:
+ * Given the weight `W` and the depth `i`,
+ * the best weight for next `i` shall be: `(L - (i+1)) / L`
+ * To be imposibbe to have a better weight, should:
+ * W > (L - (i+1)) / L => ... => i > (L - L*W - 1)
+ */
+function weighted_partial( query, chkArr ) {
+ let rate = 0;
+ let weight = 0;
+ let len = chkArr.length;
+ for( let i = 0 ; i < len ; i++ ) {
+ let chk = chkArr[i].indexOf(query)
+ if (chk != -1) {
+ rate = (len - i) / (len + 2.0 * chk);
+ weight = rate > weight ? rate : weight;
+ }
+ if (i > (len - len * weight - 1)) {
+ break; // better rating is not possible (*)
+ }
+ }
+ return weight;
+}
+
+
+// exports
+module.exports = {
+ exact,
+ partial,
+ weighted_exact,
+ weighted_partial,
+ similarity,
+ resemblance
+}
diff --git a/app/utils/mem-usage.js b/app/utils/mem-usage.js
new file mode 100644
index 0000000..766a93e
--- /dev/null
+++ b/app/utils/mem-usage.js
@@ -0,0 +1,20 @@
+/**
+ * memory usage report utility
+ */
+
+const formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
+
+function report() {
+ let memoryData = process.memoryUsage();
+
+ let memoryUsage = {
+ rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`,
+ heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`,
+ heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`,
+ external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`,
+ };
+
+ console.log(memoryUsage);
+}
+
+module.exports = { report }
diff --git a/app/utils/url-util.js b/app/utils/url-util.js
new file mode 100644
index 0000000..e86ad9b
--- /dev/null
+++ b/app/utils/url-util.js
@@ -0,0 +1,23 @@
+/**
+ * url utility
+ */
+
+const querystring = require('querystring');
+
+function struct(req, url) {
+ let url_parts = url.split('?');
+ let query = (url_parts.length > 1)
+ ? querystring.decode(url_parts[1])
+ : {};
+ return {
+ method: req.method,
+ host: req.host,
+ path: url_parts[0],
+ query: query
+ }
+}
+
+/**
+ * exports
+ */
+module.exports = { struct } \ No newline at end of file