diff options
| -rw-r--r-- | paths.js | 71 | ||||
| -rw-r--r-- | pieces/kb-util.js | 84 | ||||
| -rw-r--r-- | pieces/kbutils.js | 29 | ||||
| -rw-r--r-- | pieces/match-util.js | 65 | ||||
| -rw-r--r-- | pieces/prepare-streams.js | 36 | ||||
| -rw-r--r-- | pieces/suggest.js | 146 |
6 files changed, 240 insertions, 191 deletions
@@ -1,41 +1,45 @@ const Router = require('koa-router'); -const kbu = require('./pieces/kbutils.js'); +const kb = require('./pieces/kb-util.js'); // Prefix all routes with: /items const router = new Router({ //// prefix: '/items' }); -// let items = [ -// { id: 100, iname: 'Quartz Analog Wrist Watch', price: 'US $4.99'}, -// { id: 101, iname: 'Leather Peep Pump Heels', price: 'US $33.56'}, -// { id: 102, iname: 'Apple iPod', price: 'US $219.99'}, -// { id: 103, iname: 'Prince Phantom 97P Tennnis Racket', price: 'US $50.00'}, -// ]; +/* simple route example + let items = [ + { id: 100, iname: 'Quartz Analog Wrist Watch', price: 'US $4.99'}, + { id: 101, iname: 'Leather Peep Pump Heels', price: 'US $33.56'}, + { id: 102, iname: 'Apple iPod', price: 'US $219.99'}, + { id: 103, iname: 'Prince Phantom 97P Tennnis Racket', price: 'US $50.00'}, + ]; + + // Routes + + router.get('/items', (ctx, next) => { + ctx.body = items; + next(); + }); + + router.get('/items/:id', (ctx, next) => { + let getCurrentItem = items.filter(function(item) { + if (item.id == ctx.params.id) { + return true; + } + }); + if (getCurrentItem.length) { + ctx.body = getCurrentItem[0]; + } else { + ctx.response.status = 404; + ctx.body = 'Item Not Found'; + } + next(); + }); +*/ -// Routes -/// router.get('/items', (ctx, next) => { -/// ctx.body = items; -/// next(); -/// }); -/// -/// router.get('/items/:id', (ctx, next) => { -/// let getCurrentItem = items.filter(function(item) { -/// if (item.id == ctx.params.id) { -/// return true; -/// } -/// }); -/// -/// if (getCurrentItem.length) { -/// ctx.body = getCurrentItem[0]; -/// } else { -/// ctx.response.status = 404; -/// ctx.body = 'Item Not Found'; -/// } -/// next(); -/// }); +// Routes router.get('/search', (ctx, next) => { ctx.body = []; @@ -43,7 +47,7 @@ router.get('/search', (ctx, next) => { }); router.get('/search/:title', (ctx, next) => { - let words = kbu.keybutil.keyboardize(ctx.params.title).split(' '); + let words = kb.keyboardize(kb.clean(ctx.params.title)).split(' '); ctx.body = { params: ctx.params, results: words @@ -53,8 +57,15 @@ router.get('/search/:title', (ctx, next) => { /** other routes * + /suggest/<some search string> - * + /create + * + /prepare (products, keywords, linked-terms etc) * + /stats */ +router.get('/test', (ctx, next) => { // easy test route + // test anything ... + let result = { success: true, data: [1, 2, 3] }; + ctx.body = result; + next(); +}); + module.exports = router;
\ No newline at end of file diff --git a/pieces/kb-util.js b/pieces/kb-util.js new file mode 100644 index 0000000..5441861 --- /dev/null +++ b/pieces/kb-util.js @@ -0,0 +1,84 @@ +// suplamentary arrays (mostly for cache) +// --- -- -- - - - + +var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + +var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); + +var map = new Map(); +for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]); + + +// cache (=create a global array) +// of accended to non-accended vowels mapping +// --- -- -- - - - +accented_vowels = []; +[ + 'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ΐ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω', + 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω' +].forEach( pair => { + ap = pair.split(' '); + accented_vowels.push({ + a: ap[0], // accented + p: ap[1] // pure = non accended + }); +}); + + +// translates string to keyboard-latin keys +// (the ones that used whan typing each letter of the word) +const keyboardize = (str) => { + str = str.replace('\'',''); + var out = ''; + // [map]'s implementation is 40x faster than [for]'s + for (var i=0 ; i< str.length; i++) out += map.get(str[i]); + return out; +} + +// keyboardize an array of strings +const keyb_array = (arr) => { + kb_arr = []; + arr.forEach( w => { + kb_arr.push(keyboardize(w)); + }); + return kb_arr; +} + + +// transforms to lowercase; handles sigma-teliko +const sanitizeGR = (str) => { + str = str.toLowerCase(); + + // replace accended vowels with pure ones + accented_vowels.forEach( v => { + str = str.replaceAll(v.a, v.p); + }); + + // replace sigma on the end of words + str = str + ' '; + str = str.replaceAll('σ-', 'ς-'); + str = str.replaceAll('σ ', 'ς '); + + return str; +} + +// removes non keyword characters [+ . , !] and internal multiple-spaces +// @param txt (string): product description +const clean = (txt) => { + return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space + .replace('!','').replace('\"', '') // remove character + .replace(' ',' ').replace(' ',' '); // remove multiple spaces +} + + +// exports +// --- -- -- - - - + +module.exports = { + map, + accented_vowels, + keyboardize, + keyb_array, + sanitizeGR, + clean +};
\ No newline at end of file diff --git a/pieces/kbutils.js b/pieces/kbutils.js deleted file mode 100644 index bef37d4..0000000 --- a/pieces/kbutils.js +++ /dev/null @@ -1,29 +0,0 @@ -/** keyboardize - * ------------------------------------------------------------------------- - * translates string to keyboard-latin keys - * (the ones that used whan typing each letter of the word) - * - * @param str (string): original string (utf8 of latin or greek subgroups) - * @return (string): latin/ascii equivalent string - */ - -ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); - -kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); - -map = new Map(); -for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]); - -exports.keybutil = { - - map: map, - - keyboardize: function(str) { - str = str.replace('\'',''); - var out = ''; - // [map]'s implementation is 40x faster than [for]'s - for (var i=0 ; i< str.length; i++) out += map.get(str[i]); - return out; - } - -};
\ No newline at end of file diff --git a/pieces/match-util.js b/pieces/match-util.js new file mode 100644 index 0000000..1841d15 --- /dev/null +++ b/pieces/match-util.js @@ -0,0 +1,65 @@ +/** Ngram fuzzy match algorithm + * (simple and fast) + */ +const createNgram = (word, n) => { // Ngram creation + if (word.length <3) return word; + const vector = []; + for (let i = 0; i < word.length-n+1; ++i) { + vector.push(word.slice(i, i + n)); + } + return vector; +}; + +/** check similarity between 2 words + * based on Ngram matches of N = n letters + * @param {string} a : first word + * @param {string} b : second word + * @param {int} n : Ngram base + * @returns {float} : match percentage as a float in [0, 1] + */ +const similarity = (a, b, n) => { // Ngram match score + if (a.length > 0 && b.length > 0) { + const aNgram = createNgram(a, n); + const bNgram = createNgram(b, n); + let hits = 0; + for (let x = 0; x < aNgram.length; ++x) { + for (let y = 0; y < bNgram.length; ++y) { + if (aNgram[x] === bNgram[y]) { + hits += 1; + } + } + } + if (hits > 0) { + const union = aNgram.length + bNgram.length; + return (2.0 * hits) / union; + } + } + return 0; +}; + +/** is_exact_match + * + * check if a searching string -> query (string/latin in kb-format) + * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) + * + * @param query (string): searching string; string/latin in kb-format + * @param chkArr (array): array of synonyms; (array of utf-8/strings) + * @return (boolean): true|false + */ +function exact( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (w == query) found = true }); + return found; +} + +function partial( query, chkArr ) { + found = false; + chkArr.forEach( w => { if (w.includes(query)) found = true }); + return found; +} + +module.exports = { + exact, + partial, + similarity, +};
\ No newline at end of file diff --git a/pieces/prepare-streams.js b/pieces/prepare-streams.js new file mode 100644 index 0000000..0e10e27 --- /dev/null +++ b/pieces/prepare-streams.js @@ -0,0 +1,36 @@ +const fs = require("fs"); +// const https = require("https"); +var request = require('request-promise'); + +var calls = [ + request({ + url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json', + // headers: { ... } + }), + request({ + url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json', + // headers: { ... } + }), + // + linked-terms and replaces + // and more.. +]; + +Promise.all(calls).then(function(results) { + // do something with results[0] + // do something with results[1] + // ... +}); + + + +// write files into local fs; use them cached + +let file = fs.createWriteStream("data.txt"); + +https.get("https://www.w3.org/TR/PNG/iso_8859-1.txt", response => { + var stream = response.pipe(file); + + stream.on("finish", function() { + console.log("done"); + }); +});
\ No newline at end of file diff --git a/pieces/suggest.js b/pieces/suggest.js index 9c37efb..ab62cfd 100644 --- a/pieces/suggest.js +++ b/pieces/suggest.js @@ -1,3 +1,8 @@ +const kb = require('./kb-util.js'); +const match = require('./match-util.js'); + +var n = 2; // Ngram base + /** (Search) SUGGESTIONS ENGINE * --------------------------------------------------------------------------- * @@ -79,97 +84,8 @@ function suggestions_module(options) { var keywordsURL = options.keywords_json; var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true } - - - - /** 2.PURIFY STRING FUNCTIONS - * ------------------------------------------------------------------------- - *////////////////////////////////////////////////////////////////////////// - - - /** keyboardize - * ------------------------------------------------------------------------- - * translates string to keyboard-latin keys - * (the ones that used whan typing each letter of the word) - * - * @param str (string): original string (utf8 of latin or greek subgroups) - * @return (string): latin/ascii equivalent string - */ - - // cache (keeo in global) any-character to keyboard-latin mapping - // --- -- -- - - - - var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); - var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split(''); - const map = new Map(); - for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]); - - // "keyboardize" function - // --- -- -- - - - - function keyboardize(str) { - str = str.replace('\'',''); - var out = ''; - // [map]'s implementation is 40x faster than [for]'s - for (var i=0 ; i< str.length; i++) out += map.get(str[i]); - return out; - } - - /** sanitize_GR - * ------------------------------------------------------------------------- - * replaces greek accended vowels with non accended ones - * takes care of sigma on the end of words - * - * @param str (string) - * @return sanitized string - */ - - // first cache (=create a global array) - // of accended to non-accended vowels mapping - // --- -- -- - - - - accented_vowels = []; - [ - 'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ΐ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω', - 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω' - ].forEach( pair => { - ap = pair.split(' '); - accented_vowels.push({ - a: ap[0], // accented - p: ap[1] // pure = non accended - }); - }) - - // the actual `sanitize_GR` function code - // --- -- -- - - - - function sanitize_GR(str) { - - str = str.toLowerCase(); - - // replace accended vowels with pure ones - accented_vowels.forEach( v => { - str = str.replaceAll(v.a, v.p); - }); - - // replace sigma on the end of words - str = str + ' '; - str = str.replaceAll('σ-', 'ς-'); - str = str.replaceAll('σ ', 'ς '); - - return str; - } - - - /** clean text - * --- -- -- - - - - * removes non keyword characters [+ . , !] and internal multiple-spaces - * @param txt (string): product description - */ - function clean_text(txt) { - return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space - .replace('!','').replace('\"', '') // remove character - .replace(' ',' ').replace(' ',' '); // remove multiple spaces - } - - + /** replaces (correcting descriptions) * == construct unequivocally liked words ////////////////////////////////// * ------------------------------------------------------------------------- @@ -262,7 +178,7 @@ function suggestions_module(options) { * mark linked words shall be handled as one-(key)word * also, edit common mistakes with suggested replaces * - * @param str + * @param str * @return */ function mark_explicit_links(str) { @@ -306,41 +222,6 @@ function suggestions_module(options) { } - /** Ngram fuzzy match algorithm - * (simple and fast) - */ - const createNgram = (word, n) => { // Ngram creation - if (word.length <3) return word; - const vector = []; - for (let i = 0; i < word.length-n+1; ++i) { - vector.push(word.slice(i, i + n)); - } - return vector; - }; - const checkSimilarity = (a, b) => { // Ngram match score - if (!_allowFuzzy) return 0; - - if (a.length > 0 && b.length > 0) { - const aNgram = createNgram(a, _Ngram_base); - const bNgram = createNgram(b, _Ngram_base); - let hits = 0; - for (let x = 0; x < aNgram.length; ++x) { - for (let y = 0; y < bNgram.length; ++y) { - if (aNgram[x] === bNgram[y]) { - hits += 1; - } - } - } - if (hits > 0) { - const union = aNgram.length + bNgram.length; - return (2.0 * hits) / union; - } - } - return 0; - }; - - - // Search Endine's match functions @@ -364,7 +245,8 @@ function suggestions_module(options) { chkArr.forEach( chk => { if (!found) { chk_kb = keyboardize(chk); - if ( (chk_kb.indexOf( query ) !== -1) || (fuzzy && (checkSimilarity(chk_kb, query) > _fuzzyLimit)) ) { + if ( (chk_kb.indexOf( query ) !== -1) + || (fuzzy && (match.similarity(chk_kb, query, n) > _fuzzyLimit)) ) { found = true; result = chk; } @@ -518,7 +400,7 @@ function suggestions_module(options) { if (qAr.length == 1) { // suggest 1st word ////////////////////// - var kbq = keyboardize(q); + var kbq = kb.keyboardize(q); // loop through root-words // ... to match all possible suggestions; @@ -547,14 +429,14 @@ function suggestions_module(options) { if (qAr.length == 2) { // suggest 2nd word ///////////////////////// root = qAr[0]; last = qAr[1]; - kbroot = keyboardize(root); - kblast = keyboardize(last); + kbroot = kb.keyboardize(root); + kblast = kb.keyboardize(last); // TODO: // change [_kwlinks.forEach] to for loop // --- - _kwlinks.forEach( it => { // locate the ... - if (is_exact_match(kbroot, it.w)) { // exact match of root-word + _kwlinks.forEach( it => { // locate the ... + if (match.exact(kbroot, kb.keyb_array(it.w))) { // exact match of root-word it.c.forEach ( wo => { // loop the word-links ... chk = check_match(kblast, wo.w, true); |