diff options
| -rw-r--r-- | app.js | 4 | ||||
| -rw-r--r-- | pieces/prepare.js (renamed from pieces/prepare-streams.js) | 31 | ||||
| -rw-r--r-- | routes/dev.js | 2 | ||||
| -rw-r--r-- | routes/v1.js | 18 | ||||
| -rw-r--r-- | utils/match-util.js | 42 |
5 files changed, 84 insertions, 13 deletions
@@ -2,6 +2,8 @@ const Koa = require('koa'); // const { koaBody } = require('koa-body'); + +// load parameters require('dotenv').config(); // define app @@ -10,7 +12,7 @@ const app = new Koa(); // middleware // app.use(koaBody()); -// Require routes +// load routes let _r = require('./routes'); app.use(_r.routes()).use(_r.allowedMethods()); diff --git a/pieces/prepare-streams.js b/pieces/prepare.js index ee7ba8e..65ccee1 100644 --- a/pieces/prepare-streams.js +++ b/pieces/prepare.js @@ -31,6 +31,8 @@ const kb = require('../utils/kb-util.js'); /** create entity file from url + * used for entities like 'linked-words', 'synonyms', etc. + * * @param {string} entity: use the entity name * @param {string} url: prepared entity json-file in remote server */ @@ -50,6 +52,14 @@ function create(entity, url) { ); } +function get_data_structure() { + // data structure is an array of `{ url:.., expiration:.., path:.. }` objects +} + +function set_data_structure(node) { + +} + function load_products(url) { request(url, @@ -58,6 +68,15 @@ function load_products(url) { // body is a ready json-string; no need to parse and (re-)stringify json = JSON.parse(body); newJson = []; + + // TODO: + // + attach handle synonyms + // + mark brand-names + // + attach category-names and SAP-categories + // + construct combo words + // + remove non-important words + // + normalize popularity + json.forEach( p => { newJson.push({ id: p.id, @@ -65,11 +84,19 @@ function load_products(url) { kb: kb.keyboardize(kb.clean(p.w)) }); }); + try { - fs.writeFileSync(`${__dirname}/../data/products.json`, JSON.stringify(newJson), 'utf-8'); - // file written successfully + + fs.writeFileSync( + `${__dirname}/../data/products.json`, + JSON.stringify(newJson), + 'utf-8' + ); + return true; // file written successfully + } catch (err) { console.error(err); + return false; } } } diff --git a/routes/dev.js b/routes/dev.js index 3f142d0..cd40822 100644 --- a/routes/dev.js +++ b/routes/dev.js @@ -6,7 +6,7 @@ const Router = require('koa-router'); const urler = require('../utils/url-util.js'); -const data = require('../pieces/prepare-streams.js'); +const data = require('../pieces/prepare.js'); const bench = require('../benchmark/find.js'); const matchStr = require('../benchmark/match-str.js'); diff --git a/routes/v1.js b/routes/v1.js index 859f7b0..9690973 100644 --- a/routes/v1.js +++ b/routes/v1.js @@ -3,6 +3,7 @@ */ const Router = require('koa-router'); +const prepare = require('../pieces/prepare.js'); // Prefix all routes with: /items @@ -14,10 +15,25 @@ const router = new Router({ // Routes +router.get('/prepare/:title', (ctx) => { + let result = false; + switch (ctx.params.title) { + case 'products': + result = prepare.load_products( + 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json' + ); + break; + + default: + break; + + } + return { success: result }; +}) router.get('/search', (ctx) => { - ctx.body = { success: true, results: [] }; + ctx.body = []; }); router.get('/search/:title', (ctx) => { diff --git a/utils/match-util.js b/utils/match-util.js index b6b9fc0..5e8c648 100644 --- a/utils/match-util.js +++ b/utils/match-util.js @@ -1,3 +1,13 @@ +/** + * match utility; + * includes fuzzy and partial match functions too; + * many of them return a match-rate + */ + + +// fuzzy match +//////////////////////////////////////////////////////////////////////////////// + /** Ngram fuzzy match algorithm * (simple and fast) */ @@ -63,13 +73,16 @@ const resemblance = (a, b, n) => { } +// exact and partial match +//////////////////////////////////////////////////////////////////////////////// + /** is_exact_match * check if a searching string -> query (string/latin in kb-format) * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings) * - * @param query (string): searching string; string/latin in kb-format - * @param chkArr (array): array of synonyms; (array of utf-8/strings) - * @return (boolean): true|false + * @param {string} query: searching string; string/latin in kb-format + * @param {array} chkArr: array of synonyms; (array of utf-8/strings) + * @return {boolean}: true|false */ function exact( query, chkArr ) { found = false; @@ -89,11 +102,10 @@ function partial( query, chkArr ) { function weighted_exact( query, chkArr ) { let weight = 0; // closer to left/begin rating let len = chkArr.length; - for(let i = 0; i < len ; i++) { + for(let i = 0; i < len ; i++) { // i ~ depth if (chkArr[i] == query) { - // rating weights array depth + // weights array depth weight = (len - i + 1.0) / len; - // console.log(i, weight, query); break; } } @@ -102,22 +114,36 @@ function weighted_exact( query, chkArr ) { /** is partial match + weight rating + * + * @param query (string): searching string; string/latin in kb-format + * @param chkArr (array): array of synonyms; (array of utf-8/strings) * @returns {float} weight rates both match position and depth of match + * + * (*) optimization NOTE: + * Given the weight `W` and the depth `i`, + * the best weight for next `i` shall be: `(L - (i+1)) / L` + * To be imposibbe to have a better weight, should: + * W > (L - (i+1)) / L => ... => i > (L - L*W - 1) */ function weighted_partial( query, chkArr ) { let rate = 0; let weight = 0; let len = chkArr.length; - for(let i = 0; i < len ; i++) { + for( let i = 0 ; i < len ; i++ ) { let chk = chkArr[i].indexOf(query) if (chk != -1) { - rate = (len -i +1.0) / (len + 2.0 * chk); + rate = (len - i) / (len + 2.0 * chk); weight = rate > weight ? rate : weight; } + if (i > (len - len * weight - 1)) { + break; // better rating is not possible (*) + } } return weight; } + +// exports module.exports = { exact, partial, |