summaryrefslogtreecommitdiff
path: root/pieces
diff options
context:
space:
mode:
Diffstat (limited to 'pieces')
-rw-r--r--pieces/kb-util.js84
-rw-r--r--pieces/kbutils.js29
-rw-r--r--pieces/match-util.js65
-rw-r--r--pieces/prepare-streams.js36
-rw-r--r--pieces/suggest.js146
5 files changed, 199 insertions, 161 deletions
diff --git a/pieces/kb-util.js b/pieces/kb-util.js
new file mode 100644
index 0000000..5441861
--- /dev/null
+++ b/pieces/kb-util.js
@@ -0,0 +1,84 @@
+// suplamentary arrays (mostly for cache)
+// --- -- -- - - -
+
+var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
+
+var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
+
+var map = new Map();
+for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
+
+
+// cache (=create a global array)
+// of accended to non-accended vowels mapping
+// --- -- -- - - -
+accented_vowels = [];
+[
+ 'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ΐ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω',
+ 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω'
+].forEach( pair => {
+ ap = pair.split(' ');
+ accented_vowels.push({
+ a: ap[0], // accented
+ p: ap[1] // pure = non accended
+ });
+});
+
+
+// translates string to keyboard-latin keys
+// (the ones that used whan typing each letter of the word)
+const keyboardize = (str) => {
+ str = str.replace('\'','');
+ var out = '';
+ // [map]'s implementation is 40x faster than [for]'s
+ for (var i=0 ; i< str.length; i++) out += map.get(str[i]);
+ return out;
+}
+
+// keyboardize an array of strings
+const keyb_array = (arr) => {
+ kb_arr = [];
+ arr.forEach( w => {
+ kb_arr.push(keyboardize(w));
+ });
+ return kb_arr;
+}
+
+
+// transforms to lowercase; handles sigma-teliko
+const sanitizeGR = (str) => {
+ str = str.toLowerCase();
+
+ // replace accended vowels with pure ones
+ accented_vowels.forEach( v => {
+ str = str.replaceAll(v.a, v.p);
+ });
+
+ // replace sigma on the end of words
+ str = str + ' ';
+ str = str.replaceAll('σ-', 'ς-');
+ str = str.replaceAll('σ ', 'ς ');
+
+ return str;
+}
+
+// removes non keyword characters [+ . , !] and internal multiple-spaces
+// @param txt (string): product description
+const clean = (txt) => {
+ return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space
+ .replace('!','').replace('\"', '') // remove character
+ .replace(' ',' ').replace(' ',' '); // remove multiple spaces
+}
+
+
+// exports
+// --- -- -- - - -
+
+module.exports = {
+ map,
+ accented_vowels,
+ keyboardize,
+ keyb_array,
+ sanitizeGR,
+ clean
+}; \ No newline at end of file
diff --git a/pieces/kbutils.js b/pieces/kbutils.js
deleted file mode 100644
index bef37d4..0000000
--- a/pieces/kbutils.js
+++ /dev/null
@@ -1,29 +0,0 @@
-/** keyboardize
- * -------------------------------------------------------------------------
- * translates string to keyboard-latin keys
- * (the ones that used whan typing each letter of the word)
- *
- * @param str (string): original string (utf8 of latin or greek subgroups)
- * @return (string): latin/ascii equivalent string
- */
-
-ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
-
-kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
-
-map = new Map();
-for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
-
-exports.keybutil = {
-
- map: map,
-
- keyboardize: function(str) {
- str = str.replace('\'','');
- var out = '';
- // [map]'s implementation is 40x faster than [for]'s
- for (var i=0 ; i< str.length; i++) out += map.get(str[i]);
- return out;
- }
-
-}; \ No newline at end of file
diff --git a/pieces/match-util.js b/pieces/match-util.js
new file mode 100644
index 0000000..1841d15
--- /dev/null
+++ b/pieces/match-util.js
@@ -0,0 +1,65 @@
+/** Ngram fuzzy match algorithm
+ * (simple and fast)
+ */
+const createNgram = (word, n) => { // Ngram creation
+ if (word.length <3) return word;
+ const vector = [];
+ for (let i = 0; i < word.length-n+1; ++i) {
+ vector.push(word.slice(i, i + n));
+ }
+ return vector;
+};
+
+/** check similarity between 2 words
+ * based on Ngram matches of N = n letters
+ * @param {string} a : first word
+ * @param {string} b : second word
+ * @param {int} n : Ngram base
+ * @returns {float} : match percentage as a float in [0, 1]
+ */
+const similarity = (a, b, n) => { // Ngram match score
+ if (a.length > 0 && b.length > 0) {
+ const aNgram = createNgram(a, n);
+ const bNgram = createNgram(b, n);
+ let hits = 0;
+ for (let x = 0; x < aNgram.length; ++x) {
+ for (let y = 0; y < bNgram.length; ++y) {
+ if (aNgram[x] === bNgram[y]) {
+ hits += 1;
+ }
+ }
+ }
+ if (hits > 0) {
+ const union = aNgram.length + bNgram.length;
+ return (2.0 * hits) / union;
+ }
+ }
+ return 0;
+};
+
+/** is_exact_match
+ *
+ * check if a searching string -> query (string/latin in kb-format)
+ * matches exactly an item of the array of synonyms -> chkArr (array of utf-8/strings)
+ *
+ * @param query (string): searching string; string/latin in kb-format
+ * @param chkArr (array): array of synonyms; (array of utf-8/strings)
+ * @return (boolean): true|false
+ */
+function exact( query, chkArr ) {
+ found = false;
+ chkArr.forEach( w => { if (w == query) found = true });
+ return found;
+}
+
+function partial( query, chkArr ) {
+ found = false;
+ chkArr.forEach( w => { if (w.includes(query)) found = true });
+ return found;
+}
+
+module.exports = {
+ exact,
+ partial,
+ similarity,
+}; \ No newline at end of file
diff --git a/pieces/prepare-streams.js b/pieces/prepare-streams.js
new file mode 100644
index 0000000..0e10e27
--- /dev/null
+++ b/pieces/prepare-streams.js
@@ -0,0 +1,36 @@
+const fs = require("fs");
+// const https = require("https");
+var request = require('request-promise');
+
+var calls = [
+ request({
+ url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json',
+ // headers: { ... }
+ }),
+ request({
+ url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json',
+ // headers: { ... }
+ }),
+ // + linked-terms and replaces
+ // and more..
+];
+
+Promise.all(calls).then(function(results) {
+ // do something with results[0]
+ // do something with results[1]
+ // ...
+});
+
+
+
+// write files into local fs; use them cached
+
+let file = fs.createWriteStream("data.txt");
+
+https.get("https://www.w3.org/TR/PNG/iso_8859-1.txt", response => {
+ var stream = response.pipe(file);
+
+ stream.on("finish", function() {
+ console.log("done");
+ });
+}); \ No newline at end of file
diff --git a/pieces/suggest.js b/pieces/suggest.js
index 9c37efb..ab62cfd 100644
--- a/pieces/suggest.js
+++ b/pieces/suggest.js
@@ -1,3 +1,8 @@
+const kb = require('./kb-util.js');
+const match = require('./match-util.js');
+
+var n = 2; // Ngram base
+
/** (Search) SUGGESTIONS ENGINE
* ---------------------------------------------------------------------------
*
@@ -79,97 +84,8 @@ function suggestions_module(options) {
var keywordsURL = options.keywords_json;
var cursor_on = { none: true }; // what product is highlighted; if not on product then { none: true }
-
-
-
- /** 2.PURIFY STRING FUNCTIONS
- * -------------------------------------------------------------------------
- *//////////////////////////////////////////////////////////////////////////
-
-
- /** keyboardize
- * -------------------------------------------------------------------------
- * translates string to keyboard-latin keys
- * (the ones that used whan typing each letter of the word)
- *
- * @param str (string): original string (utf8 of latin or greek subgroups)
- * @return (string): latin/ascii equivalent string
- */
-
- // cache (keeo in global) any-character to keyboard-latin mapping
- // --- -- -- - - -
- var ORiGiNal = 'ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
- var kbKeyZed = 'sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnmqwertyuiopasdfghjklzxcvbnm0123456789- '.split('');
- const map = new Map();
- for (var i=0; i<ORiGiNal.length; i++) map.set(ORiGiNal[i], kbKeyZed[i]);
-
- // "keyboardize" function
- // --- -- -- - - -
- function keyboardize(str) {
- str = str.replace('\'','');
- var out = '';
- // [map]'s implementation is 40x faster than [for]'s
- for (var i=0 ; i< str.length; i++) out += map.get(str[i]);
- return out;
- }
-
- /** sanitize_GR
- * -------------------------------------------------------------------------
- * replaces greek accended vowels with non accended ones
- * takes care of sigma on the end of words
- *
- * @param str (string)
- * @return sanitized string
- */
-
- // first cache (=create a global array)
- // of accended to non-accended vowels mapping
- // --- -- -- - - -
- accented_vowels = [];
- [
- 'ά α', 'έ ε', 'ή η', 'ί ι', 'ϊ ι', 'ΐ ι', 'ό ο', 'ύ υ', 'ϋ υ', 'ώ ω',
- 'Ά Α', 'Έ Ε', 'Ή Η', 'Ί Ι', 'Ϊ Ι', 'Ό Ο', 'Ύ Υ', 'Ϋ Υ', 'Ώ Ω'
- ].forEach( pair => {
- ap = pair.split(' ');
- accented_vowels.push({
- a: ap[0], // accented
- p: ap[1] // pure = non accended
- });
- })
-
- // the actual `sanitize_GR` function code
- // --- -- -- - - -
- function sanitize_GR(str) {
-
- str = str.toLowerCase();
-
- // replace accended vowels with pure ones
- accented_vowels.forEach( v => {
- str = str.replaceAll(v.a, v.p);
- });
-
- // replace sigma on the end of words
- str = str + ' ';
- str = str.replaceAll('σ-', 'ς-');
- str = str.replaceAll('σ ', 'ς ');
-
- return str;
- }
-
-
- /** clean text
- * --- -- -- - - -
- * removes non keyword characters [+ . , !] and internal multiple-spaces
- * @param txt (string): product description
- */
- function clean_text(txt) {
- return txt.replace('+',' ').replace('.',' ').replace(',',' ') // change to space
- .replace('!','').replace('\"', '') // remove character
- .replace(' ',' ').replace(' ',' '); // remove multiple spaces
- }
-
-
+
/** replaces (correcting descriptions)
* == construct unequivocally liked words //////////////////////////////////
* -------------------------------------------------------------------------
@@ -262,7 +178,7 @@ function suggestions_module(options) {
* mark linked words shall be handled as one-(key)word
* also, edit common mistakes with suggested replaces
*
- * @param str
+ * @param str
* @return
*/
function mark_explicit_links(str) {
@@ -306,41 +222,6 @@ function suggestions_module(options) {
}
- /** Ngram fuzzy match algorithm
- * (simple and fast)
- */
- const createNgram = (word, n) => { // Ngram creation
- if (word.length <3) return word;
- const vector = [];
- for (let i = 0; i < word.length-n+1; ++i) {
- vector.push(word.slice(i, i + n));
- }
- return vector;
- };
- const checkSimilarity = (a, b) => { // Ngram match score
- if (!_allowFuzzy) return 0;
-
- if (a.length > 0 && b.length > 0) {
- const aNgram = createNgram(a, _Ngram_base);
- const bNgram = createNgram(b, _Ngram_base);
- let hits = 0;
- for (let x = 0; x < aNgram.length; ++x) {
- for (let y = 0; y < bNgram.length; ++y) {
- if (aNgram[x] === bNgram[y]) {
- hits += 1;
- }
- }
- }
- if (hits > 0) {
- const union = aNgram.length + bNgram.length;
- return (2.0 * hits) / union;
- }
- }
- return 0;
- };
-
-
-
// Search Endine's match functions
@@ -364,7 +245,8 @@ function suggestions_module(options) {
chkArr.forEach( chk => {
if (!found) {
chk_kb = keyboardize(chk);
- if ( (chk_kb.indexOf( query ) !== -1) || (fuzzy && (checkSimilarity(chk_kb, query) > _fuzzyLimit)) ) {
+ if ( (chk_kb.indexOf( query ) !== -1)
+ || (fuzzy && (match.similarity(chk_kb, query, n) > _fuzzyLimit)) ) {
found = true;
result = chk;
}
@@ -518,7 +400,7 @@ function suggestions_module(options) {
if (qAr.length == 1) { // suggest 1st word //////////////////////
- var kbq = keyboardize(q);
+ var kbq = kb.keyboardize(q);
// loop through root-words
// ... to match all possible suggestions;
@@ -547,14 +429,14 @@ function suggestions_module(options) {
if (qAr.length == 2) { // suggest 2nd word /////////////////////////
root = qAr[0];
last = qAr[1];
- kbroot = keyboardize(root);
- kblast = keyboardize(last);
+ kbroot = kb.keyboardize(root);
+ kblast = kb.keyboardize(last);
// TODO:
// change [_kwlinks.forEach] to for loop
// ---
- _kwlinks.forEach( it => { // locate the ...
- if (is_exact_match(kbroot, it.w)) { // exact match of root-word
+ _kwlinks.forEach( it => { // locate the ...
+ if (match.exact(kbroot, kb.keyb_array(it.w))) { // exact match of root-word
it.c.forEach ( wo => { // loop the word-links ...
chk = check_match(kblast, wo.w, true);
generated by cgit v1.2.3 (git 2.39.1) at 2026-09-22 12:02:09 +0000