1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
const fs = require('fs');
var request = require('request');
const kb = require('../utils/kb-util.js');
// const https = require("https");
/* TODO: ?? parallel read
// var request = require('request-promise');
var calls = [
request({
url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json',
// headers: { ... }
}),
request({
url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json',
// headers: { ... }
}),
// + linked-terms and replaces
// and more..
];
Promise.all(calls).then(function(results) {
// do something with results[0]
// do something with results[1]
// ...
});
*/
/** create entity file from url
* used for entities like 'linked-words', 'synonyms', etc.
*
* @param {string} entity: use the entity name
* @param {string} url: prepared entity json-file in remote server
*/
function create(entity, url) {
request(url,
function (error, response, body) {
if (!error && response.statusCode == 200) {
// body is a ready json-string; no need to parse and (re-)stringify
try {
fs.writeFileSync(`${__dirname}/../data/${entity}.json`, body, 'utf-8');
// file written successfully
} catch (err) {
console.error(err);
}
}
}
);
}
function get_data_structure() {
// data structure is an array of `{ url:.., expiration:.., path:.. }` objects
}
function set_data_structure(node) {
}
function load_products(url) {
request(url,
function (error, response, body) {
if (!error && response.statusCode == 200) {
// body is a ready json-string; no need to parse and (re-)stringify
json = JSON.parse(body);
newJson = [];
// TODO:
// + attach handle synonyms
// + mark brand-names
// + attach category-names and SAP-categories
// + construct combo words
// + remove non-important words
// + normalize popularity
json.forEach( p => {
newJson.push({
id: p.id,
w: p.w,
kb: kb.keyboardize(kb.clean(p.w))
});
});
try {
fs.writeFileSync(
`${__dirname}/../data/products.json`,
JSON.stringify(newJson),
'utf-8'
);
return true; // file written successfully
} catch (err) {
console.error(err);
return false;
}
}
}
);
}
// create data
// testing an implemenatatin of a custom local-fs-cache system
function random() {
var data = [];
for( let i = 0; i < (Math.floor(Math.random() * 100) +15) ; i++) {
data.push({ id: i, x5: i*5 });
}
try {
fs.writeFileSync(__dirname + '/../data/random.json', JSON.stringify(data), 'utf-8');
// file written successfully
} catch (err) {
console.error(err);
}
}
module.exports = {
create,
random,
load_products
}
|
