pieces/prepare-streams.js


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

const fs = require("fs");
// const https = require("https");
var request = require('request-promise');

var calls = [
    request({
        url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-keywords.json',
        // headers: { ... }
    }),
    request({
        url: 'https://storage.googleapis.com/pythia-files/uploads/json/emarket-products.json',
        // headers: { ... }
    }),
    // + linked-terms and replaces
    // and more..
];

Promise.all(calls).then(function(results) {
  // do something with results[0]
  // do something with results[1]
  // ...
});


// write files into local fs; use them cached

let file = fs.createWriteStream("data.txt");

https.get("https://www.w3.org/TR/PNG/iso_8859-1.txt", response => {
    var stream = response.pipe(file);

    stream.on("finish", function() {
        console.log("done");
    });
});