new keyword-linking procedure; reads data (json) from endpoint

1 files changed, 101 insertions, 71 deletions

diff --git a/python/products-dict-v5.py b/python/products-dict-v5.py
index 329a212..8d32586 100644
--- a/python/products-dict-v5.py
+++ b/python/products-dict-v5.py
@@ -11,6 +11,7 @@ import re       # regex
 import json     # json
 import os.path  # ...
 import sys
+from urllib.request import urlopen
 
 
 
@@ -251,6 +252,10 @@ for it in synonymOriginals :
 ## Read data
 # //////////////////////////////////////////////////////////////////////////////
 
+
+##### DEPRICATED
+##### read data straight from the database
+##
 ## # enter your 
 ## HOST = "mariadb"              # server IP address/domain name
 ## DATABASE = "emarket_laravel"      # database name
@@ -280,47 +285,61 @@ for it in synonymOriginals :
 ##         AND c.isActive AND c.IsCurrentlyActive = 1;
 ## '''
 ## results_ = get_data_from_db(crs, query)
+##
+## ---
+##
+## # enter your 
+## HOST = "127.0.0.1"              # server IP address/domain name
+## DATABASE = "emarket_laravel_dev"      # database name
+## USER = "emarket_laravel"
+## PASSWORD = "SzvRYl4Y0XU9JXVc"
+## DB_SOCKET='/cloudsql/pythia-251711:europe-west4:pythia-db-eu'
+## 
+## # connect to MySQL server
+## _dbc = mysql.connect(
+##         host=HOST,
+##         database=DATABASE,
+##         user=USER,
+##         password=PASSWORD,
+##         use_unicode=True,
+##         charset='utf8'
+##     )
+## print("Connected to:", _dbc.get_server_info())
+## 
+## 
+## # execute SQL to get all data you need
+## crs = _dbc.cursor()
+## query = '''
+##     SELECT count(pl.eys_code) as FREQuency,
+##         pl.product_id as product_id,
+##         pb.brand_name,
+##         pl.barcode, pl.skl_code, pl.eys_code,
+##         IF( pd.description IS NOT NULL , pd.description , pl.product_description) AS product_description
+##     FROM product_list as pl
+##     LEFT JOIN delivery_orders_products AS dop ON dop.product_id = pl.eys_code
+##     LEFT JOIN delivery_orders AS do ON dop.order_id = do.id
+##     LEFT JOIN product_list AS replacement ON dop.replacement_for = replacement.eys_code
+##     LEFT JOIN product_details AS pd ON pl.eys_code = pd.eys_code
+##     INNER JOIN product_brands pb ON pl.brand_id = pb.id
+##     WHERE pl.active = 1 AND pl.sap_code IS NOT NULL
+##     GROUP BY pl.product_id
+##     ORDER BY FREQuency DESC
+## '''
+## results_ = get_data_from_db(crs, query)
 
 
-# enter your 
-HOST = "127.0.0.1"              # server IP address/domain name
-DATABASE = "emarket_laravel_dev"      # database name
-USER = "emarket_laravel"
-PASSWORD = "SzvRYl4Y0XU9JXVc"
-DB_SOCKET='/cloudsql/pythia-251711:europe-west4:pythia-db-eu'
-
-# connect to MySQL server
-_dbc = mysql.connect(
-        host=HOST,
-        database=DATABASE,
-        user=USER,
-        password=PASSWORD,
-        use_unicode=True,
-        charset='utf8'
-    )
-print("Connected to:", _dbc.get_server_info())
-
-sys.exit()
-
-# execute SQL to get all data you need
-crs = _dbc.cursor()
-query = '''
-    SELECT count(pl.eys_code) as FREQuency,
-        pl.product_id as product_id,
-        pb.brand_name,
-        pl.barcode, pl.skl_code, pl.eys_code,
-        IF( pd.description IS NOT NULL , pd.description , pl.product_description) AS product_description
-    FROM product_list as pl
-    LEFT JOIN delivery_orders_products AS dop ON dop.product_id = pl.eys_code
-    LEFT JOIN delivery_orders AS do ON dop.order_id = do.id
-    LEFT JOIN product_list AS replacement ON dop.replacement_for = replacement.eys_code
-    LEFT JOIN product_details AS pd ON pl.eys_code = pd.eys_code
-    INNER JOIN product_brands pb ON pl.brand_id = pb.id
-    WHERE pl.active = 1 AND pl.sap_code IS NOT NULL
-    GROUP BY pl.product_id
-    ORDER BY FREQuency DESC
-'''
-results_ = get_data_from_db(crs, query)
+##### NEW
+## -----------------------------------------------------------------------------
+## get data from endpoint (url)
+
+
+url = "http://localhost/api/v1/productsSearch"
+json_url = urlopen(url)
+
+received = json.loads(json_url.read())
+
+# print(data)
+# sys.exit()
 
 
 
@@ -355,44 +374,55 @@ products_ = []
 
 ## LOOP through the rows to pre-proccess all products
 ## ---
-for row in results_ :
+## for row in results_ :
+for rec in received['data'] :
 
-    description = row[_COL['product_title']]        # product description
-    pid  = domeInt( row[_COL['SKU']] )      # product-id
-    fq   = domeInt( 1 )     # frequency
-    # url  = '/'+ row[_COL['path']] +'/'+ row[_COL['seoUrl']]     # product url
+    ### description = row[_COL['product_title']]        # product description
+    ### pid  = domeInt( row[_COL['SKU']] )      # product-id
+    ### fq   = domeInt( 1 )     # frequency
+    ### # url  = '/'+ row[_COL['path']] +'/'+ row[_COL['seoUrl']]     # product url
 
-    # setup product
+
+    ## process only products with images
     # ---
-    products_.append({
-        't' : description,
-        'i' : pid,
-        'f' : fq
-        # 'u' : url
-    })
+    if rec['img'] == 1 :
+
+        description = rec['txt']
+        pid = ['id']
+        fq = 1    
+
+
+        # setup product
+        # ---
+        products_.append({
+            't' : description,
+            'i' : pid,
+            'f' : fq
+            # 'u' : url
+        })
 
-    # TODO:
-    # identify brands
-    # then ...
+        # TODO:
+        # identify brands
+        # then ...
 
-    description = cleanText(description)        # clean description string before spliting
+        description = cleanText(description)        # clean description string before spliting
 
-    keys = []                                   # list of product's key(word)s
-    words = description.split()                 # split to words
-    for w in words :
-        if kbLatinString(w) not in removeList:      # if not in removeList            
-            if isSignificant(w) :                   # and if significant
-                keys.append(w)                      # keep it
+        keys = []                                   # list of product's key(word)s
+        words = description.split()                 # split to words
+        for w in words :
+            if kbLatinString(w) not in removeList:      # if not in removeList            
+                if isSignificant(w) :                   # and if significant
+                    keys.append(w)                      # keep it
 
 
-    # print(pid, description, words, keys)
+        # print(pid, description, words, keys)
 
-    # append words (and their combos) to the list
-    for w in keys :
-        rootKey( w, fq, keywords_ )
-        for w2 in keys :
-            if w2 != w and isSignificant(w2) :
-                connectKeys( w, w2, pid, fq, keywords_ )
+        # append words (and their combos) to the list
+        for w in keys :
+            rootKey( w, fq, keywords_ )
+            for w2 in keys :
+                if w2 != w and isSignificant(w2) :
+                    connectKeys( w, w2, pid, fq, keywords_ )
 
 
 ## SORT keywords
@@ -418,11 +448,11 @@ for it in keywords_ :
 ## OUTPUT final data to a json-format file
 # //////////////////////////////////////////////////////////////////////////////
 
-with open("results/keywords-v5.json", "w", encoding="utf-8") as outfile :
+with open("./keywords-v5.json", "w", encoding="utf-8") as outfile :
     data = json.dump(keywords_, outfile, sort_keys=False, indent=3, ensure_ascii=False)
 
-with open("results/minilist-v5.json", "w", encoding="utf-8") as outfile :
+with open("./minilist-v5.json", "w", encoding="utf-8") as outfile :
     data = json.dump(minilist_, outfile, sort_keys=False, indent=3, ensure_ascii=False)
 
-with open("results/products-v5.json", "w", encoding="utf-8") as outfile :
+with open("./products-v5.json", "w", encoding="utf-8") as outfile :
     data = json.dump(products_, outfile, sort_keys=False, indent=3, ensure_ascii=False)
\ No newline at end of file