diff options
Diffstat (limited to 'python')
| -rw-r--r-- | python/products-dict-v5.py | 172 |
1 files changed, 101 insertions, 71 deletions
diff --git a/python/products-dict-v5.py b/python/products-dict-v5.py index 329a212..8d32586 100644 --- a/python/products-dict-v5.py +++ b/python/products-dict-v5.py @@ -11,6 +11,7 @@ import re # regex import json # json import os.path # ... import sys +from urllib.request import urlopen @@ -251,6 +252,10 @@ for it in synonymOriginals : ## Read data # ////////////////////////////////////////////////////////////////////////////// + +##### DEPRICATED +##### read data straight from the database +## ## # enter your ## HOST = "mariadb" # server IP address/domain name ## DATABASE = "emarket_laravel" # database name @@ -280,47 +285,61 @@ for it in synonymOriginals : ## AND c.isActive AND c.IsCurrentlyActive = 1; ## ''' ## results_ = get_data_from_db(crs, query) +## +## --- +## +## # enter your +## HOST = "127.0.0.1" # server IP address/domain name +## DATABASE = "emarket_laravel_dev" # database name +## USER = "emarket_laravel" +## PASSWORD = "SzvRYl4Y0XU9JXVc" +## DB_SOCKET='/cloudsql/pythia-251711:europe-west4:pythia-db-eu' +## +## # connect to MySQL server +## _dbc = mysql.connect( +## host=HOST, +## database=DATABASE, +## user=USER, +## password=PASSWORD, +## use_unicode=True, +## charset='utf8' +## ) +## print("Connected to:", _dbc.get_server_info()) +## +## +## # execute SQL to get all data you need +## crs = _dbc.cursor() +## query = ''' +## SELECT count(pl.eys_code) as FREQuency, +## pl.product_id as product_id, +## pb.brand_name, +## pl.barcode, pl.skl_code, pl.eys_code, +## IF( pd.description IS NOT NULL , pd.description , pl.product_description) AS product_description +## FROM product_list as pl +## LEFT JOIN delivery_orders_products AS dop ON dop.product_id = pl.eys_code +## LEFT JOIN delivery_orders AS do ON dop.order_id = do.id +## LEFT JOIN product_list AS replacement ON dop.replacement_for = replacement.eys_code +## LEFT JOIN product_details AS pd ON pl.eys_code = pd.eys_code +## INNER JOIN product_brands pb ON pl.brand_id = pb.id +## WHERE pl.active = 1 AND pl.sap_code IS NOT NULL +## GROUP BY pl.product_id +## ORDER BY FREQuency DESC +## ''' +## results_ = get_data_from_db(crs, query) -# enter your -HOST = "127.0.0.1" # server IP address/domain name -DATABASE = "emarket_laravel_dev" # database name -USER = "emarket_laravel" -PASSWORD = "SzvRYl4Y0XU9JXVc" -DB_SOCKET='/cloudsql/pythia-251711:europe-west4:pythia-db-eu' - -# connect to MySQL server -_dbc = mysql.connect( - host=HOST, - database=DATABASE, - user=USER, - password=PASSWORD, - use_unicode=True, - charset='utf8' - ) -print("Connected to:", _dbc.get_server_info()) - -sys.exit() - -# execute SQL to get all data you need -crs = _dbc.cursor() -query = ''' - SELECT count(pl.eys_code) as FREQuency, - pl.product_id as product_id, - pb.brand_name, - pl.barcode, pl.skl_code, pl.eys_code, - IF( pd.description IS NOT NULL , pd.description , pl.product_description) AS product_description - FROM product_list as pl - LEFT JOIN delivery_orders_products AS dop ON dop.product_id = pl.eys_code - LEFT JOIN delivery_orders AS do ON dop.order_id = do.id - LEFT JOIN product_list AS replacement ON dop.replacement_for = replacement.eys_code - LEFT JOIN product_details AS pd ON pl.eys_code = pd.eys_code - INNER JOIN product_brands pb ON pl.brand_id = pb.id - WHERE pl.active = 1 AND pl.sap_code IS NOT NULL - GROUP BY pl.product_id - ORDER BY FREQuency DESC -''' -results_ = get_data_from_db(crs, query) +##### NEW +## ----------------------------------------------------------------------------- +## get data from endpoint (url) + + +url = "http://localhost/api/v1/productsSearch" +json_url = urlopen(url) + +received = json.loads(json_url.read()) + +# print(data) +# sys.exit() @@ -355,44 +374,55 @@ products_ = [] ## LOOP through the rows to pre-proccess all products ## --- -for row in results_ : +## for row in results_ : +for rec in received['data'] : - description = row[_COL['product_title']] # product description - pid = domeInt( row[_COL['SKU']] ) # product-id - fq = domeInt( 1 ) # frequency - # url = '/'+ row[_COL['path']] +'/'+ row[_COL['seoUrl']] # product url + ### description = row[_COL['product_title']] # product description + ### pid = domeInt( row[_COL['SKU']] ) # product-id + ### fq = domeInt( 1 ) # frequency + ### # url = '/'+ row[_COL['path']] +'/'+ row[_COL['seoUrl']] # product url - # setup product + + ## process only products with images # --- - products_.append({ - 't' : description, - 'i' : pid, - 'f' : fq - # 'u' : url - }) + if rec['img'] == 1 : + + description = rec['txt'] + pid = ['id'] + fq = 1 + + + # setup product + # --- + products_.append({ + 't' : description, + 'i' : pid, + 'f' : fq + # 'u' : url + }) - # TODO: - # identify brands - # then ... + # TODO: + # identify brands + # then ... - description = cleanText(description) # clean description string before spliting + description = cleanText(description) # clean description string before spliting - keys = [] # list of product's key(word)s - words = description.split() # split to words - for w in words : - if kbLatinString(w) not in removeList: # if not in removeList - if isSignificant(w) : # and if significant - keys.append(w) # keep it + keys = [] # list of product's key(word)s + words = description.split() # split to words + for w in words : + if kbLatinString(w) not in removeList: # if not in removeList + if isSignificant(w) : # and if significant + keys.append(w) # keep it - # print(pid, description, words, keys) + # print(pid, description, words, keys) - # append words (and their combos) to the list - for w in keys : - rootKey( w, fq, keywords_ ) - for w2 in keys : - if w2 != w and isSignificant(w2) : - connectKeys( w, w2, pid, fq, keywords_ ) + # append words (and their combos) to the list + for w in keys : + rootKey( w, fq, keywords_ ) + for w2 in keys : + if w2 != w and isSignificant(w2) : + connectKeys( w, w2, pid, fq, keywords_ ) ## SORT keywords @@ -418,11 +448,11 @@ for it in keywords_ : ## OUTPUT final data to a json-format file # ////////////////////////////////////////////////////////////////////////////// -with open("results/keywords-v5.json", "w", encoding="utf-8") as outfile : +with open("./keywords-v5.json", "w", encoding="utf-8") as outfile : data = json.dump(keywords_, outfile, sort_keys=False, indent=3, ensure_ascii=False) -with open("results/minilist-v5.json", "w", encoding="utf-8") as outfile : +with open("./minilist-v5.json", "w", encoding="utf-8") as outfile : data = json.dump(minilist_, outfile, sort_keys=False, indent=3, ensure_ascii=False) -with open("results/products-v5.json", "w", encoding="utf-8") as outfile : +with open("./products-v5.json", "w", encoding="utf-8") as outfile : data = json.dump(products_, outfile, sort_keys=False, indent=3, ensure_ascii=False)
\ No newline at end of file |