From c50d4c645cd3c04204106c4f9f026e5910afa3d5 Mon Sep 17 00:00:00 2001 From: Geo Halkiadakis Date: Fri, 17 Mar 2023 13:30:46 +0200 Subject: Code tree reorganized; older implemenatations act as a start point --- python/check-linked.py | 300 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 python/check-linked.py (limited to 'python/check-linked.py') diff --git a/python/check-linked.py b/python/check-linked.py new file mode 100644 index 0000000..2492ac5 --- /dev/null +++ b/python/check-linked.py @@ -0,0 +1,300 @@ +## LIBRARIES +# ////////////////////////////////////////////////////////////////////////////// + +# import pandas as pd # pandas for excel reading +import re # regex +import json # json +import os.path # ... +import datetime + +t0_ = datetime.datetime.now() + + +## LOCAL FUNCTIONS +# ////////////////////////////////////////////////////////////////////////////// + + +## Clean Text ... +# -> removes some general/neutral words and symbols +# -> ignores some in-line characters +# -> also strips spare spaces +# function is applied onto the full title/description +# --- +def cleanText(x) : + ignoreList = '" ( ) [ ]'.split(' ') + + for r in ignoreList : + x = x.replace(r, ' ') + + x = x.replace(' ', ' ') # remove spare spaces + x = x.replace(' ', ' ') + x = x.replace(' ', ' ') + + return x.replace(' ', ' ') # one lase (just in case) + + +# isSignificant +# decides if the term is significant to be indexed; +# a term is significant if does not contain digit-chars [0-9], comma (,) or period (.) +# --- +def isSignificant(x) : + # fisrts exclude some notable exceptions (mostly brands) + if x in ['7UP', '3ΑΛΦΑ', '17'] : + return True + + return not bool( re.match("\S*\d+\S*", x) ) + + +def kbLatinString( txt ) : + maTable = txt.maketrans( + "ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNM", + "sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnm" + ) + txt = txt.replace('\'', '') + return txt.translate(maTable).lower() + + + + +# letters-only translation to key-pressed characters (latin) +# --- +def kbLatinLetter( txt ) : + maTable = txt.maketrans( + "ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫ", + "sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviy" + ) + return txt.translate(maTable).lower() + + +## mark a link to a text +# conecting them with a dash/minus character +# --- +def markLink(lws, text) : + text_kb = kbLatinLetter(text.replace(' ', '-')) + lws_kb = kbLatinLetter(lws) + try: + index_l = text_kb.lower().index(lws_kb.lower()) + except: + return text + else: + return text[:index_l] + lws + text[index_l + len(lws):] + + +### # --- list of normalized word combinations +### replaceWords = [ +### 'HEAD & SHOULDERS; HEAD&SOULDERS', +### 'ΟΛΙΚΗΣ 'ΑΛΕΣΗΣ; Ολικής Άλεσης', +### 'Χωρίς προσθήκη ζάχαρης; Χωρίς-Ζάχαρη' +### ] + + +# --- list of linked-words +linkedWords = [ + 'Χωρίς-Γλουτένη', + 'Χωρίς-Ζάχαρη', + 'Χωρίς-Αλάτι', + 'Χωρίς-Λακτόζη', + 'Χωρίς-Συντηρητικά', + 'Χωρίς-Αλκοόλ', + 'Χωρίς-Kαφεϊνη', + 'Χωρίς-Γλυκάνισο', + 'Χωρίς-Ανθρακικό', + 'Χωρίς-Προσθήκη' + 'Υψηλής-Παστερίωσης', + 'Ολικής-Άλεσης', + 'Ολικής-Aλέσεως', + 'Χαρτί-Υγείας', + 'ρολό-υγείας', + 'χαρτί-τουαλέτας', + 'Χαρτί-Κουζίνας', + 'Μπάρες-Δημητριακών', + 'Μπαρμα-Στάθης', + 'Coca-Cola' + 'Aς-Μαγειρέψουμε', + 'ΚΡΙΣ-ΚΡΙΣ', + 'ΚΡΙ-ΚΡΙ', + 'ΕΛ-ΓΚΡΕΚΟ', + 'FREE-STEP', + 'EL-SABOR', + 'LE-PETIT-MARSEILLAIS', + 'DOUWE-EGBERTS', + 'ΕΝ-ΕΛΛΑΔΙ', + 'SPIN-SPAN', + 'CRETA-FARMS', + 'CRETA-FARM', + 'NES-CAFE', + 'Ολες-τις-Χρήσεις', +] + + +# text after "all-links" marked +# --- +def markLinkedWords(text) : + for lw in linkedWords : + text = markLink( lw, text ) + + return text + + +## PREPADE (or build) exception objects +# ////////////////////////////////////////////////////////////////////////////// + + +# --- list of possible combos to check +_2check_kb = [] +_2check = ['με', 'σε', 'για', 'όλες', 'χωρίς' ] +for it in _2check : + _2check_kb.append(kbLatinString(it)) + +linkedWordsFound = [ + { 'w' : 'se', 'links' : [] }, + { 'w' : 'oles-tis', 'links' : [] }, + { 'w' : 'xvris', 'links' : [] } +] + +def recordLink( parent, child, id ) : + if parent != '' : + for it in linkedWordsFound : + if parent == it['w'] : + is_a_new_combo = True + for li in it['links'] : + if li['w'] == child : + li['p'].append(id) + li['c'] += 1 + is_a_new_combo = False + break + if is_a_new_combo : + it['links'].append({ 'w': child, 'p': [ id ], 'c': 1 }) + + + +# --- list of words to exclude from keywords +# NOTE: +# APPLIED in PER-WORD base -> after spliting description to words +removeList = [] +removeOriginals = 'μας του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ') +for it in removeOriginals : + removeList.append(kbLatinString(it)) + + +# --- list of synonyms +# in fact +synonyms = [ + 'μπίρα μπύρα μπίρες μπύρες', + 'αυγά αβγά αυγό', + 'σίκαλης σικάλεως', + 'ξηρά ξερά', + 'ρολό ρολλό', + 'coca-cola cocacola coke', + 'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας', + 'χαρτί-κουζίνας ρολό-κουζίνας', + 'οινος κρασι', + 'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ', + 'DR-OETKER OETKER', + 'DR.BECKMANN BECKMANN', + 'NES-CAFE NESCAFE', + 'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής', + 'τσίπουρο ρακή', + 'Βρώμη Βρώμης', + 'Φράουλα Φράουλες Φράουλας', + 'Μαλλιά Μαλλιών', + 'Κέικ, Cake', + 'CRETA-FARMS CRETA-FARM', + 'MARSEILLAIS LE-PETIT-MARSEILLAIS', + 'Γαϊδούρας Γαϊδάρου', + 'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ', + 'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ', + 'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ', + 'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ', + 'Ντομάτα Ντομάτας', + 'Ελαφρύ Ελαφρά Light', + 'Εγχώρια Ελληνικό Ελληνικά', + 'τριμμένη τριμμένο', + 'Τόνος Τόνου', + 'Κριθαρένια κρίθινα' +] + + +## Read data +# ////////////////////////////////////////////////////////////////////////////// + + +_file = open ('data/eshop-products.json', "r") # JSON source file +results_ = json.loads(_file.read()) # Reading from file +_file.close() # Closing file + + +t_read = datetime.datetime.now() + +# --- Lists to fill +keywords_ = [] # all data; main exported object +minilist_ = [] +products_ = [] + +## keywords format: +## [ +## { +## w : [ word, word-synonym, ... ], +## kb : = kbLatinString(word) +## f : 150, +## c : [ +## { w: ['fish', 'fishes'], f: 150, p: [122, 254, 907] }, +## { w: ['juice'], f: 50, p: [254, 351] } +## ] +## }, +## ... +## ] +## +## --- index: +## w : words / list of synonyms (str/utf-8) +# kb : ascii-latin-keypoard format of first item of "w" list +## f : frequency (int) +## c : combos / connections (list of objects) +## p : list of product-ids found in specific words-combination (list of int) + + +records_counter = 0 +## LOOP through the rows to pre-proccess all products +## --- +for row in results_ : + records_counter += 1 + + description = row['Title'] # product description + pid = row['ID'] # product-id + fq = row['freq'] # frequency + + description = cleanText(description) # clean description string before spliting + + description = markLinkedWords(description) # ... + + keys = description.split() # split to words = keys + + is_combo_key = False + combo_key = '' + + # append words (and their combos) to the list + for w in keys : + + if kbLatinString(w) in _2check_kb : + combo_key = w + is_combo_key = True + else : + if is_combo_key : + recordLink(kbLatinString(combo_key), kbLatinString(w), pid) + is_combo_key = False + combo_key = '' + +# print(linkedWordsFound) + +# PRINT RESULTS +# --- +for ri in linkedWordsFound : + print('---', ri['w'], ':', len(ri['links'])) + + subtotal = 0 + for li in ri['links'] : + subtotal += li['c'] + + for li in ri['links'] : + ## if li['c'] > 10 or li['c']/subtotal > .2 : + print( ri['w'], li['w'], ' : ', li['c'], ' (', int(li['c']*100/subtotal), '%)' ) -- cgit v1.2.3