## LIBRARIES # ////////////////////////////////////////////////////////////////////////////// # import pandas as pd # pandas for excel reading import re # regex import json # json import os.path # ... import datetime t0_ = datetime.datetime.now() ## LOCAL FUNCTIONS # ////////////////////////////////////////////////////////////////////////////// ## Clean Text ... # -> removes some general/neutral words and symbols # -> ignores some in-line characters # -> also strips spare spaces # function is applied onto the full title/description # --- def cleanText(x) : ignoreList = '" ( ) [ ]'.split(' ') for r in ignoreList : x = x.replace(r, ' ') x = x.replace(' ', ' ') # remove spare spaces x = x.replace(' ', ' ') x = x.replace(' ', ' ') return x.replace(' ', ' ') # one lase (just in case) # isSignificant # decides if the term is significant to be indexed; # a term is significant if does not contain digit-chars [0-9], comma (,) or period (.) # --- def isSignificant(x) : # fisrts exclude some notable exceptions (mostly brands) if x in ['7UP', '3ΑΛΦΑ', '17'] : return True return not bool( re.match("\S*\d+\S*", x) ) def kbLatinString( txt ) : maTable = txt.maketrans( "ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNM", "sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnm" ) txt = txt.replace('\'', '') return txt.translate(maTable).lower() # letters-only translation to key-pressed characters (latin) # --- def kbLatinLetter( txt ) : maTable = txt.maketrans( "ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫ", "sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviy" ) return txt.translate(maTable).lower() ## mark a link to a text # conecting them with a dash/minus character # --- def markLink(lws, text) : text_kb = kbLatinLetter(text.replace(' ', '-')) lws_kb = kbLatinLetter(lws) try: index_l = text_kb.lower().index(lws_kb.lower()) except: return text else: return text[:index_l] + lws + text[index_l + len(lws):] ### # --- list of normalized word combinations ### replaceWords = [ ### 'HEAD & SHOULDERS; HEAD&SOULDERS', ### 'ΟΛΙΚΗΣ 'ΑΛΕΣΗΣ; Ολικής Άλεσης', ### 'Χωρίς προσθήκη ζάχαρης; Χωρίς-Ζάχαρη' ### ] # --- list of linked-words linkedWords = [ 'Χωρίς-Γλουτένη', 'Χωρίς-Ζάχαρη', 'Χωρίς-Αλάτι', 'Χωρίς-Λακτόζη', 'Χωρίς-Συντηρητικά', 'Χωρίς-Αλκοόλ', 'Χωρίς-Kαφεϊνη', 'Χωρίς-Γλυκάνισο', 'Χωρίς-Ανθρακικό', 'Χωρίς-Προσθήκη' 'Υψηλής-Παστερίωσης', 'Ολικής-Άλεσης', 'Ολικής-Aλέσεως', 'Χαρτί-Υγείας', 'ρολό-υγείας', 'χαρτί-τουαλέτας', 'Χαρτί-Κουζίνας', 'Μπάρες-Δημητριακών', 'Μπαρμα-Στάθης', 'Coca-Cola' 'Aς-Μαγειρέψουμε', 'ΚΡΙΣ-ΚΡΙΣ', 'ΚΡΙ-ΚΡΙ', 'ΕΛ-ΓΚΡΕΚΟ', 'FREE-STEP', 'EL-SABOR', 'LE-PETIT-MARSEILLAIS', 'DOUWE-EGBERTS', 'ΕΝ-ΕΛΛΑΔΙ', 'SPIN-SPAN', 'CRETA-FARMS', 'CRETA-FARM', 'NES-CAFE', 'Ολες-τις-Χρήσεις', ] # text after "all-links" marked # --- def markLinkedWords(text) : for lw in linkedWords : text = markLink( lw, text ) return text ## PREPADE (or build) exception objects # ////////////////////////////////////////////////////////////////////////////// # --- list of possible combos to check _2check_kb = [] _2check = ['με', 'σε', 'για', 'όλες', 'χωρίς' ] for it in _2check : _2check_kb.append(kbLatinString(it)) linkedWordsFound = [ { 'w' : 'se', 'links' : [] }, { 'w' : 'oles-tis', 'links' : [] }, { 'w' : 'xvris', 'links' : [] } ] def recordLink( parent, child, id ) : if parent != '' : for it in linkedWordsFound : if parent == it['w'] : is_a_new_combo = True for li in it['links'] : if li['w'] == child : li['p'].append(id) li['c'] += 1 is_a_new_combo = False break if is_a_new_combo : it['links'].append({ 'w': child, 'p': [ id ], 'c': 1 }) # --- list of words to exclude from keywords # NOTE: # APPLIED in PER-WORD base -> after spliting description to words removeList = [] removeOriginals = 'μας του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ') for it in removeOriginals : removeList.append(kbLatinString(it)) # --- list of synonyms # in fact synonyms = [ 'μπίρα μπύρα μπίρες μπύρες', 'αυγά αβγά αυγό', 'σίκαλης σικάλεως', 'ξηρά ξερά', 'ρολό ρολλό', 'coca-cola cocacola coke', 'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας', 'χαρτί-κουζίνας ρολό-κουζίνας', 'οινος κρασι', 'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ', 'DR-OETKER OETKER', 'DR.BECKMANN BECKMANN', 'NES-CAFE NESCAFE', 'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής', 'τσίπουρο ρακή', 'Βρώμη Βρώμης', 'Φράουλα Φράουλες Φράουλας', 'Μαλλιά Μαλλιών', 'Κέικ, Cake', 'CRETA-FARMS CRETA-FARM', 'MARSEILLAIS LE-PETIT-MARSEILLAIS', 'Γαϊδούρας Γαϊδάρου', 'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ', 'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ', 'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ', 'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ', 'Ντομάτα Ντομάτας', 'Ελαφρύ Ελαφρά Light', 'Εγχώρια Ελληνικό Ελληνικά', 'τριμμένη τριμμένο', 'Τόνος Τόνου', 'Κριθαρένια κρίθινα' ] ## Read data # ////////////////////////////////////////////////////////////////////////////// _file = open ('data/eshop-products.json', "r") # JSON source file results_ = json.loads(_file.read()) # Reading from file _file.close() # Closing file t_read = datetime.datetime.now() # --- Lists to fill keywords_ = [] # all data; main exported object minilist_ = [] products_ = [] ## keywords format: ## [ ## { ## w : [ word, word-synonym, ... ], ## kb : = kbLatinString(word) ## f : 150, ## c : [ ## { w: ['fish', 'fishes'], f: 150, p: [122, 254, 907] }, ## { w: ['juice'], f: 50, p: [254, 351] } ## ] ## }, ## ... ## ] ## ## --- index: ## w : words / list of synonyms (str/utf-8) # kb : ascii-latin-keypoard format of first item of "w" list ## f : frequency (int) ## c : combos / connections (list of objects) ## p : list of product-ids found in specific words-combination (list of int) records_counter = 0 ## LOOP through the rows to pre-proccess all products ## --- for row in results_ : records_counter += 1 description = row['Title'] # product description pid = row['ID'] # product-id fq = row['freq'] # frequency description = cleanText(description) # clean description string before spliting description = markLinkedWords(description) # ... keys = description.split() # split to words = keys is_combo_key = False combo_key = '' # append words (and their combos) to the list for w in keys : if kbLatinString(w) in _2check_kb : combo_key = w is_combo_key = True else : if is_combo_key : recordLink(kbLatinString(combo_key), kbLatinString(w), pid) is_combo_key = False combo_key = '' # print(linkedWordsFound) # PRINT RESULTS # --- for ri in linkedWordsFound : print('---', ri['w'], ':', len(ri['links'])) subtotal = 0 for li in ri['links'] : subtotal += li['c'] for li in ri['links'] : ## if li['c'] > 10 or li['c']/subtotal > .2 : print( ri['w'], li['w'], ' : ', li['c'], ' (', int(li['c']*100/subtotal), '%)' )