1 files changed, 300 insertions, 0 deletions
diff --git a/python/check-linked.py b/python/check-linked.py
new file mode 100644
index 0000000..2492ac5
--- /dev/null
+++ b/python/check-linked.py
@@ -0,0 +1,300 @@
+## LIBRARIES
+# //////////////////////////////////////////////////////////////////////////////
+
+# import pandas as pd               # pandas for excel reading
+import re       # regex
+import json     # json
+import os.path  # ...
+import datetime
+
+t0_ = datetime.datetime.now()
+
+
+## LOCAL FUNCTIONS
+# //////////////////////////////////////////////////////////////////////////////
+
+
+## Clean Text ...
+# -> removes some general/neutral words and symbols
+# -> ignores some in-line characters 
+# -> also strips spare spaces
+# function is applied onto the full title/description
+# ---
+def cleanText(x) :
+    ignoreList = '" ( ) [ ]'.split(' ')
+
+    for r in ignoreList :
+        x = x.replace(r, ' ')
+       
+    x = x.replace('    ', ' ')      # remove spare spaces
+    x = x.replace('   ', ' ')
+    x = x.replace('  ', ' ')
+
+    return x.replace('  ', ' ')     # one lase (just in case)
+
+
+# isSignificant
+# decides if the term is significant to be indexed;
+# a term is significant if does not contain digit-chars [0-9], comma (,) or period (.)
+# ---
+def isSignificant(x) :
+    # fisrts exclude some notable exceptions (mostly brands)
+    if x in ['7UP', '3ΑΛΦΑ', '17'] :
+        return True
+
+    return not bool( re.match("\S*\d+\S*", x) )
+
+
+def kbLatinString( txt ) :
+    maTable = txt.maketrans(
+        "ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNM",
+        "sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnm"
+    )
+    txt = txt.replace('\'', '')
+    return txt.translate(maTable).lower()
+
+
+
+
+# letters-only translation to key-pressed characters (latin)
+# ---
+def kbLatinLetter( txt ) :
+    maTable = txt.maketrans(
+        "ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫ",
+        "sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviy"
+    )
+    return txt.translate(maTable).lower()
+
+
+## mark a link to a text
+# conecting them with a dash/minus character
+# ---
+def markLink(lws, text) :
+    text_kb = kbLatinLetter(text.replace(' ', '-'))
+    lws_kb  = kbLatinLetter(lws)
+    try:
+        index_l = text_kb.lower().index(lws_kb.lower())
+    except:
+        return text
+    else:
+        return text[:index_l] + lws + text[index_l + len(lws):]
+
+
+### # --- list of normalized word combinations
+### replaceWords = [
+###     'HEAD & SHOULDERS; HEAD&SOULDERS',
+###     'ΟΛΙΚΗΣ 'ΑΛΕΣΗΣ; Ολικής Άλεσης',
+###     'Χωρίς προσθήκη ζάχαρης; Χωρίς-Ζάχαρη'
+### ]
+
+
+# --- list of linked-words
+linkedWords = [
+    'Χωρίς-Γλουτένη',
+    'Χωρίς-Ζάχαρη',
+    'Χωρίς-Αλάτι',
+    'Χωρίς-Λακτόζη',
+    'Χωρίς-Συντηρητικά',
+    'Χωρίς-Αλκοόλ',
+    'Χωρίς-Kαφεϊνη',
+    'Χωρίς-Γλυκάνισο',
+    'Χωρίς-Ανθρακικό',
+    'Χωρίς-Προσθήκη'
+    'Υψηλής-Παστερίωσης',
+    'Ολικής-Άλεσης',
+    'Ολικής-Aλέσεως',
+    'Χαρτί-Υγείας',
+    'ρολό-υγείας',
+    'χαρτί-τουαλέτας',
+    'Χαρτί-Κουζίνας',
+    'Μπάρες-Δημητριακών',
+    'Μπαρμα-Στάθης',
+    'Coca-Cola'
+    'Aς-Μαγειρέψουμε',
+    'ΚΡΙΣ-ΚΡΙΣ',
+    'ΚΡΙ-ΚΡΙ',
+    'ΕΛ-ΓΚΡΕΚΟ',
+    'FREE-STEP',
+    'EL-SABOR',
+    'LE-PETIT-MARSEILLAIS',
+    'DOUWE-EGBERTS',
+    'ΕΝ-ΕΛΛΑΔΙ',
+    'SPIN-SPAN',
+    'CRETA-FARMS',
+    'CRETA-FARM',
+    'NES-CAFE',
+    'Ολες-τις-Χρήσεις',
+]
+
+
+# text after "all-links" marked
+# ---
+def markLinkedWords(text) :
+    for lw in linkedWords :
+        text = markLink( lw, text )
+    
+    return text
+
+
+## PREPADE (or build) exception objects
+# //////////////////////////////////////////////////////////////////////////////
+
+
+# --- list of possible combos to check
+_2check_kb = []
+_2check = ['με', 'σε', 'για', 'όλες', 'χωρίς' ]
+for it in _2check :
+    _2check_kb.append(kbLatinString(it))
+
+linkedWordsFound = [
+    { 'w' : 'se',  'links' : [] },
+    { 'w' : 'oles-tis',  'links' : [] },
+    { 'w' : 'xvris',  'links' : [] }
+]
+
+def recordLink( parent, child, id ) :
+    if parent != '' :
+        for it in linkedWordsFound :
+            if parent == it['w'] :
+                is_a_new_combo = True
+                for li in it['links'] :
+                    if li['w'] == child :
+                        li['p'].append(id)
+                        li['c'] += 1
+                        is_a_new_combo = False
+                        break
+                if is_a_new_combo :
+                    it['links'].append({ 'w': child,  'p': [ id ],  'c': 1 })
+
+
+
+# --- list of words to exclude from keywords
+# NOTE:
+# APPLIED in PER-WORD base -> after spliting description to words
+removeList = []
+removeOriginals = 'μας του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ')
+for it in removeOriginals :
+    removeList.append(kbLatinString(it))
+
+
+# --- list of synonyms
+# in fact
+synonyms = [
+    'μπίρα μπύρα μπίρες μπύρες',
+    'αυγά αβγά αυγό',
+    'σίκαλης σικάλεως',
+    'ξηρά ξερά',
+    'ρολό ρολλό',
+    'coca-cola cocacola coke',
+    'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας',
+    'χαρτί-κουζίνας ρολό-κουζίνας',
+    'οινος κρασι',
+    'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ',
+    'DR-OETKER OETKER',
+    'DR.BECKMANN BECKMANN',
+    'NES-CAFE NESCAFE',
+    'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής',
+    'τσίπουρο ρακή',
+    'Βρώμη Βρώμης',
+    'Φράουλα Φράουλες Φράουλας',
+    'Μαλλιά Μαλλιών',
+    'Κέικ, Cake',
+    'CRETA-FARMS CRETA-FARM',
+    'MARSEILLAIS LE-PETIT-MARSEILLAIS',
+    'Γαϊδούρας Γαϊδάρου',
+    'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ',
+    'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ',
+    'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ',
+    'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ',
+    'Ντομάτα Ντομάτας',
+    'Ελαφρύ Ελαφρά Light',
+    'Εγχώρια Ελληνικό Ελληνικά',
+    'τριμμένη τριμμένο',
+    'Τόνος Τόνου',
+    'Κριθαρένια κρίθινα'
+]
+
+
+## Read data
+# //////////////////////////////////////////////////////////////////////////////
+
+
+_file = open ('data/eshop-products.json', "r")      # JSON source file
+results_ = json.loads(_file.read())                 # Reading from file
+_file.close()                                       # Closing file
+
+
+t_read = datetime.datetime.now()
+
+# --- Lists to fill
+keywords_ = []     # all data; main exported object
+minilist_ = []
+products_ = []
+
+## keywords format:
+## [
+##    {
+##       w : [ word, word-synonym, ... ],
+##      kb : = kbLatinString(word)
+##       f : 150,
+##       c : [
+##             { w: ['fish', 'fishes'],  f: 150,  p: [122, 254, 907] },
+##             { w: ['juice'],           f: 50,   p: [254, 351] }
+##       ]
+##    },
+##    ...
+## ]
+##
+## --- index:
+## w : words / list of synonyms (str/utf-8)
+# kb : ascii-latin-keypoard format of first item of "w" list
+## f : frequency (int)
+## c : combos / connections (list of objects)
+## p : list of product-ids found in specific words-combination (list of int)
+
+
+records_counter = 0
+## LOOP through the rows to pre-proccess all products
+## ---
+for row in results_ :
+    records_counter += 1
+
+    description = row['Title']      # product description
+    pid  = row['ID']                # product-id
+    fq   = row['freq']              # frequency
+
+    description = cleanText(description)        # clean description string before spliting
+
+    description = markLinkedWords(description)  # ...
+
+    keys = description.split()                  # split to words = keys
+
+    is_combo_key = False
+    combo_key = ''
+
+    # append words (and their combos) to the list
+    for w in keys :
+
+        if kbLatinString(w) in _2check_kb :
+            combo_key = w
+            is_combo_key = True
+        else :
+            if is_combo_key :
+                recordLink(kbLatinString(combo_key), kbLatinString(w), pid)
+                is_combo_key = False
+                combo_key = ''
+
+# print(linkedWordsFound)
+
+# PRINT RESULTS
+# ---
+for ri in linkedWordsFound :
+    print('---', ri['w'], ':', len(ri['links']))
+
+    subtotal = 0
+    for li in ri['links'] :
+        subtotal += li['c']
+
+    for li in ri['links'] :
+        ## if li['c'] > 10 or li['c']/subtotal > .2 :
+        print( ri['w'], li['w'], ' : ', li['c'], ' (', int(li['c']*100/subtotal), '%)' )