path: root/python/check-linked.py

## LIBRARIES
# //////////////////////////////////////////////////////////////////////////////

# import pandas as pd               # pandas for excel reading
import re       # regex
import json     # json
import os.path  # ...
import datetime

t0_ = datetime.datetime.now()


## LOCAL FUNCTIONS
# //////////////////////////////////////////////////////////////////////////////


## Clean Text ...
# -> removes some general/neutral words and symbols
# -> ignores some in-line characters 
# -> also strips spare spaces
# function is applied onto the full title/description
# ---
def cleanText(x) :
    ignoreList = '" ( ) [ ]'.split(' ')

    for r in ignoreList :
        x = x.replace(r, ' ')
       
    x = x.replace('    ', ' ')      # remove spare spaces
    x = x.replace('   ', ' ')
    x = x.replace('  ', ' ')

    return x.replace('  ', ' ')     # one lase (just in case)


# isSignificant
# decides if the term is significant to be indexed;
# a term is significant if does not contain digit-chars [0-9], comma (,) or period (.)
# ---
def isSignificant(x) :
    # fisrts exclude some notable exceptions (mostly brands)
    if x in ['7UP', '3ΑΛΦΑ', '17'] :
        return True

    return not bool( re.match("\S*\d+\S*", x) )


def kbLatinString( txt ) :
    maTable = txt.maketrans(
        "ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫQWERTYUIOPASDFGHJKLZXCVBNM",
        "sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviyqwertyuiopasdfghjklzxcvbnm"
    )
    txt = txt.replace('\'', '')
    return txt.translate(maTable).lower()




# letters-only translation to key-pressed characters (latin)
# ---
def kbLatinLetter( txt ) :
    maTable = txt.maketrans(
        "ςερτυθιοπασδφγηξκλζχψωβνμΕΡΤΥΘΙΟΠΑΣΔΦΓΗΞΚΛΖΧΨΩΒΝΜάέήίόύώϊΐϋΆΈΉΊΌΎΏΪΫ",
        "sertyuiopasdfghjklzxcvbnmertyuiopasdfghjklzxcvbnmaehioyviiyaehioyviy"
    )
    return txt.translate(maTable).lower()


## mark a link to a text
# conecting them with a dash/minus character
# ---
def markLink(lws, text) :
    text_kb = kbLatinLetter(text.replace(' ', '-'))
    lws_kb  = kbLatinLetter(lws)
    try:
        index_l = text_kb.lower().index(lws_kb.lower())
    except:
        return text
    else:
        return text[:index_l] + lws + text[index_l + len(lws):]


### # --- list of normalized word combinations
### replaceWords = [
###     'HEAD & SHOULDERS; HEAD&SOULDERS',
###     'ΟΛΙΚΗΣ 'ΑΛΕΣΗΣ; Ολικής Άλεσης',
###     'Χωρίς προσθήκη ζάχαρης; Χωρίς-Ζάχαρη'
### ]


# --- list of linked-words
linkedWords = [
    'Χωρίς-Γλουτένη',
    'Χωρίς-Ζάχαρη',
    'Χωρίς-Αλάτι',
    'Χωρίς-Λακτόζη',
    'Χωρίς-Συντηρητικά',
    'Χωρίς-Αλκοόλ',
    'Χωρίς-Kαφεϊνη',
    'Χωρίς-Γλυκάνισο',
    'Χωρίς-Ανθρακικό',
    'Χωρίς-Προσθήκη'
    'Υψηλής-Παστερίωσης',
    'Ολικής-Άλεσης',
    'Ολικής-Aλέσεως',
    'Χαρτί-Υγείας',
    'ρολό-υγείας',
    'χαρτί-τουαλέτας',
    'Χαρτί-Κουζίνας',
    'Μπάρες-Δημητριακών',
    'Μπαρμα-Στάθης',
    'Coca-Cola'
    'Aς-Μαγειρέψουμε',
    'ΚΡΙΣ-ΚΡΙΣ',
    'ΚΡΙ-ΚΡΙ',
    'ΕΛ-ΓΚΡΕΚΟ',
    'FREE-STEP',
    'EL-SABOR',
    'LE-PETIT-MARSEILLAIS',
    'DOUWE-EGBERTS',
    'ΕΝ-ΕΛΛΑΔΙ',
    'SPIN-SPAN',
    'CRETA-FARMS',
    'CRETA-FARM',
    'NES-CAFE',
    'Ολες-τις-Χρήσεις',
]


# text after "all-links" marked
# ---
def markLinkedWords(text) :
    for lw in linkedWords :
        text = markLink( lw, text )
    
    return text


## PREPADE (or build) exception objects
# //////////////////////////////////////////////////////////////////////////////


# --- list of possible combos to check
_2check_kb = []
_2check = ['με', 'σε', 'για', 'όλες', 'χωρίς' ]
for it in _2check :
    _2check_kb.append(kbLatinString(it))

linkedWordsFound = [
    { 'w' : 'se',  'links' : [] },
    { 'w' : 'oles-tis',  'links' : [] },
    { 'w' : 'xvris',  'links' : [] }
]

def recordLink( parent, child, id ) :
    if parent != '' :
        for it in linkedWordsFound :
            if parent == it['w'] :
                is_a_new_combo = True
                for li in it['links'] :
                    if li['w'] == child :
                        li['p'].append(id)
                        li['c'] += 1
                        is_a_new_combo = False
                        break
                if is_a_new_combo :
                    it['links'].append({ 'w': child,  'p': [ id ],  'c': 1 })



# --- list of words to exclude from keywords
# NOTE:
# APPLIED in PER-WORD base -> after spliting description to words
removeList = []
removeOriginals = 'μας του της των από στο στον & r s ft l τ e g h k m n o p s x'.split(' ')
for it in removeOriginals :
    removeList.append(kbLatinString(it))


# --- list of synonyms
# in fact
synonyms = [
    'μπίρα μπύρα μπίρες μπύρες',
    'αυγά αβγά αυγό',
    'σίκαλης σικάλεως',
    'ξηρά ξερά',
    'ρολό ρολλό',
    'coca-cola cocacola coke',
    'χαρτί-υγείας ρολό-υγείας χαρτί-τουαλέτας',
    'χαρτί-κουζίνας ρολό-κουζίνας',
    'οινος κρασι',
    'ΚΑΤΣΕΛΗΣ ΚΑΤΣΕΛΗ',
    'DR-OETKER OETKER',
    'DR.BECKMANN BECKMANN',
    'NES-CAFE NESCAFE',
    'Ολικής-Άλεσης Ολικής-Aλέσεως Ολικής',
    'τσίπουρο ρακή',
    'Βρώμη Βρώμης',
    'Φράουλα Φράουλες Φράουλας',
    'Μαλλιά Μαλλιών',
    'Κέικ, Cake',
    'CRETA-FARMS CRETA-FARM',
    'MARSEILLAIS LE-PETIT-MARSEILLAIS',
    'Γαϊδούρας Γαϊδάρου',
    'ΚΑΛΟΓΕΡΑΚΗΣ ΚΑΛΟΓΕΡΑΚΗ',
    'ΚΑΪΔΑΝΤΖΗΣ ΚΑΪΔΑΝΤΖΗ',
    'ΥΦΑΝΤΗΣ ΥΦΑΝΤΗ',
    'ΣΥΝΑΓΡΙΔΑ ΣΥΝΑΓΡΙΔΕΣ',
    'Ντομάτα Ντομάτας',
    'Ελαφρύ Ελαφρά Light',
    'Εγχώρια Ελληνικό Ελληνικά',
    'τριμμένη τριμμένο',
    'Τόνος Τόνου',
    'Κριθαρένια κρίθινα'
]


## Read data
# //////////////////////////////////////////////////////////////////////////////


_file = open ('data/eshop-products.json', "r")      # JSON source file
results_ = json.loads(_file.read())                 # Reading from file
_file.close()                                       # Closing file


t_read = datetime.datetime.now()

# --- Lists to fill
keywords_ = []     # all data; main exported object
minilist_ = []
products_ = []

## keywords format:
## [
##    {
##       w : [ word, word-synonym, ... ],
##      kb : = kbLatinString(word)
##       f : 150,
##       c : [
##             { w: ['fish', 'fishes'],  f: 150,  p: [122, 254, 907] },
##             { w: ['juice'],           f: 50,   p: [254, 351] }
##       ]
##    },
##    ...
## ]
##
## --- index:
## w : words / list of synonyms (str/utf-8)
# kb : ascii-latin-keypoard format of first item of "w" list
## f : frequency (int)
## c : combos / connections (list of objects)
## p : list of product-ids found in specific words-combination (list of int)


records_counter = 0
## LOOP through the rows to pre-proccess all products
## ---
for row in results_ :
    records_counter += 1

    description = row['Title']      # product description
    pid  = row['ID']                # product-id
    fq   = row['freq']              # frequency

    description = cleanText(description)        # clean description string before spliting

    description = markLinkedWords(description)  # ...

    keys = description.split()                  # split to words = keys

    is_combo_key = False
    combo_key = ''

    # append words (and their combos) to the list
    for w in keys :

        if kbLatinString(w) in _2check_kb :
            combo_key = w
            is_combo_key = True
        else :
            if is_combo_key :
                recordLink(kbLatinString(combo_key), kbLatinString(w), pid)
                is_combo_key = False
                combo_key = ''

# print(linkedWordsFound)

# PRINT RESULTS
# ---
for ri in linkedWordsFound :
    print('---', ri['w'], ':', len(ri['links']))

    subtotal = 0
    for li in ri['links'] :
        subtotal += li['c']

    for li in ri['links'] :
        ## if li['c'] > 10 or li['c']/subtotal > .2 :
        print( ri['w'], li['w'], ' : ', li['c'], ' (', int(li['c']*100/subtotal), '%)' )