scrapeje/scrapeje.py

#IMPORTO MODULI
#selenium: scraper | bs4: parser | re: regex | os: interazione con os | shutil: interazione con la shell
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import re
import os
import shutil
from jinja2 import Template
import time

restaurant_url=''
driver=''
page = ''
nrist=''
restaurant_address=''
tel=''
result=''
soup = ''
nristmax = '0'
nome=[]
desc=[]
npezzi=[]
prezzo=[]
prezzoN=[]


def inputurl():  
    global restaurant_url
    global nristmax
    #INPUT
    #prende l url della pagina justeat del ristorante in input
    print ("\nesempio: https://www.justeat.it/restaurants-pizzeria-la-garganica-bologna/menu")
    restaurant_url = input('INSERISCI IL LINK DELLA PAGINA DEL RISTORANTE: ')
    # PER DEBUG
    #restaurant_url = 'https://www.justeat.it/restaurants-pizzeriadelrondone-bologna/menu'


def animazione():
    def typewriter(text, delay=0.1):
            for letter in text:
                print(letter, end='', flush=True)
                time.sleep(delay)
            print()

    typewriter("▁ ▂ ▃ ▄ ▅ ▆ ▇ █ ▇ ▆ ▅ ▄ ▃ ▁", 0.09)


def scraper():
    global driver
    global page
    global restaurant_url

    #FIREFOX
    opts = webdriver.FirefoxOptions()
    opts.headless = True
    driver = webdriver.Firefox(options=opts)
    driver.get(restaurant_url)

    #CHROMIUM/CHROME
    # driver = webdriver.Chrome()
    # driver.get(restaurant_url)  
     
    animazione() 

    wait = WebDriverWait(driver, 16)
    wait.until(EC.presence_of_element_located((By.CLASS_NAME, "c-menuItems-price")))

    page = driver.page_source

    with open('JEmenu.html', 'w') as f:
        f.write(page)


def parser():
    global soup
    with open('JEmenu.html', 'r') as f:
        page = f.read()

    soup = BeautifulSoup(page, "html.parser")


def stora_tutto():
    global nome
    global desc
    global npezzi
    global prezzo
    global prezzoN
    global soup
    global nrist
    global restaurant_address
    global tel
    global result

    #Stora nome ristorante
    nrist = soup.title.text[8:-32]

    #Stora telefono del ristorante
    pattern = re.compile(r'allergenPhoneNumber')
    script_tags = soup.find_all('script', string=pattern)
    pattern = re.compile(r'"allergenPhoneNumber":"(\d+)"')
    tel = re.search(pattern, script_tags[0].next)
    if tel:
        tel = tel.group(1)

    #Stato ristorante
    restaurant_is_open = menu = soup.find(attrs={"data-js-test":"order-status-wrapper"}).text
    restaurant_is_open = restaurant_is_open.replace('\n', ' ')
    regex = r" {4,}"
    result = re.split(regex, restaurant_is_open)

    #indirizzo ristorante
    restaurant_address = soup.find(attrs={"data-js-test":"header-restaurantAddress"}).text

    #cicla le schede prodotto
    menu = soup.find(attrs={"data-test-id": "menu-item"})
    for menu in soup.find_all(attrs={"data-test-id": "menu-item"}):
        att=menu
        #riempie la lista "nome" 
        for att in menu.find(attrs={"data-test-id": "menu-item-name"}):
            if att != type(None):
                if att != " ":
                    nome.append(att.lstrip().splitlines()[0])
                    break
            else:
                continue

        #riempie la lista "desc" 
        att=menu.find("p", class_="c-menuItems-description")
        if att != None:
            for att in menu.find("p", class_="c-menuItems-description"):
                desc.append(att.lstrip().splitlines()[0])
        else:
            desc.append(None)

        #riempie la lista "prezzo" 
        for att in menu.find(attrs={"data-js-test": "menu-item-price"}):
            #prezzo.append(att.lstrip().splitlines()[0])
            counter=0
            if att != " " and counter % 2 == 0:
                prezzo.append(att.lstrip())
                counter+=1
            else:
                counter+=1
                continue
            continue
        
        #riempie la lista "npezzi" 
        att=menu.find_all(attrs={"data-test-id": "menu-item-description"})
        if att != None:
            if menu.text.find("pezzo") > 0 or menu.text.find("pezzi") > 0:
                npezzi.append(menu.text.splitlines()[7].lstrip())
            else:
                npezzi.append(None)
        continue

def parserdarubrica():
    global parser
    global stora_tutto
    folder_path = ('./DATI_RUBRICA/')
    for filename in os.listdir(folder_path):
        if filename.startswith(scelta):
            print(filename)                    
            filename = filename.replace(' ', ' ')
            print(filename)
            shutil.copy (folder_path + filename, f'./JEmenu.html')
            parser()                    
            stora_tutto()
        else:
            print ("numero inesistente")
            continue  


def stampa_liste():   #PER DEBUG
    print("\n")
    for x in range(len(nome)):
        #  print("\n")
        print(nome[x])
        print(desc[x])
        print(npezzi[x])
        print(prezzo[x])


def stampa_info():
    print("-" * (len(desc)) + "\n")
    print(nrist)
    print(restaurant_address.strip())
    print("Telefono:",tel,"\n")

    doppione = ""
    for i in range(len(result)):
        if re.search(r"[a-zA-Z]", result[i]):
            if (result[i]) == doppione:
                continue
            else:
                print(result[i])
        elif re.search(r"\d", result[i]):
            print(result[i], result[i+1])
            doppione = (result[i+1])
    print("\n" + "-" * (len(desc))+ "\n") 


def genera_prezzoN():
    #Genera la lista prezzoN[] che è un clone di "prezzo[] ma con i valori float anzichè string"
    global prezzoN
    prezzoN = prezzo.copy()
    for i in range(len(prezzo)):
        if "€" in prezzo[i]:
            prezzoN[i] = float(prezzo[i].replace("€", "").replace(",", ".").replace("da ", ""))
        elif "Non" in prezzo[i]:
            prezzoN[i] = 0 
            prezzoN[i] = float(prezzoN[i])
        else:
            prezzoN[i] = 99999
    # for x in range(len(nome)): #PER DEBUG
    #     print(prezzoN[x])

html=''
def genera_frontend():
    # IMPOSTA E GENERA pagina2.html CON IL FILE htmlpage.html 
    global nome
    global desc
    global npezzi
    global prezzo
    global prezzoN
    global html
    with open("template.html", "r") as file:
        template_content = file.read()

    template = Template(template_content)
    html = template.render(
        nrist=nrist, 
        nome=nome,
        desc=desc,
        npezzi=npezzi,
        prezzo=prezzo,
        prezzoN=prezzoN
        )
    with open("pagina2.html", "w") as file:
        file.write(html)


def salvainrubrica():
    global nristmax
    maxn=0
    def trova_nuovo_numero():  
        global maxn
        file_list = os.listdir('./DATI_RUBRICA')
        number_list = []
        # Estare il numero
        for file_name in file_list:
            if file_name[0].isdigit():  
                number_list.append(int(file_name.split('-')[0])) 
        # trova il massimo
        if number_list:
            max_number = max(number_list)
            maxn=(max_number + 1) 

    if os.path.exists('rubrica.txt'):
        #SE LA RUBRICA ESISTE
        with open('rubrica.txt', 'a+') as rubrica:
            rubrica.seek(0)
            data = rubrica.read()
            if restaurant_url not in data:
                #QUANDO IL RISTORATE NON E' PRESENTE IN RUBRICA
                saveit = input('Vuoi salvare il ristorante in rubrica? [Y|N] ')
                #PER DEBUG
                #saveit = "y"
                if saveit.upper() in ['YES', 'Y', 'SI', 'S']: 
                    os.makedirs("DATI_RUBRICA", exist_ok=True)
                    trova_nuovo_numero()
                    nristmax=(str(maxn) + "-" + nrist + '.html')
                    shutil.move ('JEmenu.html', f'./DATI_RUBRICA/{nristmax}')
                    if data:
                        rubrica.write('\n')
                    rubrica.write(nristmax + '\n')
                    rubrica.write(nrist + '\n')
                    rubrica.write(restaurant_url + '\n')
                    print ('\nIl ristorante"',nrist + '" è stato salvato in rubrica\n')
                else:
                    #QUANDO IL RISTORATE E' GIA' PRESENTE IN RUBRICA
                    os.remove("JEmenu.html")
            else:
                #QUANDO IL RISTORATE E' GIA' PRESENTE IN RUBRICA
                os.remove("JEmenu.html")
        
    else:
        #SE LA RUBRICA NON ESISTE
        with open('rubrica.txt', 'a+') as rubrica:
            rubrica.seek(0)
            data = rubrica.read()
            #CHIEDE SE SI VUOLE SALVARE ALTRIMENTI PULISCE
            saveit = input('Vuoi salvare il ristorante in rubrica? [Y|N] ')
            #PER DEBUG
            #saveit = "y"
            if saveit.upper() in ['YES', 'Y', 'SI', 'S']:        
                os.makedirs("DATI_RUBRICA", exist_ok=True)
                nristmax=("1" + "-" + nrist + '.html')
                shutil.move ('JEmenu.html', f'./DATI_RUBRICA/{nristmax}')

                if data:
                    rubrica.write('\n')
                rubrica.write(nristmax + '\n')
                rubrica.write(nrist + '\n')
                rubrica.write(restaurant_url + '\n')
                print ('\nIl ristorante "' + nrist + '" è stato salvato in rubrica')
                print ('(Riavvia lo script per ordinare da rubrica)\n')
            else:
                #PULISCE
                os.remove("JEmenu.html")
                os.remove("rubrica.txt")
    


#############################################################################################
#############################################################################################
#############################################################################################
    

if os.path.exists('rubrica.txt') and os.path.exists('./DATI_RUBRICA'):

        #wr = input('\nVuoi vedere la rubrica? [Y|N] ')
        wr = 's'
        if wr.upper() in ['YES', 'Y', 'SI', 'S']: 
            print( '\n', os.listdir('./DATI_RUBRICA'), '\n')

            #scelta = input("Scegli un numero esistente o premi Enter per metterre un link: ")            
            scelta = '1'
            parserdarubrica()

        else:
            inputurl()
            scraper()
            parser()
            stora_tutto()  
            driver.quit()
else:
    inputurl()
    scraper()
    parser() 
    stora_tutto()  
    driver.quit()

stampa_liste()
stampa_info()
genera_prezzoN()    
salvainrubrica()
genera_frontend()

driver = webdriver.Firefox()
animazione()
driver.get("file://" + os.path.abspath("pagina2.html"))




# #PER DEBUG
# for x in range(len(prezzoN)):
#     print(prezzoN[x])
#     print("\n" + "-" * 25 + "\n") 
#     print ("lista prezzo:  ",len(prezzo))
#     print ("lista prezzoN: ",len(prezzoN))
#     print ("lista nome:    ",len(nome))
#     print("\n" + "-" * 25 + "\n") 


# #PER DEBUG
#stampa lunghezza liste
# print("lista nome:    ",len(nome))
# print("lista desc:    ",len(desc))
# print("lista npezzi:  ",len(npezzi))
# print("lista prezzi:  ",len(prezzo)) #sono stringhe
# print("lista prezziN: ",len(prezzoN)) #sono numeri
# print("\n" + "-" * 25 + "\n") 

#FA SCHIFO
# html += '   var popup = window.open("", "Popup", "width=200,height=100,top=" + ((window.innerHeight - 100) / 2) + ",left=" + ((window.innerWidth - 200) / 2));' 
# html += '   popup.document.write("<p>Prodotto aggiunto</p>");'                      
# html += '   setTimeout(function(){ popup.close(); }, 1000);'