#IMPORTO MODULI #selenium: scraper | bs4: parser | re: regex | os: interazione con os | shutil: interazione con la shell from selenium import webdriver from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By from bs4 import BeautifulSoup import re import os import shutil from jinja2 import Template import time restaurant_url='' driver='' page = '' nrist='' restaurant_address='' tel='' result='' soup = '' nristmax = '0' nome=[] desc=[] npezzi=[] prezzo=[] prezzoN=[] def inputurl(): global restaurant_url global nristmax #INPUT #prende l url della pagina justeat del ristorante in input print ("\nesempio: https://www.justeat.it/restaurants-pizzeria-la-garganica-bologna/menu") restaurant_url = input('INSERISCI IL LINK DELLA PAGINA DEL RISTORANTE: ') # PER DEBUG #restaurant_url = 'https://www.justeat.it/restaurants-pizzeriadelrondone-bologna/menu' def animazione(): def typewriter(text, delay=0.1): for letter in text: print(letter, end='', flush=True) time.sleep(delay) print() typewriter("▁ ▂ ▃ ▄ ▅ ▆ ▇ █ ▇ ▆ ▅ ▄ ▃ ▁", 0.09) def scraper(): global driver global page global restaurant_url #FIREFOX opts = webdriver.FirefoxOptions() opts.headless = True driver = webdriver.Firefox(options=opts) driver.get(restaurant_url) #CHROMIUM/CHROME # driver = webdriver.Chrome() # driver.get(restaurant_url) animazione() def typewriter(text, delay=0.1): for letter in text: print(letter, end='', flush=True) time.sleep(delay) print() typewriter("▁ ▂ ▃ ▄ ▅ ▆ ▇ █ ▇ ▆ ▅ ▄ ▃ ▁", 0.09) wait = WebDriverWait(driver, 16) wait.until(EC.presence_of_element_located((By.CLASS_NAME, "c-menuItems-price"))) page = driver.page_source with open('JEmenu.html', 'w') as f: f.write(page) def parser(): global soup with open('JEmenu.html', 'r') as f: page = f.read() soup = BeautifulSoup(page, "html.parser") def stora_tutto(): global nome global desc global npezzi global prezzo global prezzoN global soup global nrist global restaurant_address global tel global result #Stora nome ristorante nrist = soup.title.text[8:-32] #Stora telefono del ristorante pattern = re.compile(r'allergenPhoneNumber') script_tags = soup.find_all('script', string=pattern) pattern = re.compile(r'"allergenPhoneNumber":"(\d+)"') tel = re.search(pattern, script_tags[0].next) if tel: tel = tel.group(1) #Stato ristorante restaurant_is_open = menu = soup.find(attrs={"data-js-test":"order-status-wrapper"}).text restaurant_is_open = restaurant_is_open.replace('\n', ' ') regex = r" {4,}" result = re.split(regex, restaurant_is_open) #indirizzo ristorante restaurant_address = soup.find(attrs={"data-js-test":"header-restaurantAddress"}).text #cicla le schede prodotto menu = soup.find(attrs={"data-test-id": "menu-item"}) for menu in soup.find_all(attrs={"data-test-id": "menu-item"}): att=menu #riempie la lista "nome" for att in menu.find(attrs={"data-test-id": "menu-item-name"}): if att != type(None): if att != " ": nome.append(att.lstrip().splitlines()[0]) break else: continue #riempie la lista "desc" att=menu.find("p", class_="c-menuItems-description") if att != None: for att in menu.find("p", class_="c-menuItems-description"): desc.append(att.lstrip().splitlines()[0]) else: desc.append(None) #riempie la lista "prezzo" for att in menu.find(attrs={"data-js-test": "menu-item-price"}): #prezzo.append(att.lstrip().splitlines()[0]) counter=0 if att != " " and counter % 2 == 0: prezzo.append(att.lstrip()) counter+=1 else: counter+=1 continue continue #riempie la lista "npezzi" att=menu.find_all(attrs={"data-test-id": "menu-item-description"}) if att != None: if menu.text.find("pezzo") > 0 or menu.text.find("pezzi") > 0: npezzi.append(menu.text.splitlines()[7].lstrip()) else: npezzi.append(None) continue def parserdarubrica(): global parser global stora_tutto folder_path = ('./DATI_RUBRICA/') for filename in os.listdir(folder_path): if filename.startswith(scelta): print(filename) filename = filename.replace(' ', ' ') print(filename) shutil.copy (folder_path + filename, f'./JEmenu.html') parser() stora_tutto() else: print ("numero inesistente") continue def stampa_liste(): #PER DEBUG print("\n") for x in range(len(nome)): # print("\n") print(nome[x]) print(desc[x]) print(npezzi[x]) print(prezzo[x]) def stampa_info(): print("-" * (len(desc)) + "\n") print(nrist) print(restaurant_address.strip()) print("Telefono:",tel,"\n") doppione = "" for i in range(len(result)): if re.search(r"[a-zA-Z]", result[i]): if (result[i]) == doppione: continue else: print(result[i]) elif re.search(r"\d", result[i]): print(result[i], result[i+1]) doppione = (result[i+1]) print("\n" + "-" * (len(desc))+ "\n") def genera_prezzoN(): #Genera la lista prezzoN[] che è un clone di "prezzo[] ma con i valori float anzichè string" global prezzoN prezzoN = prezzo.copy() for i in range(len(prezzo)): if "€" in prezzo[i]: prezzoN[i] = float(prezzo[i].replace("€", "").replace(",", ".").replace("da ", "")) elif "Non" in prezzo[i]: prezzoN[i] = 0 prezzoN[i] = float(prezzoN[i]) else: prezzoN[i] = 99999 # for x in range(len(nome)): #PER DEBUG # print(prezzoN[x]) html='' def genera_frontend(): # IMPOSTA E GENERA pagina2.html CON IL FILE htmlpage.html global nome global desc global npezzi global prezzo global prezzoN global html with open("template.html", "r") as file: template_content = file.read() template = Template(template_content) html = template.render( nrist=nrist, nome=nome, desc=desc, npezzi=npezzi, prezzo=prezzo, prezzoN=prezzoN ) with open("pagina2.html", "w") as file: file.write(html) def salvainrubrica(): global nristmax maxn=0 def trova_nuovo_numero(): global maxn file_list = os.listdir('./DATI_RUBRICA') number_list = [] # Estare il numero for file_name in file_list: if file_name[0].isdigit(): number_list.append(int(file_name.split('-')[0])) # trova il massimo if number_list: max_number = max(number_list) maxn=(max_number + 1) if os.path.exists('rubrica.txt'): #SE LA RUBRICA ESISTE with open('rubrica.txt', 'a+') as rubrica: rubrica.seek(0) data = rubrica.read() if restaurant_url not in data: #QUANDO IL RISTORATE NON E' PRESENTE IN RUBRICA saveit = input('Vuoi salvare il ristorante in rubrica? [Y|N] ') #PER DEBUG #saveit = "y" if saveit.upper() in ['YES', 'Y', 'SI', 'S']: os.makedirs("DATI_RUBRICA", exist_ok=True) trova_nuovo_numero() nristmax=(str(maxn) + "-" + nrist + '.html') shutil.move ('JEmenu.html', f'./DATI_RUBRICA/{nristmax}') if data: rubrica.write('\n') rubrica.write(nristmax + '\n') rubrica.write(nrist + '\n') rubrica.write(restaurant_url + '\n') print ('\nIl ristorante"',nrist + '" è stato salvato in rubrica\n') else: #QUANDO IL RISTORATE E' GIA' PRESENTE IN RUBRICA os.remove("JEmenu.html") else: #QUANDO IL RISTORATE E' GIA' PRESENTE IN RUBRICA os.remove("JEmenu.html") else: #SE LA RUBRICA NON ESISTE with open('rubrica.txt', 'a+') as rubrica: rubrica.seek(0) data = rubrica.read() #CHIEDE SE SI VUOLE SALVARE ALTRIMENTI PULISCE saveit = input('Vuoi salvare il ristorante in rubrica? [Y|N] ') #PER DEBUG #saveit = "y" if saveit.upper() in ['YES', 'Y', 'SI', 'S']: os.makedirs("DATI_RUBRICA", exist_ok=True) nristmax=("1" + "-" + nrist + '.html') shutil.move ('JEmenu.html', f'./DATI_RUBRICA/{nristmax}') if data: rubrica.write('\n') rubrica.write(nristmax + '\n') rubrica.write(nrist + '\n') rubrica.write(restaurant_url + '\n') print ('\nIl ristorante "' + nrist + '" è stato salvato in rubrica') print ('(Riavvia lo script per ordinare da rubrica)\n') else: #PULISCE os.remove("JEmenu.html") os.remove("rubrica.txt") os.remove("logo.gif") ############################################################################################# ############################################################################################# ############################################################################################# if os.path.exists('rubrica.txt') and os.path.exists('./DATI_RUBRICA'): #wr = input('\nVuoi vedere la rubrica? [Y|N] ') wr = 's' if wr.upper() in ['YES', 'Y', 'SI', 'S']: print( '\n', os.listdir('./DATI_RUBRICA'), '\n') scelta = input("Scegli un numero esistente o premi Enter per metterre un link: ") #scelta = '1' parserdarubrica() else: inputurl() scraper() parser() stora_tutto() driver.quit() else: inputurl() scraper() parser() stora_tutto() driver.quit() stampa_liste() stampa_info() genera_prezzoN() salvainrubrica() genera_frontend() driver = webdriver.Firefox() driver.get("file://" + os.path.abspath("pagina2.html")) # #PER DEBUG # for x in range(len(prezzoN)): # print(prezzoN[x]) # print("\n" + "-" * 25 + "\n") # print ("lista prezzo: ",len(prezzo)) # print ("lista prezzoN: ",len(prezzoN)) # print ("lista nome: ",len(nome)) # print("\n" + "-" * 25 + "\n") # #PER DEBUG #stampa lunghezza liste # print("lista nome: ",len(nome)) # print("lista desc: ",len(desc)) # print("lista npezzi: ",len(npezzi)) # print("lista prezzi: ",len(prezzo)) #sono stringhe # print("lista prezziN: ",len(prezzoN)) #sono numeri # print("\n" + "-" * 25 + "\n") #FA SCHIFO # html += ' var popup = window.open("", "Popup", "width=200,height=100,top=" + ((window.innerHeight - 100) / 2) + ",left=" + ((window.innerWidth - 200) / 2));' # html += ' popup.document.write("

Prodotto aggiunto

");' # html += ' setTimeout(function(){ popup.close(); }, 1000);'