scossa/scrapeje
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
scossa
2023-11-27 03:45:23 +01:00
parent c81acebd9a
commit 57cdc465b8
1 changed files with 247 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

345
scrapeje.py
View File

@@ -9,13 +9,6 @@ import re
import os import os
import shutil import shutil
#Inizializzo liste
nome=[]
desc=[]
npezzi=[]
prezzo=[]
prezzoN=[]
restaurant_url='' restaurant_url=''
def inputurl(): def inputurl():
@@ -25,74 +18,53 @@ def inputurl():
print ("\nesempio: https://www.justeat.it/restaurants-pizzeria-la-garganica-bologna/menu") print ("\nesempio: https://www.justeat.it/restaurants-pizzeria-la-garganica-bologna/menu")
restaurant_url = input('INSERISCI IL LINK DELLA PAGINA DEL RISTORANTE: ') restaurant_url = input('INSERISCI IL LINK DELLA PAGINA DEL RISTORANTE: ')
#PER DEBUG #PER DEBUG
#restaurant_url = 'https://www.justeat.it/restaurants-dolceirnerio/menu' #restaurant_url = 'https://www.justeat.it/restaurants-pizzeriadelrondone-bologna/menu'
# if os.path.exists('rubrica.txt'): nome=[]
# if os.path.exists('./DATI_RUBRICA'): desc=[]
inputurl() npezzi=[]
# wr = input('\nVuoi vedere la rubrica? [Y|N] ') prezzo=[]
# if wr.upper() in ['YES', 'Y', 'SI', 'S']: prezzoN=[]
# print( '\n', os.listdir('./DATI_RUBRICA'), '\n') nrist=''
# scelta = input("Scegli un numero esistente o premi Enter per metterre un link: ") restaurant_address=''
# files = os.listdir('./DATI_RUBRICA/') tel=''
# for file in files: result=''
# if scelta in file: def stora_tutto():
# restaurant_url = (file) global nome
# else: global desc
# print ("numero inesistente") global npezzi
# inputurl() global prezzo
# else: global prezzoN
# inputurl() global soup
# else: global nrist
# inputurl() global restaurant_address
# else: global tel
# inputurl() global result
#Stora nome ristorante
nrist = soup.title.text[8:-32]
#SCRAPE #Stora telefono del ristorante
driver = webdriver.Chrome() pattern = re.compile(r'allergenPhoneNumber')
driver.get(restaurant_url) script_tags = soup.find_all('script', string=pattern)
pattern = re.compile(r'"allergenPhoneNumber":"(\d+)"')
wait = WebDriverWait(driver, 16) tel = re.search(pattern, script_tags[0].next)
wait.until(EC.presence_of_element_located((By.CLASS_NAME, "c-menuItems-price"))) if tel:
page = driver.page_source
with open('JEmenu.html', 'w') as f:
f.write(page)
#PARSER
with open('JEmenu.html', 'r') as f:
page = f.read()
soup = BeautifulSoup(page, "html.parser")
#Stora nome ristorante
nrist = soup.title.text[8:-32]
#Stora telefono del ristorante
pattern = re.compile(r'allergenPhoneNumber')
script_tags = soup.find_all('script', string=pattern)
pattern = re.compile(r'"allergenPhoneNumber":"(\d+)"')
tel = re.search(pattern, script_tags[0].next)
if tel:
tel = tel.group(1) tel = tel.group(1)
#Stato ristorante #Stato ristorante
restaurant_is_open = menu = soup.find(attrs={"data-js-test":"order-status-wrapper"}).text restaurant_is_open = menu = soup.find(attrs={"data-js-test":"order-status-wrapper"}).text
restaurant_is_open = restaurant_is_open.replace('\n', ' ') restaurant_is_open = restaurant_is_open.replace('\n', ' ')
regex = r" {4,}" regex = r" {4,}"
result = re.split(regex, restaurant_is_open) result = re.split(regex, restaurant_is_open)
#indirizzo ristorante #indirizzo ristorante
restaurant_address = soup.find(attrs={"data-js-test":"header-restaurantAddress"}).text restaurant_address = soup.find(attrs={"data-js-test":"header-restaurantAddress"}).text
#cicla le schede prodotto #cicla le schede prodotto
menu = soup.find(attrs={"data-test-id": "menu-item"}) menu = soup.find(attrs={"data-test-id": "menu-item"})
for menu in soup.find_all(attrs={"data-test-id": "menu-item"}): for menu in soup.find_all(attrs={"data-test-id": "menu-item"}):
att=menu att=menu
#riempie la lista "nome" #riempie la lista "nome"
for att in menu.find(attrs={"data-test-id": "menu-item-name"}): for att in menu.find(attrs={"data-test-id": "menu-item-name"}):
@@ -132,26 +104,65 @@ for menu in soup.find_all(attrs={"data-test-id": "menu-item"}):
npezzi.append(None) npezzi.append(None)
continue continue
#Chiude chromium
driver.quit()
# #stampa liste driver=''
print("\n") page = ''
for x in range(len(nome)): def scraper():
global driver
global page
global restaurant_url
driver = webdriver.Chrome()
driver.get(restaurant_url)
wait = WebDriverWait(driver, 16)
wait.until(EC.presence_of_element_located((By.CLASS_NAME, "c-menuItems-price")))
page = driver.page_source
with open('JEmenu.html', 'w') as f:
f.write(page)
soup = ''
def parser():
global soup
with open('JEmenu.html', 'r') as f:
page = f.read()
soup = BeautifulSoup(page, "html.parser")
def parserdarubrica():
global parser
global stora_tutto
folder_path = ('./DATI_RUBRICA/')
for filename in os.listdir(folder_path):
if filename.startswith(scelta):
print(filename)
filename = filename.replace(' ', ' ')
print(filename)
shutil.copy (folder_path + filename, f'./JEmenu.html')
parser()
stora_tutto()
else:
print ("numero inesistente")
continue
def stampa_liste():
print("\n")
for x in range(len(nome)):
# print("\n") # print("\n")
print(nome[x]) print(nome[x])
print(desc[x]) print(desc[x])
print(npezzi[x]) print(npezzi[x])
print(prezzo[x]) print(prezzo[x])
#stampa info ristorante def stampa_info():
print("-" * (len(desc)) + "\n") print("-" * (len(desc)) + "\n")
print(nrist) print(nrist)
print(restaurant_address.strip()) print(restaurant_address.strip())
print("Telefono:",tel,"\n") print("Telefono:",tel,"\n")
doppione = "" doppione = ""
for i in range(len(result)): for i in range(len(result)):
if re.search(r"[a-zA-Z]", result[i]): if re.search(r"[a-zA-Z]", result[i]):
if (result[i]) == doppione: if (result[i]) == doppione:
continue continue
@@ -160,12 +171,12 @@ for i in range(len(result)):
elif re.search(r"\d", result[i]): elif re.search(r"\d", result[i]):
print(result[i], result[i+1]) print(result[i], result[i+1])
doppione = (result[i+1]) doppione = (result[i+1])
print("\n" + "-" * (len(desc))+ "\n") print("\n" + "-" * (len(desc))+ "\n")
def genera_prezzoN():
#Genera la lista prezzoN[] che è un clone di "prezzo[] ma con i valori float anzichè string" #Genera la lista prezzoN[] che è un clone di "prezzo[] ma con i valori float anzichè string"
prezzoN = prezzo.copy() prezzoN = prezzo.copy()
for i in range(len(prezzo)): for i in range(len(prezzo)):
if "€" in prezzo[i]: if "€" in prezzo[i]:
prezzoN[i] = float(prezzo[i].replace("€", "").replace(",", ".").replace("da ", "")) prezzoN[i] = float(prezzo[i].replace("€", "").replace(",", ".").replace("da ", ""))
elif "Non" in prezzo[i]: elif "Non" in prezzo[i]:
@@ -175,6 +186,108 @@ for i in range(len(prezzo)):
prezzoN[i] = 99999 prezzoN[i] = 99999
def salvainrubrica():
maxn=0
def trova_nuovo_numero():
global maxn
file_list = os.listdir('./DATI_RUBRICA')
number_list = []
# Estare il numero
for file_name in file_list:
if file_name[0].isdigit():
number_list.append(int(file_name.split('-')[0]))
# trova il massimo
if number_list:
max_number = max(number_list)
maxn=(max_number + 1)
if os.path.exists('rubrica.txt'):
#SE LA RUBRICA ESISTE
with open('rubrica.txt', 'a+') as rubrica:
rubrica.seek(0)
data = rubrica.read()
if restaurant_url not in data:
#QUANDO IL RISTORATE NON E' PRESENTE IN RUBRICA
saveit = input('Vuoi salvare il ristorante in rubrica? [Y|N] ')
#PER DEBUG
#saveit = "y"
if saveit.upper() in ['YES', 'Y', 'SI', 'S']:
os.makedirs("DATI_RUBRICA", exist_ok=True)
trova_nuovo_numero()
nristmax=(str(maxn) + "-" + nrist + '.html')
shutil.move ('JEmenu.html', f'./DATI_RUBRICA/{nristmax}')
if data:
rubrica.write('\n')
rubrica.write(nristmax + '\n')
rubrica.write(nrist + '\n')
rubrica.write(restaurant_url + '\n')
else:
#QUANDO IL RISTORATE E' GIA' PRESENTE IN RUBRICA
os.remove("JEmenu.html")
else:
#QUANDO IL RISTORATE E' GIA' PRESENTE IN RUBRICA
os.remove("JEmenu.html")
else:
#SE LA RUBRICA NON ESISTE
with open('rubrica.txt', 'a+') as rubrica:
rubrica.seek(0)
data = rubrica.read()
#CHIEDE SE SI VUOLE SALVARE ALTRIMENTI PULISCE
saveit = input('Vuoi salvare il ristorante in rubrica? [Y|N] ')
#PER DEBUG
#saveit = "y"
if saveit.upper() in ['YES', 'Y', 'SI', 'S']:
os.makedirs("DATI_RUBRICA", exist_ok=True)
nristmax=("1" + "-" + nrist + '.html')
shutil.move ('JEmenu.html', f'./DATI_RUBRICA/{nristmax}')
if data:
rubrica.write('\n')
rubrica.write(nristmax + '\n')
rubrica.write(nrist + '\n')
rubrica.write(restaurant_url + '\n')
else:
#PULISCE
os.remove("JEmenu.html")
os.remove("rubrica.txt")
#############################################################################################
#############################################################################################
#############################################################################################
if os.path.exists('rubrica.txt') and os.path.exists('./DATI_RUBRICA'):
wr = input('\nVuoi vedere la rubrica? [Y|N] ')
if wr.upper() in ['YES', 'Y', 'SI', 'S']:
print( '\n', os.listdir('./DATI_RUBRICA'), '\n')
scelta = input("Scegli un numero esistente o premi Enter per metterre un link: ")
parserdarubrica()
else:
inputurl()
scraper()
parser()
stora_tutto()
driver.quit()
else:
inputurl()
scraper()
parser()
stora_tutto()
driver.quit()
stampa_liste()
stampa_info()
genera_prezzoN()
salvainrubrica()
# #PER DEBUG # #PER DEBUG
# for x in range(len(nome)): # for x in range(len(nome)):
# print(prezzoN[x]) # print(prezzoN[x])
@@ -223,68 +336,4 @@ for i in range(len(prezzo)):
#PROPORRE DI SALVARE IN RUBRICA
maxn=0
def trova_nuovo_numero():
global maxn
file_list = os.listdir('./DATI_RUBRICA')
number_list = []
# Estare il numero
for file_name in file_list:
if file_name[0].isdigit():
number_list.append(int(file_name.split('-')[0]))
# trova il massimo
if number_list:
max_number = max(number_list)
maxn=(max_number + 1)
if os.path.exists('rubrica.txt'):
#SE LA RUBRICA ESISTE
with open('rubrica.txt', 'a+') as rubrica:
rubrica.seek(0)
data = rubrica.read()
if restaurant_url not in data:
#QUANDO IL RISTORATE NON E' PRESENTE IN RUBRICA
saveit = input('Vuoi salvare il ristorante in rubrica? [Y|N] ')
#PER DEBUG
#saveit = "y"
if saveit.upper() in ['YES', 'Y', 'SI', 'S']:
os.makedirs("DATI_RUBRICA", exist_ok=True)
trova_nuovo_numero()
nristmax=(str(maxn) + "-" + nrist + '.html')
shutil.move ('JEmenu.html', f'./DATI_RUBRICA/{nristmax}')
if data:
rubrica.write('\n')
rubrica.write(nristmax + '\n')
rubrica.write(nrist + '\n')
rubrica.write(restaurant_url + '\n')
else:
#QUANDO IL RISTORATE E' GIA' PRESENTE IN RUBRICA
os.remove("JEmenu.html")
else:
#QUANDO IL RISTORATE E' GIA' PRESENTE IN RUBRICA
os.remove("JEmenu.html")
else:
#SE LA RUBRICA NON ESISTE
with open('rubrica.txt', 'a+') as rubrica:
rubrica.seek(0)
data = rubrica.read()
#CHIEDE SE SI VUOLE SALVARE ALTRIMENTI PULISCE
saveit = input('Vuoi salvare il ristorante in rubrica? [Y|N] ')
#PER DEBUG
#saveit = "y"
if saveit.upper() in ['YES', 'Y', 'SI', 'S']:
os.makedirs("DATI_RUBRICA", exist_ok=True)
nristmax=("1" + "-" + nrist + '.html')
shutil.move ('JEmenu.html', f'./DATI_RUBRICA/{nristmax}')
if data:
rubrica.write('\n')
rubrica.write(nristmax + '\n')
rubrica.write(nrist + '\n')
rubrica.write(restaurant_url + '\n')
else:
#PULISCE
os.remove("JEmenu.html")
os.remove("rubrica.txt")