scossa/scrapeje
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
scossa
2023-11-25 20:38:36 +01:00
parent a6eeb26534
commit 2a3fddea3d
2 changed files with 46 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
scrapeje.py
View File

@@ -1,14 +1,15 @@
import re import re
#import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
#import cloudscraper
#import json
from selenium import webdriver from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
#import requests
#import cloudscraper
#import json
#Inizializzo liste
nome=[] nome=[]
desc=[] desc=[]
npezzi=[] npezzi=[]
@@ -17,43 +18,23 @@ prezzof=[]
scripto=[] scripto=[]
#INPUT
#prende l url della pagina justeat del ristorante in input #prende l url della pagina justeat del ristorante in input
#url = "https://www.justeat.it/restaurants-pizzeria-girasole-bologna/menu" #restaurant_url = input('link della pagina justeat del ristorante: ')
#url = input('link della pagina justeat del ristorante: ')
#input ('inserisci il link della pagina justeat del ristorante: ')
# print("\n") #PER DEBUG
# url = print(Fore.WHITE + Style.DIM + "es https://www.justeat.it/NOME_RISTORANTE/menu" + Style.RESET_ALL)
# url = input("inserisci il link della pagina justeat del ristorante: ")
#scrape html scavalcando cloudflare
# scraper = cloudscraper.create_scraper(browser={'browser': 'firefox','platform': 'windows','mobile': False})
# page = scraper.get(url).content #usa input manuale
# page = scraper.get("https://www.justeat.it/restaurants-saporedialeppo/menu").content #usa input automatico
#crea il file html
# headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
# page = requests.get(url, headers=headers).content
restaurant_url = "https://www.justeat.it/restaurants-saporedialeppo/menu"
restaurant_url = 'https://www.justeat.it/restaurants-gelateria-ice-cream-casalecchio-di-reno-40033/menu'
restaurant_url = 'https://www.justeat.it/restaurants-pizzeria-la-garganica-bologna/menu' restaurant_url = 'https://www.justeat.it/restaurants-pizzeria-la-garganica-bologna/menu'
#restaurant_url = 'https://www.justeat.it/restaurants-viavaipizzaekebab/menu'
#INIZIALLIZZO PARSER
driver = webdriver.Chrome() driver = webdriver.Chrome()
driver.get(restaurant_url) driver.get(restaurant_url)
wait = WebDriverWait(driver, 10) wait = WebDriverWait(driver, 10)
wait.until(EC.presence_of_element_located((By.CLASS_NAME, "c-menuItems-price--offline"))) wait.until(EC.presence_of_element_located((By.CLASS_NAME, "c-menuItems-price--offline")))
page = driver.page_source page = driver.page_source
with open('JEmenu.html', 'w') as f: with open('JEmenu.html', 'w') as f:
f.write(page) f.write(page)
@@ -61,40 +42,28 @@ with open('JEmenu.html', 'w') as f:
with open('JEmenu.html', 'r') as f: with open('JEmenu.html', 'r') as f:
page = f.read() page = f.read()
#parser #PARSER
soup = BeautifulSoup(page, "html.parser") soup = BeautifulSoup(page, "html.parser")
#Stora nome ristorante #Stora nome ristorante
nrist = soup.title.text[8:-32] nrist = soup.title.text[8:-32]
#Stora telefono del ristorante
# #Stora telefono del ristorante
# Define the regex pattern
pattern = re.compile(r'allergenPhoneNumber') pattern = re.compile(r'allergenPhoneNumber')
script_tags = soup.find_all('script', string=pattern)
# Find all script tags that match the regex pattern
script_tags = soup.find_all('script', text=pattern)
# Define the regex pattern
pattern = re.compile(r'"allergenPhoneNumber":"(\d+)"') pattern = re.compile(r'"allergenPhoneNumber":"(\d+)"')
# Search for the phone number in the given string
tel = re.search(pattern, script_tags[0].next) tel = re.search(pattern, script_tags[0].next)
if tel: if tel:
tel = tel.group(1) tel = tel.group(1)
#Stato ristorante #Stato ristorante
restaurant_is_open = menu = soup.find(attrs={"data-js-test":"order-status-wrapper"}).text restaurant_is_open = menu = soup.find(attrs={"data-js-test":"order-status-wrapper"}).text
restaurant_is_open = restaurant_is_open.replace('\n', ' ') restaurant_is_open = restaurant_is_open.replace('\n', ' ')
regex = r" {4,}" regex = r" {4,}"
result = re.split(regex, restaurant_is_open) result = re.split(regex, restaurant_is_open)
#indirizzo ristorante
restaurant_address = soup.find(attrs={"data-js-test":"header-restaurantAddress"}).text restaurant_address = soup.find(attrs={"data-js-test":"header-restaurantAddress"}).text
#cicla le schede prodotto #cicla le schede prodotto
@@ -110,7 +79,6 @@ for menu in soup.find_all(attrs={"data-test-id": "menu-item"}):
else: else:
continue continue
#riempie la lista "desc" #riempie la lista "desc"
att=menu.find("p", class_="c-menuItems-description") att=menu.find("p", class_="c-menuItems-description")
if att != None: if att != None:
@@ -119,10 +87,10 @@ for menu in soup.find_all(attrs={"data-test-id": "menu-item"}):
else: else:
desc.append(None) desc.append(None)
#riempie la lista "prezzo"
for att in menu.find(attrs={"data-js-test": "menu-item-price"}):
prezzo.append(att.lstrip().splitlines()[0])
#riempie la lista "prezzo"
#for att in menu.find(attrs={"data-js-test": "menu-item-price"}):
# prezzo.append(att.lstrip().splitlines()[0])
#riempie la lista "npezzi" #riempie la lista "npezzi"
@@ -134,23 +102,25 @@ for menu in soup.find_all(attrs={"data-test-id": "menu-item"}):
npezzi.append(None) npezzi.append(None)
continue continue
# #stampa liste # #stampa liste
for x in range(len(nome)): for x in range(len(nome)):
print("\n") print("\n")
print(nome[x]) print(nome[x])
print(desc[x]) print(desc[x])
print(npezzi[x]) print(npezzi[x])
print(prezzo[x]) # print(prezzo[x])
regex = r"(?:da\s+)?([\d.]+)" regex = r"(?:da\s+)?([\d.]+)"
match = re.search(regex, prezzo[x]) # match = re.search(regex, prezzo[x])
if match: # if match:
numero = match.group(1) # numero = match.group(1)
print(numero) # print(numero)
prezzof.append(int(numero)) # prezzof.append(int(numero))
else: # else:
prezzof.append(0) # prezzof.append(0)
print(prezzof[x]) # print(prezzof[x])
#stampa lunghezza liste #stampa lunghezza liste
print("\n") print("\n")
@@ -171,19 +141,17 @@ print("telefono:",tel)
print("lista nome:",len(nome)) print("lista nome:",len(nome))
print("lista desc:",len(desc)) print("lista desc:",len(desc))
print("lista npezzi:",len(npezzi)) print("lista npezzi:",len(npezzi))
print("lista prezzi:",len(prezzo)) #sono stringhe ovvero ci sono anche prezzi come "da 1,00 €" (servirà formattarla in double per poter fare i conti) #print("lista prezzi:",len(prezzo)) #sono stringhe ovvero ci sono anche prezzi come "da 1,00 €" (servirà formattarla in double per poter fare i conti)
# Generazione del codice HTML # Generazione del codice HTML
html = "<html><body>" html = "<html><body>"
for x in range(len(nome)): for x in range(len(nome)):
html += "<h2>" + nome[x] + "</h2>" html += "<h2>" + nome[x] + "</h2>"
# html += "<p>" + desc[x] + "</p>" # html += "<p>" + desc[x] + "</p>"
html += "<p>N. pezzi disponibili: " + str(npezzi[x]) + "</p>" html += "<p>N. pezzi disponibili: " + str(npezzi[x]) + "</p>"
html += "<p>Prezzo: €" + str(prezzo[x]) + "</p>" # html += "<p>Prezzo: €" + str(prezzo[x]) + "</p>"
html += "<button onclick=\"aggiungiProdotto('" + nome[x] + "', '" + str(prezzof[x]) + "')\">+1</button>" # html += "<button onclick=\"aggiungiProdotto('" + nome[x] + "', '" + str(prezzof[x]) + "')\">+1</button>"
html += "<br><br>" html += "<br><br>"
html += "<br><hr><h2>Prodotti aggiunti</h2>" html += "<br><hr><h2>Prodotti aggiunti</h2>"
@@ -197,6 +165,7 @@ html += "}"
html += "</script>" html += "</script>"
html += "</body></html>" html += "</body></html>"
# Salvataggio su file # Salvataggio su file
with open("pagina.html", "w") as file: with open("pagina.html", "w") as file:
file.write(html) file.write(html)

17
setup.sh Normal file → Executable file
View File

@@ -1,8 +1,15 @@
#!/bin/sh #!/bin/bash
python -m venv .venv
source .venv/bin/activate
#pip3 install jawanndenn #pip3 install requests
pip3 install requests #pip3 install BeautifulSoup4
pip3 install BeautifulSoup4 #pip3 install colorama
#pip3 install selenium
#NON PIU UTILIZZATI
#pip3 install cloudscraper #pip3 install cloudscraper
#pip3 install json #pip3 install json
pip3 install colorama
#echo "USA LO SCRIPT ALL'INTERNO DEL VENV"
#echo "PER USCIRE SCRIVI 'deactivated'"