diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2367f98..d297934 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: # python security linter - repo: https://github.com/PyCQA/bandit - rev: "1.7.1" + rev: "1.8.6" hooks: - id: bandit args: ["-s", "B101"] diff --git a/Dockerfile b/Dockerfile index b8a6b87..b3d06ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM selenium/standalone-chrome +FROM python:3.11.14-trixie USER root @@ -12,8 +12,3 @@ RUN apt-get update && apt-get install -y python3-venv RUN python3 -m venv venv_container && \ . venv_container/bin/activate && \ pip install -r requirements.txt - -COPY . . -RUN chmod +x ./scripts/run_website.sh - -CMD ["./scripts/run_website.sh"] diff --git a/lunchbot/alsterfood_scraping.py b/lunchbot/alsterfood_scraping.py index 6aa3bd6..7a0514e 100644 --- a/lunchbot/alsterfood_scraping.py +++ b/lunchbot/alsterfood_scraping.py @@ -2,12 +2,9 @@ import re from datetime import datetime, timedelta +import requests +import yaml from bs4 import BeautifulSoup -from selenium import webdriver -from selenium.webdriver.chrome.service import Service as ChromeService -from selenium.webdriver.common.by import By -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.support.ui import WebDriverWait from lunchbot.utils import generate_hash @@ -28,140 +25,123 @@ def fetch_todays_lunch_menu(url: str): List of dictionaries with the dishes, prices and info. Example: [{"name": "Dish 1", "price": "€4.50", "info": "vegan"}, ...] """ - # Set up Chrome options (adjust as needed) - chrome_options = webdriver.ChromeOptions() - # chrome_options.headless = True # Run Chrome in headless mode (without GUI) - chrome_options.add_argument("--headless=new") - chrome_options.add_argument("--no-sandbox") - chrome_options.add_argument("--disable-gpu") - chrome_options.add_argument("--disable-features=dbus") - - # Specify the path to your ChromeDriver executable - # chrome_path = "/opt/homebrew/Caskroom/chromedriver/120.0.6099.71/chromedriver-mac-arm64/chromedriver" # mac - chrome_path = "/usr/bin/chromedriver" dishes_list = [] - - # Initialize the Chrome driver - with webdriver.Chrome( - service=ChromeService(executable_path=chrome_path), options=chrome_options - ) as driver: - # Navigate to the URL - driver.get(url) - - # Wait for up to 10 seconds for an element with ID 'target_element_id' to be present - logger.info("Waiting for page to be ready...") - wait = WebDriverWait(driver, 30) - wait.until(EC.presence_of_element_located((By.ID, "openings"))) - logger.info("Page is ready!") - - # Get the page source after JavaScript execution - page_source = driver.page_source - # Parse the HTML content using BeautifulSoup - soup = BeautifulSoup(page_source, "html.parser") - - # find all the different cards (corresponding to different days) - card_divs = soup.find_all("div", class_="card-content black-text") - - # ----------- Find today's card ----------- - # Get today's date and select the corresponding card from the menu - today_date_datetime = datetime.now() - # today_date_datetime = datetime(2023, 12, 12) # set a specific date for debugging - today_date = today_date_datetime.strftime("%d.%m.%Y") - - todays_card = None - # Iterate through each card div - for card_div in card_divs: - # Check if today's date is in the card-title primary-text - title_div = card_div.find("div", class_="card-title primary-text") - if title_div and today_date in title_div.text: - # Print the original card div - todays_card = card_div - - # if no card was found, try to find the next possible date - if todays_card is None: - logger.warning(f"Could not find today's card for date {today_date}") - logger.warning("Trying to find the next possible date...") - - # iterate through the upcoming next 7 days - for the first match, take the card - for i in range(1, 8): - next_date = (today_date_datetime + timedelta(days=i)).strftime( - "%d.%m.%Y" - ) - logger.info(f"Looking for date {next_date}") - for card_div in card_divs: - # Check if today's date is in the card-title primary-text - title_div = card_div.find("div", class_="card-title primary-text") - if title_div and next_date in title_div.text: - # Print the original card div - todays_card = card_div - logger.info(f"Found date {next_date}") - break - if todays_card is not None: + # Get the page source + page_source = requests.get(url + "/en", timeout=20).text + # Parse the HTML content using BeautifulSoup + soup = BeautifulSoup(page_source, "html.parser") + + # find all the different cards (corresponding to different days) + card_divs = soup.find_all("div", class_="card-content black-text") + + # ----------- Find today's card ----------- + # Get today's date and select the corresponding card from the menu + today_date_datetime = datetime.now() + # today_date_datetime = datetime(2023, 12, 12) # set a specific date for debugging + today_date = today_date_datetime.strftime("%d.%m.%Y") + + todays_card = None + # Iterate through each card div + for card_div in card_divs: + # Check if today's date is in the card-title primary-text + title_div = card_div.find("div", class_="card-title primary-text") + if title_div and today_date in title_div.text: + # Print the original card div + todays_card = card_div + + # if no card was found, try to find the next possible date + if todays_card is None: + logger.warning(f"Could not find today's card for date {today_date}") + logger.warning("Trying to find the next possible date...") + + # iterate through the upcoming next 7 days - for the first match, take the card + for i in range(1, 8): + next_date = (today_date_datetime + timedelta(days=i)).strftime("%d.%m.%Y") + logger.info(f"Looking for date {next_date}") + for card_div in card_divs: + # Check if today's date is in the card-title primary-text + title_div = card_div.find("div", class_="card-title primary-text") + if title_div and next_date in title_div.text: + # Print the original card div + todays_card = card_div + logger.info(f"Found date {next_date}") break - - regex = re.compile("entry entry-item *") - menu_entries_table = todays_card.find_all("table", {"class": regex}) - - # check which of the charts has the actual means (and not the soups) - for i, entry_list in enumerate(menu_entries_table): - print(entry_list.text) - if "soup" in entry_list.text.lower(): - print("Skipping soup entries") - continue - entries_list = entry_list - - # debugging (if website structure is changed once again) - # for j, entry in enumerate(entries_list.find_all("tr")): - # logger.info(80 * "-") - # logger.info(f"Entry: {j}") - # logger.info(entry.text) - # logger.info(80 * "-") - - n_entries = len(entries_list.find_all("tr")) - 1 - logger.info(f"Found {n_entries} entries in the menu for date {today_date}") - - # ----------- Find different dishes ----------- - # entries are alternating: dish name, dish info + price etc - for i in range(1, n_entries, 2): - logger.info(80 * "-") - logger.info(f"--- Entry {i} ---") - dish_name = entries_list.find_all("tr")[i].text - logger.info(f"Dish name: '{dish_name}'") - dish_info = entries_list.find_all("tr")[i + 1].text - logger.info(f"Dish info: '{dish_info}'") - - dish_veg_label = None - - if "vegan" in dish_info.lower(): - dish_veg_label = "vegan" - elif "vegetarisch" in dish_info.lower(): - dish_veg_label = "vegetarian" - else: - dish_veg_label = "meat" - logger.info(f"Veg label: '{dish_veg_label}'") - - dish_price = dish_info.split("€")[-1].strip() + " €" - logger.info(f"Dish price: '{dish_price}'") - - dishes_list.append( - { - "name": dish_name, - "info": dish_veg_label, - "price": dish_price, - "canteen": "DESY Canteen", - "hash": generate_hash(dish_name), - } - ) - - # print() - # for dish in dishes_list: - # print(dish) - # print() + if todays_card is not None: + break + + regex = re.compile("entry entry-item *") + menu_entries_table = todays_card.find_all("table", {"class": regex}) + + # check which of the charts has the actual means (and not the soups) + for i, entry_list in enumerate(menu_entries_table): + print(entry_list.text) + if "soup" in entry_list.text.lower(): + print("Skipping soup entries") + continue + entries_list = entry_list + + # debugging (if website structure is changed once again) + # for j, entry in enumerate(entries_list.find_all("tr")): + # logger.info(80 * "-") + # logger.info(f"Entry: {j}") + # logger.info(entry.text) + # logger.info(80 * "-") + + n_entries = len(entries_list.find_all("tr")) - 1 + logger.info(f"Found {n_entries} entries in the menu for date {today_date}") + + # ----------- Find different dishes ----------- + # entries are alternating: dish name, dish info + price etc + for i in range(1, n_entries, 2): + logger.info(80 * "-") + logger.info(f"--- Entry {i} ---") + dish_name = entries_list.find_all("tr")[i].text + logger.info(f"Dish name: '{dish_name}'") + dish_info = entries_list.find_all("tr")[i + 1].text + logger.info(f"Dish info: '{dish_info}'") + # find the div with class "price-text" inside dish_info + dish_price = entries_list.find_all("tr")[i + 1].find( + "div", class_="price-text" + ) + if dish_price: + dish_price = dish_price.text.split(" ")[-1].strip() + " €" + else: + dish_price = "N/A" + + dish_veg_label = None + + if "vegan" in dish_info.lower(): + dish_veg_label = "vegan" + elif "vegetarisch" in dish_info.lower(): + dish_veg_label = "vegetarian" + else: + dish_veg_label = "meat" + logger.info(f"Veg label: '{dish_veg_label}'") + logger.info(f"Dish price: '{dish_price}'") + + dishes_list.append( + { + "name": dish_name, + "info": dish_veg_label, + "price": dish_price, + "canteen": "DESY Canteen", + "hash": generate_hash(dish_name), + } + ) + + print() + print(yaml.dump(dishes_list, allow_unicode=True)) + print() return dishes_list if __name__ == "__main__": + # setup logger + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) url = "https://desy.myalsterfood.de/" todays_menu = fetch_todays_lunch_menu(url) diff --git a/lunchbot/cfel_scraping.py b/lunchbot/cfel_scraping.py index 18a1f11..e8331b0 100644 --- a/lunchbot/cfel_scraping.py +++ b/lunchbot/cfel_scraping.py @@ -1,10 +1,10 @@ import logging +import requests +import yaml from bs4 import BeautifulSoup -from selenium import webdriver -from selenium.webdriver.chrome.service import Service as ChromeService -from lunchbot.utils import translate_german_food_description_to_english, generate_hash +from lunchbot.utils import generate_hash, translate_german_food_description_to_english logger = logging.getLogger(__name__) @@ -23,92 +23,71 @@ def fetch_todays_lunch_menu(url: str): List of dictionaries with the dishes, prices and info. Example: [{"name": "Dish 1", "price": "€4.50", "info": "vegan"}, ...] """ - # Set up Chrome options (adjust as needed) - chrome_options = webdriver.ChromeOptions() - # chrome_options.headless = True # Run Chrome in headless mode (without GUI) - chrome_options.add_argument("--headless=new") - chrome_options.add_argument("--no-sandbox") - chrome_options.add_argument("--disable-gpu") - chrome_options.add_argument("--disable-features=dbus") - - # Specify the path to your ChromeDriver executable - # chrome_path = "/opt/homebrew/Caskroom/chromedriver/120.0.6099.71/chromedriver-mac-arm64/chromedriver" # mac - chrome_path = "/usr/bin/chromedriver" - dishes_list = [] - # Initialize the Chrome driver - with webdriver.Chrome( - service=ChromeService(executable_path=chrome_path), options=chrome_options - ) as driver: - # Navigate to the URL - driver.get(url) - - # Wait for up to 10 seconds for an element with ID 'target_element_id' to be present - logger.info("Waiting for page to be ready...") - # wait = WebDriverWait(driver, 20) - # wait.until(EC.presence_of_element_located((By.ID, "openings"))) - logger.info("Page is ready!") - - # Get the page source after JavaScript execution - page_source = driver.page_source - # Parse the HTML content using BeautifulSoup - soup = BeautifulSoup(page_source, "html.parser") - - # find all the different cards (corresponding to different days) - card_divs = soup.find_all("div", class_="aw-meal row no-margin-xs") - - for hit in card_divs: - dish_name = None - for d in hit.find_all("p", class_="aw-meal-description"): - if dish_name is None: - dish_name = d.text - else: - raise ValueError("Multiple meal descriptions found") - - # check if the dish is vegetarian or vegan - vegetarian = None - vegan = None - for d in hit.find_all("p", class_="small aw-meal-attributes"): - if vegetarian is None: - vegetarian = "vegetarian" in d.text.lower() - if vegan is None: - vegan = "vegan" in d.text.lower() - - dish_price = None - for d in hit.find_all("div", class_="col-sm-2 no-padding-xs aw-meal-price"): - if dish_price is None: - dish_price = d.text - else: - raise ValueError("Multiple meal prices found") - - logger.info(20 * "-") - logger.info(dish_name) - logger.info(f"vegetarian {vegetarian}") - logger.info(f"vegan: {vegan}") - logger.info(20 * "-") - - if vegan: - dish_info = "vegan" - elif vegetarian: - dish_info = "vegetarian" + # Get the page source + page_source = requests.get(url, timeout=20).text + # Parse the HTML content using BeautifulSoup + soup = BeautifulSoup(page_source, "html.parser") + + # find all the different cards (corresponding to different days) + card_divs = soup.find_all("div", class_="aw-meal row no-margin-xs") + + for hit in card_divs: + dish_name = None + for d in hit.find_all("p", class_="aw-meal-description"): + if dish_name is None: + dish_name = d.text + else: + raise ValueError("Multiple meal descriptions found") + + # check if the dish is vegetarian or vegan + vegetarian = None + vegan = None + for d in hit.find_all("p", class_="small aw-meal-attributes"): + if vegetarian is None: + vegetarian = "vegetarian" in d.text.lower() + if vegan is None: + vegan = "vegan" in d.text.lower() + + dish_price = None + for d in hit.find_all("div", class_="col-sm-2 no-padding-xs aw-meal-price"): + if dish_price is None: + dish_price = d.text.replace(",", ".") else: - dish_info = "" - - # Translate the dish description to English - dish_name_translated = translate_german_food_description_to_english(dish_name) - - dishes_list.append( - { - "name": dish_name_translated, - "info": dish_info, - "price": dish_price, - "canteen": "Cafe CFEL", - "hash": generate_hash(dish_name), - } - ) - - logger.info(80 * "-") + raise ValueError("Multiple meal prices found") + + logger.info(20 * "-") + logger.info(dish_name) + logger.info(f"vegetarian {vegetarian}") + logger.info(f"vegan: {vegan}") + logger.info(20 * "-") + + if vegan: + dish_info = "vegan" + elif vegetarian: + dish_info = "vegetarian" + else: + dish_info = "" + + # Translate the dish description to English + dish_name_translated = translate_german_food_description_to_english(dish_name) + + dishes_list.append( + { + "name": dish_name_translated, + "info": dish_info, + "price": dish_price, + "canteen": "Cafe CFEL", + "hash": generate_hash(dish_name), + } + ) + + logger.info(80 * "-") + + print() + print(yaml.dump(dishes_list, allow_unicode=True)) + print() return dishes_list diff --git a/lunchbot/utils.py b/lunchbot/utils.py index 3906018..d9c4d16 100644 --- a/lunchbot/utils.py +++ b/lunchbot/utils.py @@ -1,9 +1,13 @@ """Utils for lunchbot.""" +import hashlib import logging +import os +from dotenv import load_dotenv from openai import OpenAI -import hashlib + +load_dotenv() # Loads variables from .env file logger = logging.getLogger(__name__) @@ -42,7 +46,7 @@ def translate_german_food_description_to_english( The answer or a tuple of the prompt and the answer (if return_prompt_answer=True). """ - client = OpenAI() + client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) if system_content is None: system_content = ( diff --git a/scripts/run_website.sh b/scripts/run_website.sh deleted file mode 100755 index 1821f30..0000000 --- a/scripts/run_website.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# run this script in the root of the repo to handle pythonpath correctly -# i.e. do ./scripts/run_website.sh in the root of the repo - -source setup.sh -python lunchbot/website.py