Skip to content

Remove selenium requirement #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
joschkabirk merged 3 commits into from
Nov 25, 2025
Merged

Remove selenium requirement #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b3c0213
Remove selenium dependency in alsterfood webscraping
joschkabirk Nov 25, 2025
c1a73be
Remove selenium requirement from cfel scraping
joschkabirk Nov 25, 2025
b43f0b5
Update Dockerfile & remove `run_website.sh`
joschkabirk Nov 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ repos:

# python security linter
- repo: https://github.com/PyCQA/bandit
rev: "1.7.1"
rev: "1.8.6"
hooks:
- id: bandit
args: ["-s", "B101"]
Expand Down
7 changes: 1 addition & 6 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM selenium/standalone-chrome
FROM python:3.11.14-trixie

USER root

Expand All @@ -12,8 +12,3 @@ RUN apt-get update && apt-get install -y python3-venv
RUN python3 -m venv venv_container && \
. venv_container/bin/activate && \
pip install -r requirements.txt

COPY . .
RUN chmod +x ./scripts/run_website.sh

CMD ["./scripts/run_website.sh"]
244 changes: 112 additions & 132 deletions lunchbot/alsterfood_scraping.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,9 @@
import re
from datetime import datetime, timedelta

import requests
import yaml
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

from lunchbot.utils import generate_hash

Expand All @@ -28,140 +25,123 @@ def fetch_todays_lunch_menu(url: str):
List of dictionaries with the dishes, prices and info.
Example: [{"name": "Dish 1", "price": "€4.50", "info": "vegan"}, ...]
"""
# Set up Chrome options (adjust as needed)
chrome_options = webdriver.ChromeOptions()
# chrome_options.headless = True # Run Chrome in headless mode (without GUI)
chrome_options.add_argument("--headless=new")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--disable-features=dbus")

# Specify the path to your ChromeDriver executable
# chrome_path = "/opt/homebrew/Caskroom/chromedriver/120.0.6099.71/chromedriver-mac-arm64/chromedriver" # mac
chrome_path = "/usr/bin/chromedriver"

dishes_list = []

# Initialize the Chrome driver
with webdriver.Chrome(
service=ChromeService(executable_path=chrome_path), options=chrome_options
) as driver:
# Navigate to the URL
driver.get(url)

# Wait for up to 10 seconds for an element with ID 'target_element_id' to be present
logger.info("Waiting for page to be ready...")
wait = WebDriverWait(driver, 30)
wait.until(EC.presence_of_element_located((By.ID, "openings")))
logger.info("Page is ready!")

# Get the page source after JavaScript execution
page_source = driver.page_source
# Parse the HTML content using BeautifulSoup
soup = BeautifulSoup(page_source, "html.parser")

# find all the different cards (corresponding to different days)
card_divs = soup.find_all("div", class_="card-content black-text")

# ----------- Find today's card -----------
# Get today's date and select the corresponding card from the menu
today_date_datetime = datetime.now()
# today_date_datetime = datetime(2023, 12, 12) # set a specific date for debugging
today_date = today_date_datetime.strftime("%d.%m.%Y")

todays_card = None
# Iterate through each card div
for card_div in card_divs:
# Check if today's date is in the card-title primary-text
title_div = card_div.find("div", class_="card-title primary-text")
if title_div and today_date in title_div.text:
# Print the original card div
todays_card = card_div

# if no card was found, try to find the next possible date
if todays_card is None:
logger.warning(f"Could not find today's card for date {today_date}")
logger.warning("Trying to find the next possible date...")

# iterate through the upcoming next 7 days - for the first match, take the card
for i in range(1, 8):
next_date = (today_date_datetime + timedelta(days=i)).strftime(
"%d.%m.%Y"
)
logger.info(f"Looking for date {next_date}")
for card_div in card_divs:
# Check if today's date is in the card-title primary-text
title_div = card_div.find("div", class_="card-title primary-text")
if title_div and next_date in title_div.text:
# Print the original card div
todays_card = card_div
logger.info(f"Found date {next_date}")
break
if todays_card is not None:
# Get the page source
page_source = requests.get(url + "/en", timeout=20).text
# Parse the HTML content using BeautifulSoup
soup = BeautifulSoup(page_source, "html.parser")

# find all the different cards (corresponding to different days)
card_divs = soup.find_all("div", class_="card-content black-text")

# ----------- Find today's card -----------
# Get today's date and select the corresponding card from the menu
today_date_datetime = datetime.now()
# today_date_datetime = datetime(2023, 12, 12) # set a specific date for debugging
today_date = today_date_datetime.strftime("%d.%m.%Y")

todays_card = None
# Iterate through each card div
for card_div in card_divs:
# Check if today's date is in the card-title primary-text
title_div = card_div.find("div", class_="card-title primary-text")
if title_div and today_date in title_div.text:
# Print the original card div
todays_card = card_div

# if no card was found, try to find the next possible date
if todays_card is None:
logger.warning(f"Could not find today's card for date {today_date}")
logger.warning("Trying to find the next possible date...")

# iterate through the upcoming next 7 days - for the first match, take the card
for i in range(1, 8):
next_date = (today_date_datetime + timedelta(days=i)).strftime("%d.%m.%Y")
logger.info(f"Looking for date {next_date}")
for card_div in card_divs:
# Check if today's date is in the card-title primary-text
title_div = card_div.find("div", class_="card-title primary-text")
if title_div and next_date in title_div.text:
# Print the original card div
todays_card = card_div
logger.info(f"Found date {next_date}")
break

regex = re.compile("entry entry-item *")
menu_entries_table = todays_card.find_all("table", {"class": regex})

# check which of the charts has the actual means (and not the soups)
for i, entry_list in enumerate(menu_entries_table):
print(entry_list.text)
if "soup" in entry_list.text.lower():
print("Skipping soup entries")
continue
entries_list = entry_list

# debugging (if website structure is changed once again)
# for j, entry in enumerate(entries_list.find_all("tr")):
# logger.info(80 * "-")
# logger.info(f"Entry: {j}")
# logger.info(entry.text)
# logger.info(80 * "-")

n_entries = len(entries_list.find_all("tr")) - 1
logger.info(f"Found {n_entries} entries in the menu for date {today_date}")

# ----------- Find different dishes -----------
# entries are alternating: dish name, dish info + price etc
for i in range(1, n_entries, 2):
logger.info(80 * "-")
logger.info(f"--- Entry {i} ---")
dish_name = entries_list.find_all("tr")[i].text
logger.info(f"Dish name: '{dish_name}'")
dish_info = entries_list.find_all("tr")[i + 1].text
logger.info(f"Dish info: '{dish_info}'")

dish_veg_label = None

if "vegan" in dish_info.lower():
dish_veg_label = "vegan"
elif "vegetarisch" in dish_info.lower():
dish_veg_label = "vegetarian"
else:
dish_veg_label = "meat"
logger.info(f"Veg label: '{dish_veg_label}'")

dish_price = dish_info.split("€")[-1].strip() + " €"
logger.info(f"Dish price: '{dish_price}'")

dishes_list.append(
{
"name": dish_name,
"info": dish_veg_label,
"price": dish_price,
"canteen": "DESY Canteen",
"hash": generate_hash(dish_name),
}
)

# print()
# for dish in dishes_list:
# print(dish)
# print()
if todays_card is not None:
break

regex = re.compile("entry entry-item *")
menu_entries_table = todays_card.find_all("table", {"class": regex})

# check which of the charts has the actual means (and not the soups)
for i, entry_list in enumerate(menu_entries_table):
print(entry_list.text)
if "soup" in entry_list.text.lower():
print("Skipping soup entries")
continue
entries_list = entry_list

# debugging (if website structure is changed once again)
# for j, entry in enumerate(entries_list.find_all("tr")):
# logger.info(80 * "-")
# logger.info(f"Entry: {j}")
# logger.info(entry.text)
# logger.info(80 * "-")

n_entries = len(entries_list.find_all("tr")) - 1
logger.info(f"Found {n_entries} entries in the menu for date {today_date}")

# ----------- Find different dishes -----------
# entries are alternating: dish name, dish info + price etc
for i in range(1, n_entries, 2):
logger.info(80 * "-")
logger.info(f"--- Entry {i} ---")
dish_name = entries_list.find_all("tr")[i].text
logger.info(f"Dish name: '{dish_name}'")
dish_info = entries_list.find_all("tr")[i + 1].text
logger.info(f"Dish info: '{dish_info}'")
# find the div with class "price-text" inside dish_info
dish_price = entries_list.find_all("tr")[i + 1].find(
"div", class_="price-text"
)
if dish_price:
dish_price = dish_price.text.split(" ")[-1].strip() + " €"
else:
dish_price = "N/A"

dish_veg_label = None

if "vegan" in dish_info.lower():
dish_veg_label = "vegan"
elif "vegetarisch" in dish_info.lower():
dish_veg_label = "vegetarian"
else:
dish_veg_label = "meat"
logger.info(f"Veg label: '{dish_veg_label}'")
logger.info(f"Dish price: '{dish_price}'")

dishes_list.append(
{
"name": dish_name,
"info": dish_veg_label,
"price": dish_price,
"canteen": "DESY Canteen",
"hash": generate_hash(dish_name),
}
)

print()
print(yaml.dump(dishes_list, allow_unicode=True))
print()

return dishes_list


if __name__ == "__main__":
# setup logger
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
url = "https://desy.myalsterfood.de/"
todays_menu = fetch_todays_lunch_menu(url)
Loading