IngredientsProject/Ingredients.py at main · jwhowell/IngredientsProject · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
from bs4 import BeautifulSoup
import requests
import json
import random
import time
import pandas as pd
import csv

base ="https://www.halfbakedharvest.com/sitemap_index.xml"
header = {"User-Agent": "ingredient_bot/1.0 (+mailto:jakehowell@duck.com; for educational purposes)"}

def parseXML(url):
    site_mapXML = requests.get(url, headers = header,)

    soup = BeautifulSoup(site_mapXML.text, "xml")

    site_maps = []
    for loc_tag in soup.find_all("loc"):
        loc_tag = loc_tag.get_text(strip = True)
        if loc_tag[-3:] == "jpg" or loc_tag[-3:] == "png" or loc_tag[-4:] == "jpeg":
            continue
        elif loc_tag[-3:] != "jpg" or loc_tag[-3:] == "png" and loc_tag not in site_maps:
            site_maps.append(loc_tag)
    return site_maps

site_maps = parseXML(base)
print(site_maps)

df = pd.read_csv("output.csv")

s = requests.Session()

errlog = []
for site_map in site_maps:
    links = []
    links.append(parseXML(site_map))
    number_of_links = len(links[0])
    for url in links[0]:
        number_of_links -= 1
        print(f"{number_of_links} more links.")

        if url in set(df['url'].values):
            print(f"Url: {url}, already exists in dataset")
            continue
        resp = s.get(url, headers = header)
        soup = BeautifulSoup(resp.text, "html.parser")

        scripts = soup.find("script", type = "application/ld+json")
        try:
            data = json.loads(scripts.string)
            info = data["@graph"][0]
            headline = info["headline"]
            keywords = info["keywords"]
            date_published = info["datePublished"]

            ingredients = soup.find_all("span", class_ = "wprm-recipe-ingredient-name")
            ingredient_list = [span.get_text(strip=True) for span in ingredients]

        except AttributeError:
            errlog.append([url, "AttributeError"])
            continue

        except KeyError:
            errlog.append([url, "KeyError"])
            continue

        if ingredient_list == []:
            continue

        if url not in df['url']:
            df.loc[len(df)] = [headline, date_published, url, ingredient_list, keywords]

        resp.raise_for_status()  # raise for 4xx/5xx
        print(f"Success: {url} (status {resp.status_code})")

        delay = random.uniform(1, 5)
        print(f"Sleeping {delay:.2f}s before next request...")
        time.sleep(delay)

df["ingredients"] = df["ingredients"].apply(json.dumps)
df["keywords"] = df["keywords"].apply(json.dumps)

df.to_csv("output.csv", index= False)
with open("err.csv", "a", newline= '') as file:
    writer = csv.writer(file)
    writer.writerow(errlog)

print(df.info())