web_scrape/scrape.py at main · thisguycodez/web_scrape · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
from urllib.request import urlopen
from bs4 import BeautifulSoup
from mechanize import Browser
from html2text import html2text


# get the url page to scrape
url_2_scrape = "https://www.amazon.com/s?k=shoes+for+men&crid=38H5KG61ITBPE&sprefix=shoes%2Caps%2C156&ref=nb_sb_ss_ts-a-p_4_5"


# declare browser
br = Browser()

# disable the handle robots method
br.set_handle_robots(False)

# add headers
br.addheaders = [('Referer', 'https://www.amazon.com'), ('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]

# open browser emulator
br.open(url_2_scrape)

# parse the html response
html = BeautifulSoup(br.response().read(), 'html.parser')


# list of all main tags to loop thorugh  a-section
body = html.find('body', class_="a-m-us")
list_tags = body.find_all('div', class_="a-section")


# file to save it in
items_file = 'items.csv'
file = open(items_file, 'w')
headers = 'item, price \n'
file.write(headers)


# loop through cards to find title and price
for tag in list_tags:
	# from here is where the magic happens


# span tag -(item) a-size-base-plus
	item = tag.find('span', class_="a-size-base-plus")

# span tag -(price) a-price
	price = tag.find('span', class_="a-price")

	cur_line = f"{item}, {price}"

	# using html2text to remove tags
	cur_line = html2text(cur_line)


	# avoiding 'None' values
	if 'None' in cur_line:
		continue
	else:
		print(cur_line)
		file.write(cur_line)


file.close()